sync update from LAN repo (source ji @ 85078a31)

This commit is contained in:
gozeros committed 2026-10-21 15:09:16 +08:00
1 parent 5d6627af50
commit e602d88695
864 files changed
+99150 -6346

No files matched your search

@@ -0,0 +1,197 @@
"""User-owned analysis projects and immutable project/thread bindings."""
from __future__ import annotations
from copy import deepcopy
from datetime import UTC, datetime
from uuid import uuid4
from sqlalchemy import ForeignKey, String, Text, delete, select
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
MODES = frozenset({"qa", "strategy", "solution"})
CONFIG_FIELDS = frozenset({"agent_id", "report_config", "strategy_config"})
EDITABLE_FIELDS = frozenset({"name", "background", "knowledge_base_ids", "assistant_knowledge_base_ids"}) | CONFIG_FIELDS
class AnalysisProjectConfigRow(Base):
"""Additive table: upgrades do not alter existing project/thread tables."""
__tablename__ = "analysis_project_configs"
key: Mapped[str] = mapped_column(String(80), primary_key=True)
config: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
class AnalysisProjectRow(Base):
__tablename__ = "analysis_projects"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
user_id: Mapped[str] = mapped_column(String(255), index=True)
name: Mapped[str] = mapped_column(String(160))
mode: Mapped[str] = mapped_column(String(16))
background: Mapped[str] = mapped_column(Text, default="")
knowledge_base_ids: Mapped[list] = mapped_column(PortableJSON(), default=list)
assistant_knowledge_base_ids: Mapped[list] = mapped_column(PortableJSON(), default=list)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime())
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime())
class AnalysisProjectThreadRow(Base):
__tablename__ = "analysis_project_threads"
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
project_id: Mapped[str] = mapped_column(ForeignKey("analysis_projects.id"), index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime())
def _dict(row):
data = row.to_dict()
for key in ("created_at", "updated_at"):
if isinstance(data.get(key), datetime):
data[key] = data[key].isoformat()
return deepcopy(data)
class AnalysisProjectStore:
"""SQL in persistent deployments; isolated in-memory fallback for memory mode.
Every public project operation requires an explicit owner. The unscoped
thread lookup is only for the Gateway's ownership and run-policy checks.
Mode has no update path, and bindings are separate from client-editable metadata.
"""
def __init__(self, session_factory=None):
self._sf = session_factory
self._projects: dict[str, dict] = {}
self._threads: dict[str, dict] = {}
self._configs: dict[str, dict] = {}
async def config_for(self, key: str) -> dict:
if not self._sf:
return deepcopy(self._configs.get(key, {}))
async with self._sf() as session:
row = await session.get(AnalysisProjectConfigRow, key)
return deepcopy(row.config) if row else {}
async def _enrich(self, project: dict | None) -> dict | None:
if project is None:
return None
return {
**project,
"agent_id": None,
"report_config": {},
"strategy_config": {},
**await self.config_for(f"project:{project['id']}"),
}
async def create(self, user_id: str, *, mode: str, **fields) -> dict:
if mode not in MODES or set(fields) - EDITABLE_FIELDS:
raise ValueError("Invalid project fields")
now = datetime.now(UTC)
row = AnalysisProjectRow(
id=str(uuid4()),
user_id=user_id,
mode=mode,
name=fields["name"],
background=fields.get("background", ""),
knowledge_base_ids=list(fields.get("knowledge_base_ids", [])),
assistant_knowledge_base_ids=list(fields.get("assistant_knowledge_base_ids", [])),
created_at=now,
updated_at=now,
)
if self._sf:
async with self._sf() as session:
session.add(row)
session.add(AnalysisProjectConfigRow(key=f"project:{row.id}", config={key: deepcopy(fields[key]) for key in CONFIG_FIELDS if key in fields}))
await session.commit()
else:
self._projects[row.id] = _dict(row)
self._configs[f"project:{row.id}"] = {key: deepcopy(fields[key]) for key in CONFIG_FIELDS if key in fields}
return await self._enrich(_dict(row))
async def list(self, user_id: str) -> list[dict]:
if not self._sf:
return [await self._enrich(deepcopy(p)) for p in sorted(self._projects.values(), key=lambda p: p["updated_at"], reverse=True) if p["user_id"] == user_id]
async with self._sf() as session:
result = await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.user_id == user_id).order_by(AnalysisProjectRow.updated_at.desc()))
rows = [_dict(row) for row in result.scalars()]
return [await self._enrich(row) for row in rows]
async def get(self, project_id: str, user_id: str) -> dict | None:
if not self._sf:
row = self._projects.get(project_id)
return await self._enrich(deepcopy(row)) if row and row["user_id"] == user_id else None
async with self._sf() as session:
row = (await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.id == project_id, AnalysisProjectRow.user_id == user_id))).scalar_one_or_none()
data = _dict(row) if row else None
return await self._enrich(data)
async def update(self, project_id: str, user_id: str, **fields) -> dict | None:
if set(fields) - EDITABLE_FIELDS:
raise ValueError("项目模式创建后不可更改")
if not self._sf:
row = self._projects.get(project_id)
if not row or row["user_id"] != user_id:
return None
row.update({key: deepcopy(value) for key, value in fields.items() if key not in CONFIG_FIELDS}, updated_at=datetime.now(UTC).isoformat())
self._configs.setdefault(f"project:{project_id}", {}).update({key: deepcopy(value) for key, value in fields.items() if key in CONFIG_FIELDS})
return await self._enrich(deepcopy(row))
async with self._sf() as session:
row = (await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.id == project_id, AnalysisProjectRow.user_id == user_id).with_for_update())).scalar_one_or_none()
if row is None:
return None
for key, value in fields.items():
if key not in CONFIG_FIELDS:
setattr(row, key, deepcopy(value))
options = await session.get(AnalysisProjectConfigRow, f"project:{project_id}")
config = {**(options.config if options else {}), **{key: deepcopy(value) for key, value in fields.items() if key in CONFIG_FIELDS}}
await session.merge(AnalysisProjectConfigRow(key=f"project:{project_id}", config=config))
row.updated_at = datetime.now(UTC)
await session.commit()
data = _dict(row)
return await self._enrich(data)
async def bind_thread(self, project_id: str, user_id: str, thread_id: str, *, agent_id: str | None = None) -> None:
if await self.get(project_id, user_id) is None:
raise ValueError("Project not found")
row = AnalysisProjectThreadRow(thread_id=thread_id, project_id=project_id, created_at=datetime.now(UTC))
if not self._sf:
if thread_id in self._threads:
raise ValueError("Thread already bound")
self._threads[thread_id] = _dict(row)
self._configs[f"thread:{thread_id}"] = {"agent_id": agent_id}
return
async with self._sf() as session:
session.add(row)
session.add(AnalysisProjectConfigRow(key=f"thread:{thread_id}", config={"agent_id": agent_id}))
await session.commit()
async def project_for_thread(self, thread_id: str) -> dict | None:
if not self._sf:
binding = self._threads.get(thread_id)
return await self._enrich(deepcopy(self._projects.get(binding["project_id"]))) if binding else None
async with self._sf() as session:
row = (await session.execute(select(AnalysisProjectRow).join(AnalysisProjectThreadRow).where(AnalysisProjectThreadRow.thread_id == thread_id))).scalar_one_or_none()
data = _dict(row) if row else None
return await self._enrich(data)
async def thread_ids(self, project_id: str, user_id: str) -> list[str]:
if await self.get(project_id, user_id) is None:
return []
if not self._sf:
return [key for key, value in reversed(list(self._threads.items())) if value["project_id"] == project_id]
async with self._sf() as session:
return list((await session.execute(select(AnalysisProjectThreadRow.thread_id).where(AnalysisProjectThreadRow.project_id == project_id).order_by(AnalysisProjectThreadRow.created_at.desc()))).scalars())
async def unbind_failed_creation(self, thread_id: str) -> None:
if not self._sf:
self._threads.pop(thread_id, None)
self._configs.pop(f"thread:{thread_id}", None)
return
async with self._sf() as session:
await session.execute(delete(AnalysisProjectThreadRow).where(AnalysisProjectThreadRow.thread_id == thread_id))
await session.execute(delete(AnalysisProjectConfigRow).where(AnalysisProjectConfigRow.key == f"thread:{thread_id}"))
await session.commit()
@@ -1,4 +1,21 @@
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
_assistant_knowledge_store = None
def register_assistant_knowledge_store(store) -> None:
global _assistant_knowledge_store
_assistant_knowledge_store = store
def get_assistant_knowledge_store():
return _assistant_knowledge_store
__all__ = [
name
for name in globals()
if name.startswith("AssistantKnowledge")
or name in {"make_assistant_knowledge_store", "register_assistant_knowledge_store", "get_assistant_knowledge_store"}
]
@@ -1,7 +1,7 @@
"""ORM model for the global business → agent / chain mapping (业务链条管理).
A "business mapping" row binds a fixed business code (3Q / 6BF / 6-1…6-4 / 7BF /
8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
A "business mapping" row binds a fixed business code (3Q / 6BF / 3q-1…3q-5 /
6-1…6-4 / 7BF / 7-1…7-4 / 8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
account can resolve it). Used to auto-pick the business chain when a task page
launches the multi-agent roundtable.
@@ -21,7 +21,7 @@ _UNSET = object()
# Fixed business rows (业务码 → 展示名). Seeded on first boot; agent/chain left empty.
# 8BF 页面尚未建好,先占位保留行。顺序即管理表的展示顺序。
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
# 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
# 3q-1…3q-5 / 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
{"business_code": "3Q", "label": "3QFX"},
{"business_code": "6BF", "label": "任务FX"},
@@ -29,6 +29,11 @@ DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
{"business_code": "8BF", "label": "8BF"},
{"business_code": "AFX", "label": "AGENTFX"},
# 独立配置位,与上方业务码无关;固定排在表尾。
{"business_code": "3q-1", "label": "3q-1"},
{"business_code": "3q-2", "label": "3q-2"},
{"business_code": "3q-3", "label": "3q-3"},
{"business_code": "3q-4", "label": "3q-4"},
{"business_code": "3q-5", "label": "3q-5"},
{"business_code": "6-1", "label": "6-1"},
{"business_code": "6-2", "label": "6-2"},
{"business_code": "6-3", "label": "6-3"},
@@ -9,13 +9,17 @@ from deerflow.persistence.compact_menu_overrides.keys import (
normalize_compact_menu_type,
)
from deerflow.persistence.compact_menu_overrides.memory import MemoryCompactMenuOverrideStore
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.compact_menu_overrides.model import (
CompactMenuOverrideRow,
UserCompactMenuOverrideConfigRow,
)
from deerflow.persistence.compact_menu_overrides.sql import CompactMenuOverrideRepository
__all__ = [
"ALLOWED_COMPACT_MENU_TYPES",
"CompactMenuOverrideRepository",
"CompactMenuOverrideRow",
"UserCompactMenuOverrideConfigRow",
"CompactMenuOverrideStore",
"DEFAULT_COMPACT_MENU_TYPE",
"MemoryCompactMenuOverrideStore",
@@ -1,4 +1,10 @@
"""Abstract compact-sidebar menu override persistence API."""
"""Abstract compact-sidebar menu override persistence API.
The regular rows are the administrator's default for a compact-menu variant.
Personal layouts live in a separate complete snapshot per user and variant;
the router overlays that snapshot while allowing newly-added default node ids
to flow through.
"""
from __future__ import annotations
@@ -22,3 +28,20 @@ class CompactMenuOverrideStore(ABC):
) -> list[dict[str, Any]]:
"""Atomically replace the compact-menu override set for one type."""
raise NotImplementedError
@abstractmethod
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
"""Return one user's snapshot for a type, or ``None`` if absent."""
raise NotImplementedError
@abstractmethod
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
menu_type: str | None = None,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
"""Replace one user's snapshot; an empty list resets to defaults."""
raise NotImplementedError
@@ -38,6 +38,7 @@ def _public(row: dict[str, Any]) -> dict[str, Any]:
class MemoryCompactMenuOverrideStore(CompactMenuOverrideStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._user_rows: dict[tuple[str, str], list[dict[str, Any]]] = {}
async def list_overrides(self, menu_type: str | None = None) -> list[dict[str, Any]]:
t = normalize_compact_menu_type(menu_type)
@@ -62,3 +63,36 @@ class MemoryCompactMenuOverrideStore(CompactMenuOverrideStore):
stored = _normalize(r, t, updated_by)
self._rows[str(stored["node_id"])] = stored
return await self.list_overrides(t)
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
t = normalize_compact_menu_type(menu_type)
rows = self._user_rows.get((str(user_id), t))
return [dict(r) for r in rows] if rows is not None else None
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
menu_type: str | None = None,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
t = normalize_compact_menu_type(menu_type)
key = (str(user_id), t)
if not rows:
self._user_rows.pop(key, None)
return None
by_id: dict[str, dict[str, Any]] = {}
for row in rows:
node_id = row.get("node_id")
if not node_id:
continue
stored = _normalize(row, t, updated_by or str(user_id))
# Expose the public id in a personal document just like the API.
public = _public(stored)
by_id[str(public["node_id"])] = public
if not by_id:
self._user_rows.pop(key, None)
return None
self._user_rows[key] = [dict(r) for r in by_id.values()]
return [dict(r) for r in self._user_rows[key]]
@@ -12,7 +12,7 @@ from sqlalchemy import Index, Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class CompactMenuOverrideRow(Base):
@@ -28,3 +28,17 @@ class CompactMenuOverrideRow(Base):
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
class UserCompactMenuOverrideConfigRow(Base):
"""A user's complete snapshot for one compact-menu variant."""
__tablename__ = "user_compact_menu_override_configs"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
menu_type: Mapped[str] = mapped_column(String(32), primary_key=True)
overrides: Mapped[list[dict]] = mapped_column(PortableJSON(), nullable=False, default=list)
updated_by: Mapped[str | None] = mapped_column(String(64))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
@@ -2,7 +2,7 @@
from __future__ import annotations
from datetime import datetime
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, or_, select
@@ -15,7 +15,10 @@ from deerflow.persistence.compact_menu_overrides.keys import (
public_node_id,
storage_node_id,
)
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.compact_menu_overrides.model import (
CompactMenuOverrideRow,
UserCompactMenuOverrideConfigRow,
)
def _to_dict(row: CompactMenuOverrideRow) -> dict[str, Any]:
@@ -42,6 +45,41 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C
)
def _normalize_user_row(data: dict[str, Any], menu_type: str) -> dict[str, Any]:
placement = str(data.get("placement") or "sidebar")
return {
"node_id": public_node_id(str(data["node_id"]), menu_type),
"menu_type": menu_type,
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
"sort_order": int(data.get("sort_order", 0)),
"label": (data.get("label") or None),
}
def _user_rows(row: UserCompactMenuOverrideConfigRow) -> list[dict[str, Any]]:
value = row.overrides
if isinstance(value, str):
import json
try:
value = json.loads(value)
except (TypeError, ValueError):
value = []
if not isinstance(value, list):
value = []
menu_type = normalize_compact_menu_type(row.menu_type)
updated_at = row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at
return [
{
**_normalize_user_row(item, menu_type),
"updated_by": row.updated_by,
"updated_at": updated_at,
}
for item in value
if isinstance(item, dict) and item.get("node_id")
]
class CompactMenuOverrideRepository(CompactMenuOverrideStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -76,6 +114,54 @@ class CompactMenuOverrideRepository(CompactMenuOverrideStore):
await session.refresh(row)
return [_to_dict(row) for row in new_rows]
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
t = normalize_compact_menu_type(menu_type)
async with self._sf() as session:
row = await session.get(UserCompactMenuOverrideConfigRow, (str(user_id), t))
return _user_rows(row) if row is not None else None
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
menu_type: str | None = None,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
t = normalize_compact_menu_type(menu_type)
key = str(user_id)
by_id: dict[str, dict[str, Any]] = {}
for data in rows:
node_id = data.get("node_id")
if node_id:
normalized = _normalize_user_row(data, t)
by_id[normalized["node_id"]] = normalized
async with self._sf() as session:
row = await session.get(UserCompactMenuOverrideConfigRow, (key, t))
if not by_id:
if row is not None:
await session.delete(row)
await session.commit()
return None
now = datetime.now(UTC)
payload = list(by_id.values())
if row is None:
row = UserCompactMenuOverrideConfigRow(
user_id=key,
menu_type=t,
overrides=payload,
updated_by=updated_by or key,
)
session.add(row)
else:
row.overrides = payload
row.updated_by = updated_by or key
row.updated_at = now
await session.commit()
# ``expire_on_commit=False`` keeps this writer-side row usable and
# avoids a replica-lag-prone post-commit refresh.
return _user_rows(row)
def _menu_type_filter(menu_type: str):
if menu_type == DEFAULT_COMPACT_MENU_TYPE:
@@ -1,38 +1,52 @@
"""ORM model for user feedback on runs."""
"""ORM model for user feedback on assistant answers."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class FeedbackRow(Base):
__tablename__ = "feedback"
# Prefix-unique instead of UniqueConstraint: a plain unique over three
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
# mysql_length is ignored on SQLite/Postgres.
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
class FeedbackRow(Base):
__tablename__ = "feedback"
# ``target_key`` is a SHA-256 digest of user/thread/message/agent. A
# digest keeps the idempotency index well below MySQL's legacy utf8mb4
# key-size limit; indexing all four source strings would exceed it.
__table_args__ = (
Index("uq_feedback_target_key", "target_key", unique=True),
Index("ix_feedback_agent_created", "agent_id", "created_at"),
)
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
message_id: Mapped[str | None] = mapped_column(String(64))
# message_id is an optional RunEventStore event identifier —
# allows feedback to target a specific message or the entire run
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
message_id: Mapped[str | None] = mapped_column(String(64))
# ``message_id`` is the LangChain/checkpoint message id serialized inside
# a RunEventStore ``llm.ai.response`` event. It stays nullable solely for
# legacy run-level feedback rows created before answer-level feedback.
agent_id: Mapped[str] = mapped_column(String(128), nullable=False, default="lead_agent")
target_key: Mapped[str] = mapped_column(String(64), nullable=False)
rating: Mapped[int] = mapped_column(nullable=False)
# +1 (thumbs-up) or -1 (thumbs-down)
comment: Mapped[str | None] = mapped_column(Text)
# Optional text feedback from the user
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
comment: Mapped[str | None] = mapped_column(Text)
# Optional user detail. The API exposes this as ``detail`` and retains
# ``comment`` as a compatibility alias for older clients.
reasons: Mapped[list[str]] = mapped_column(PortableJSON(), default=list)
# Stable machine-readable reason codes for negative feedback.
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,225 @@
"""Transparent user-feedback scoring for assistant answers.
This module deliberately uses no model or heuristic content grading. It
summarizes explicit thumbs feedback and selected problem reasons only. The
result is an answer-quality signal, not a complete evaluation of an agent's
planning, reasoning trajectory, tool use, latency, reliability, or safety.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from typing import Any
FEEDBACK_REASONS = (
"incorrect_or_incomplete",
"unmet_expectations",
"slow_or_problematic",
"style_or_tone",
"safety_or_legal",
"other",
)
# The five response-quality dimensions and weights are fixed product policy.
# A negative reason is evidence only for the explicitly mapped dimensions;
# operational, communication, safety, and uncategorized complaints remain in
# the problem breakdown instead of being mislabeled as task-quality evidence.
DIMENSION_CONFIG: dict[str, dict[str, Any]] = {
"correctness": {
"weight": 0.30,
"mapped_reasons": ["incorrect_or_incomplete"],
},
"completeness": {
"weight": 0.20,
"mapped_reasons": ["incorrect_or_incomplete"],
},
"evidence_grounding": {
"weight": 0.20,
"mapped_reasons": ["incorrect_or_incomplete"],
},
"relevance": {
"weight": 0.15,
"mapped_reasons": ["unmet_expectations"],
},
"actionability": {
"weight": 0.15,
"mapped_reasons": ["unmet_expectations"],
},
}
# Beta(8, 2): prior mean 80/100 with ten equivalent observations. This keeps
# one or two votes from producing an extreme public score while real evidence
# steadily dominates. With no real sample the API returns ``score=null`` so
# the prior is never presented as earned performance.
PRIOR_POSITIVE = 8
PRIOR_NEGATIVE = 2
PRIOR_EQUIVALENT_SAMPLES = PRIOR_POSITIVE + PRIOR_NEGATIVE
PRIOR_MEAN = PRIOR_POSITIVE / PRIOR_EQUIVALENT_SAMPLES
def _posterior_score(successes: int, failures: int) -> float | None:
evidence_count = successes + failures
if evidence_count == 0:
return None
value = 100.0 * (successes + PRIOR_POSITIVE) / (
evidence_count + PRIOR_EQUIVALENT_SAMPLES
)
return round(value, 1)
def _score_band(score: float | None) -> dict[str, Any]:
if score is None:
return {
"key": "insufficient_data",
"label": "数据不足",
"min": 0,
"max": 100,
}
if score >= 90:
return {
"key": "excellent",
"label": "优秀",
"min": 90,
"max": 100,
}
if score >= 80:
return {
"key": "good",
"label": "良好",
"min": 80,
"max": 89.9,
}
if score >= 70:
return {
"key": "needs_attention",
"label": "需关注",
"min": 70,
"max": 79.9,
}
return {
"key": "poor",
"label": "较差",
"min": 0,
"max": 69.9,
}
def _confidence_level(sample_count: int) -> str:
if sample_count == 0:
return "none"
if sample_count < 10:
return "low"
if sample_count < 30:
return "medium"
return "high"
def calculate_answer_quality_score(
rows: Iterable[Mapping[str, Any]],
*,
agent_id: str,
) -> dict[str, Any]:
"""Aggregate explicit answer feedback into a transparent 0-100 score."""
records = list(rows)
positive_count = sum(1 for row in records if int(row.get("rating") or 0) == 1)
negative_rows = [row for row in records if int(row.get("rating") or 0) == -1]
negative_count = len(negative_rows)
sample_count = positive_count + negative_count
reason_counts = {reason: 0 for reason in FEEDBACK_REASONS}
negative_reason_sets: list[set[str]] = []
for row in negative_rows:
raw_reasons = row.get("reasons")
if not isinstance(raw_reasons, list):
raw_reasons = []
reason_set = {
str(reason)
for reason in raw_reasons
if str(reason) in reason_counts
}
negative_reason_sets.append(reason_set)
for reason in reason_set:
reason_counts[reason] += 1
dimensions: dict[str, dict[str, Any]] = {}
weighted_total = 0.0
available_weight = 0.0
for name, config in DIMENSION_CONFIG.items():
mapped = set(config["mapped_reasons"])
failures = sum(1 for reasons in negative_reason_sets if reasons & mapped)
evidence_count = positive_count + failures
dimension_score = _posterior_score(positive_count, failures)
weight = float(config["weight"])
dimensions[name] = {
"weight": weight,
"score": dimension_score,
"evidence_count": evidence_count,
"positive_count": positive_count,
"negative_count": failures,
"mapped_reasons": list(config["mapped_reasons"]),
}
if dimension_score is not None:
weighted_total += weight * dimension_score
available_weight += weight
task_score = (
round(weighted_total / available_weight, 1)
if available_weight > 0
else None
)
score = _posterior_score(positive_count, negative_count)
task_success_rate = (
round(positive_count / sample_count, 4) if sample_count else None
)
reason_breakdown = {
reason: {
"count": count,
"rate": round(count / negative_count, 4) if negative_count else 0.0,
}
for reason, count in reason_counts.items()
}
return {
"agent_id": agent_id,
"score": score,
"score_band": _score_band(score),
"confidence": round(sample_count / (sample_count + 10), 4),
"confidence_level": _confidence_level(sample_count),
"sample_count": sample_count,
"positive_count": positive_count,
"negative_count": negative_count,
"task_success_rate": task_success_rate,
"task_score": task_score,
"dimensions": dimensions,
"reason_breakdown": reason_breakdown,
"prior": {
"positive": PRIOR_POSITIVE,
"negative": PRIOR_NEGATIVE,
"mean": PRIOR_MEAN,
"equivalent_samples": PRIOR_EQUIVALENT_SAMPLES,
},
"formula": {
"scope": "user_feedback_answer_quality",
"overall": (
"100 * (positive_count + 8) / (sample_count + 10); "
"null when sample_count is zero"
),
"confidence": "sample_count / (sample_count + 10)",
"dimension": (
"100 * (positive_count + 8) / "
"(positive_count + mapped_negative_count + 10); "
"null when dimension evidence_count is zero"
),
"task_score": (
"weighted mean of evidenced dimensions using weights "
"30/20/20/15/15, renormalized over available evidence"
),
"reason_rate_denominator": "negative_count",
"limitations": (
"This is not a complete agent-capability score and does not "
"grade hidden reasoning, planning, skill/tool use, latency, "
"reliability, or safety without separate evidence."
),
},
}
@@ -1,17 +1,23 @@
"""SQLAlchemy-backed feedback storage.
"""SQLAlchemy-backed assistant-answer feedback storage.
Each method acquires its own short-lived session.
Each method acquires its own short-lived session. Writes are idempotent on a
SHA-256 target key derived from user/thread/message/agent; the digest avoids
oversized compound indexes on legacy MySQL utf8mb4 deployments.
"""
from __future__ import annotations
import hashlib
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import case, func, select
from sqlalchemy import case, delete, func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.feedback.scoring import FEEDBACK_REASONS, calculate_answer_quality_score
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
@@ -20,12 +26,51 @@ class FeedbackRepository:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: FeedbackRow) -> dict:
d = row.to_dict()
val = d.get("created_at")
if isinstance(val, datetime):
d["created_at"] = val.isoformat()
return d
def _row_to_dict(row: FeedbackRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
data["reasons"] = list(data.get("reasons") or [])
data["detail"] = data.get("comment")
return data
@staticmethod
def make_target_key(
*,
user_id: str | None,
thread_id: str,
run_id: str,
message_id: str | None,
agent_id: str,
) -> str:
"""Build the portable idempotency identity for one answer rating.
Historical run-level callers have no message id. Their deterministic
``run:<id>`` surrogate preserves the old one-rating-per-run behavior.
"""
message_key = message_id or f"run:{run_id}"
raw = "\x1f".join(
(
user_id or "",
thread_id,
message_key,
agent_id or "lead_agent",
)
)
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
@staticmethod
def _normalize_reasons(reasons: list[str] | None) -> list[str]:
allowed = set(FEEDBACK_REASONS)
normalized: list[str] = []
for reason in reasons or []:
value = str(reason).strip()
if value in allowed and value not in normalized:
normalized.append(value)
return normalized
async def create(
self,
@@ -35,34 +80,29 @@ class FeedbackRepository:
rating: int,
user_id: str | None | _AutoSentinel = AUTO,
message_id: str | None = None,
agent_id: str = "lead_agent",
reasons: list[str] | None = None,
comment: str | None = None,
) -> dict:
"""Create a feedback record. rating must be +1 or -1."""
if rating not in (1, -1):
raise ValueError(f"rating must be +1 or -1, got {rating}")
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.create")
row = FeedbackRow(
feedback_id=str(uuid.uuid4()),
) -> dict[str, Any]:
"""Create or update the same answer target idempotently."""
return await self.upsert(
run_id=run_id,
thread_id=thread_id,
user_id=resolved_user_id,
message_id=message_id,
rating=rating,
user_id=user_id,
message_id=message_id,
agent_id=agent_id,
reasons=reasons,
comment=comment,
created_at=datetime.now(UTC),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def get(
self,
feedback_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict | None:
) -> dict[str, Any] | None:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.get")
async with self._sf() as session:
row = await session.get(FeedbackRow, feedback_id)
@@ -72,38 +112,101 @@ class FeedbackRepository:
return None
return self._row_to_dict(row)
async def get_by_target(
self,
*,
thread_id: str,
run_id: str,
message_id: str | None,
agent_id: str,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict[str, Any] | None:
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.get_by_target",
)
target_key = self.make_target_key(
user_id=resolved_user_id,
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=agent_id,
)
async with self._sf() as session:
row = await session.scalar(
select(FeedbackRow).where(FeedbackRow.target_key == target_key)
)
return self._row_to_dict(row) if row is not None else None
async def list_by_run(
self,
thread_id: str,
run_id: str,
*,
limit: int = 100,
message_id: str | None = None,
user_id: str | None | _AutoSentinel = AUTO,
) -> list[dict]:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_run")
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
) -> list[dict[str, Any]]:
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.list_by_run",
)
stmt = select(FeedbackRow).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
)
if message_id is not None:
stmt = stmt.where(FeedbackRow.message_id == message_id)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
return [self._row_to_dict(row) for row in result.scalars()]
async def list_by_thread(
self,
thread_id: str,
*,
limit: int = 100,
limit: int = 1000,
user_id: str | None | _AutoSentinel = AUTO,
) -> list[dict]:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread")
) -> list[dict[str, Any]]:
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.list_by_thread",
)
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
return [self._row_to_dict(row) for row in result.scalars()]
async def list_by_message_ids(
self,
*,
thread_id: str,
message_ids: list[str],
user_id: str | None | _AutoSentinel = AUTO,
) -> list[dict[str, Any]]:
"""Load current-user feedback for a bounded visible-message batch."""
if not message_ids:
return []
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.list_by_message_ids",
)
stmt = select(FeedbackRow).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.message_id.in_(message_ids),
)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(row) for row in result.scalars()]
async def delete(
self,
@@ -111,7 +214,10 @@ class FeedbackRepository:
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete")
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.delete",
)
async with self._sf() as session:
row = await session.get(FeedbackRow, feedback_id)
if row is None:
@@ -129,84 +235,152 @@ class FeedbackRepository:
thread_id: str,
rating: int,
user_id: str | None | _AutoSentinel = AUTO,
message_id: str | None = None,
agent_id: str = "lead_agent",
reasons: list[str] | None = None,
comment: str | None = None,
) -> dict:
"""Create or update feedback for (thread_id, run_id, user_id). rating must be +1 or -1."""
) -> dict[str, Any]:
"""Create or update feedback for one user/thread/message/agent target."""
if rating not in (1, -1):
raise ValueError(f"rating must be +1 or -1, got {rating}")
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.upsert")
async with self._sf() as session:
stmt = select(FeedbackRow).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
FeedbackRow.user_id == resolved_user_id,
)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
if row is not None:
row.rating = rating
row.comment = comment
row.created_at = datetime.now(UTC)
else:
row = FeedbackRow(
feedback_id=str(uuid.uuid4()),
run_id=run_id,
thread_id=thread_id,
user_id=resolved_user_id,
rating=rating,
comment=comment,
created_at=datetime.now(UTC),
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.upsert",
)
normalized_agent_id = str(agent_id or "lead_agent")[:128]
normalized_reasons = self._normalize_reasons(reasons)
target_key = self.make_target_key(
user_id=resolved_user_id,
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=normalized_agent_id,
)
# The unique digest handles cross-worker races. On a simultaneous
# insert, retry as an update in a fresh transaction.
for attempt in range(2):
now = datetime.now(UTC)
async with self._sf() as session:
row = await session.scalar(
select(FeedbackRow).where(FeedbackRow.target_key == target_key)
)
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
if row is None:
row = FeedbackRow(
feedback_id=str(uuid.uuid4()),
run_id=run_id,
thread_id=thread_id,
user_id=resolved_user_id,
message_id=message_id,
agent_id=normalized_agent_id,
target_key=target_key,
rating=rating,
reasons=normalized_reasons,
comment=comment,
created_at=now,
updated_at=now,
)
session.add(row)
else:
row.run_id = run_id
row.rating = rating
row.reasons = normalized_reasons
row.comment = comment
row.updated_at = now
try:
await session.commit()
except IntegrityError:
await session.rollback()
if attempt == 0:
continue
raise
# Runtime factories use expire_on_commit=False. Avoid a
# post-commit refresh, unsafe behind a read replica.
return self._row_to_dict(row)
raise RuntimeError("unreachable feedback upsert retry state")
async def delete_by_run(
self,
*,
thread_id: str,
run_id: str,
message_id: str | None = None,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
"""Delete the current user's feedback for a run. Returns True if a record was deleted."""
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete_by_run")
"""Delete this user's feedback for a run or one message in the run."""
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.delete_by_run",
)
conditions = [
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
FeedbackRow.user_id == resolved_user_id,
]
if message_id is not None:
conditions.append(FeedbackRow.message_id == message_id)
async with self._sf() as session:
stmt = select(FeedbackRow).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
FeedbackRow.user_id == resolved_user_id,
)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
if row is None:
return False
await session.delete(row)
result = await session.execute(delete(FeedbackRow).where(*conditions))
await session.commit()
return True
return bool(result.rowcount)
async def delete_by_target(
self,
*,
thread_id: str,
run_id: str,
message_id: str,
agent_id: str,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.delete_by_target",
)
target_key = self.make_target_key(
user_id=resolved_user_id,
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=agent_id,
)
async with self._sf() as session:
result = await session.execute(
delete(FeedbackRow).where(FeedbackRow.target_key == target_key)
)
await session.commit()
return bool(result.rowcount)
async def list_by_thread_grouped(
self,
thread_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict[str, dict]:
"""Return feedback grouped by run_id for a thread: {run_id: feedback_dict}."""
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread_grouped")
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
async with self._sf() as session:
result = await session.execute(stmt)
return {row.run_id: self._row_to_dict(row) for row in result.scalars()}
) -> dict[str, dict[str, Any]]:
"""Legacy run-keyed view, retaining the newest row for each run."""
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict:
records = await self.list_by_thread(thread_id, user_id=user_id)
return {record["run_id"]: record for record in records}
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict[str, Any]:
"""Aggregate feedback stats for a run using database-side counting."""
stmt = select(
func.count().label("total"),
func.coalesce(func.sum(case((FeedbackRow.rating == 1, 1), else_=0)), 0).label("positive"),
func.coalesce(func.sum(case((FeedbackRow.rating == -1, 1), else_=0)), 0).label("negative"),
).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
func.coalesce(
func.sum(case((FeedbackRow.rating == 1, 1), else_=0)),
0,
).label("positive"),
func.coalesce(
func.sum(case((FeedbackRow.rating == -1, 1), else_=0)),
0,
).label("negative"),
).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).one()
return {
@@ -215,3 +389,20 @@ class FeedbackRepository:
"positive": row.positive,
"negative": row.negative,
}
async def aggregate_by_agent(self, agent_id: str) -> dict[str, Any]:
"""Return the documented user-feedback answer-quality score."""
normalized_agent_id = str(agent_id or "lead_agent")[:128]
stmt = select(FeedbackRow.rating, FeedbackRow.reasons).where(
FeedbackRow.agent_id == normalized_agent_id
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
return calculate_answer_quality_score(
(
{"rating": row.rating, "reasons": list(row.reasons or [])}
for row in rows
),
agent_id=normalized_agent_id,
)
@@ -0,0 +1,22 @@
"""Durable, owner-scoped records for the public-document workbench."""
from __future__ import annotations
from deerflow.persistence.gongwen_documents.memory import MemoryGongwenDocumentStore
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore, SqlGongwenDocumentStore
__all__ = [
"GongwenDocumentRow",
"GongwenDocumentStore",
"GongwenDocumentVersionRow",
"MemoryGongwenDocumentStore",
"SqlGongwenDocumentStore",
"make_gongwen_document_store",
]
def make_gongwen_document_store(session_factory):
if session_factory is None:
return MemoryGongwenDocumentStore()
return SqlGongwenDocumentStore(session_factory)
@@ -0,0 +1,75 @@
"""Small in-memory fallback used when persistence is disabled in tests."""
from __future__ import annotations
from copy import deepcopy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore
class MemoryGongwenDocumentStore(GongwenDocumentStore):
def __init__(self) -> None:
self._documents: dict[str, dict[str, Any]] = {}
self._versions: dict[str, list[dict[str, Any]]] = {}
@staticmethod
def _owned(document: dict[str, Any] | None, user_id: str | None) -> bool:
return bool(document and document.get("user_id") == user_id)
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC).isoformat()
row = {
"id": str(data["id"]), "user_id": data.get("user_id"), "title": str(data.get("title") or ""),
"document_type": str(data.get("document_type") or "notice"), "source": str(data.get("source") or "upload"),
"status": str(data.get("status") or "checking"), "template_key": data.get("template_key"),
"research_session_id": data.get("research_session_id"), "original_version_id": None, "latest_version_id": None,
"issue_counts": deepcopy(data.get("issue_counts")), "error": data.get("error"), "created_at": now, "updated_at": now,
}
self._documents[row["id"]] = row
return deepcopy(row)
async def list_documents(self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
rows = [r for r in self._documents.values() if r.get("user_id") == user_id]
if source:
rows = [r for r in rows if r.get("source") == source]
if status:
rows = [r for r in rows if r.get("status") == status]
return deepcopy(sorted(rows, key=lambda item: item["updated_at"], reverse=True)[:limit])
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
row = self._documents.get(document_id)
return deepcopy(row) if self._owned(row, user_id) else None
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
if not self._owned(self._documents.get(document_id), user_id):
return []
return deepcopy(list(reversed(self._versions.get(document_id, []))))
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
if not self._owned(self._documents.get(document_id), user_id):
return None
return next((deepcopy(row) for row in self._versions.get(document_id, []) if row["id"] == version_id), None)
async def append_version(self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]] | None:
document = self._documents.get(document_id)
if not self._owned(document, user_id):
return None
rows = self._versions.setdefault(document_id, [])
row = {**deepcopy(version), "document_id": document_id, "version_no": len(rows) + 1, "created_at": datetime.now(UTC).isoformat()}
rows.append(row)
document.update(deepcopy(document_patch))
document["latest_version_id"] = row["id"]
if row["kind"] == "original" and not document.get("original_version_id"):
document["original_version_id"] = row["id"]
document["updated_at"] = datetime.now(UTC).isoformat()
return deepcopy(document), deepcopy(row)
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
document = self._documents.get(document_id)
if not self._owned(document, user_id):
return None
document.update(deepcopy(patch))
document["updated_at"] = datetime.now(UTC).isoformat()
return deepcopy(document)
@@ -0,0 +1,68 @@
"""ORM rows for the public-document library.
The binary files deliberately stay outside the database. A document row is
the durable library entry; every uploaded original, generated output and
format-repaired output is an immutable version row beneath it.
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class GongwenDocumentRow(Base):
__tablename__ = "gongwen_documents"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
document_type: Mapped[str] = mapped_column(String(48), nullable=False, default="notice", index=True)
# upload | generated. Keep source separate from status so the library can
# filter both axes independently.
source: Mapped[str] = mapped_column(String(24), nullable=False, default="upload", index=True)
# checking | review_required | verified | repaired | failed
status: Mapped[str] = mapped_column(String(32), nullable=False, default="checking", index=True)
template_key: Mapped[str | None] = mapped_column(String(96), nullable=True)
research_session_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
original_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
latest_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
issue_counts: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
class GongwenDocumentVersionRow(Base):
__tablename__ = "gongwen_document_versions"
__table_args__ = (
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
sa.Index("ix_gongwen_document_versions_document_created", "document_id", "created_at"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
document_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
version_no: Mapped[int] = mapped_column(Integer, nullable=False)
# original | generated | repaired | markdown
kind: Mapped[str] = mapped_column(String(32), nullable=False)
file_name: Mapped[str] = mapped_column(String(512), nullable=False)
storage_key: Mapped[str] = mapped_column(String(1024), nullable=False)
content_type: Mapped[str] = mapped_column(String(128), nullable=False, default="application/octet-stream")
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
sha256: Mapped[str] = mapped_column(String(64), nullable=False, default="")
# Exact review snapshot for this version. Later rule updates must not
# rewrite history or make past repair decisions impossible to explain.
review_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -0,0 +1,203 @@
"""SQL and in-memory stores for public-document library metadata."""
from __future__ import annotations
import json
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
def _json_load(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except json.JSONDecodeError:
return None
def _json_dump(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
def _as_dict(row: GongwenDocumentRow | GongwenDocumentVersionRow) -> dict[str, Any]:
result = row.to_dict()
for key, value in tuple(result.items()):
if isinstance(value, datetime):
result[key] = value.isoformat()
if "issue_counts" in result:
result["issue_counts"] = _json_load(result["issue_counts"])
if "review_snapshot" in result:
result["review_snapshot"] = _json_load(result["review_snapshot"])
return result
class GongwenDocumentStore(ABC):
@abstractmethod
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_documents(
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def append_version(
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
) -> tuple[dict[str, Any], dict[str, Any]] | None:
raise NotImplementedError
@abstractmethod
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
class SqlGongwenDocumentStore(GongwenDocumentStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _owned(stmt, user_id: str | None):
return stmt.where(GongwenDocumentRow.user_id == user_id)
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
row = GongwenDocumentRow(
id=str(data["id"]),
user_id=data.get("user_id"),
title=str(data.get("title") or ""),
document_type=str(data.get("document_type") or "notice"),
source=str(data.get("source") or "upload"),
status=str(data.get("status") or "checking"),
template_key=data.get("template_key"),
research_session_id=data.get("research_session_id"),
issue_counts=_json_dump(data.get("issue_counts")),
error=data.get("error"),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _as_dict(row)
async def list_documents(
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
) -> list[dict[str, Any]]:
stmt = self._owned(select(GongwenDocumentRow), user_id)
if source:
stmt = stmt.where(GongwenDocumentRow.source == source)
if status:
stmt = stmt.where(GongwenDocumentRow.status == status)
stmt = stmt.order_by(GongwenDocumentRow.updated_at.desc()).limit(max(1, min(limit, 200)))
async with self._sf() as session:
return [_as_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _as_dict(row) if row else None
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
async with self._sf() as session:
if (await session.execute(stmt)).scalar_one_or_none() is None:
return []
versions = await session.execute(
select(GongwenDocumentVersionRow)
.where(GongwenDocumentVersionRow.document_id == document_id)
.order_by(GongwenDocumentVersionRow.version_no.desc())
)
return [_as_dict(row) for row in versions.scalars()]
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
async with self._sf() as session:
if (await session.execute(stmt)).scalar_one_or_none() is None:
return None
row = (
await session.execute(
select(GongwenDocumentVersionRow).where(
GongwenDocumentVersionRow.document_id == document_id,
GongwenDocumentVersionRow.id == version_id,
)
)
).scalar_one_or_none()
return _as_dict(row) if row else None
async def append_version(
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
) -> tuple[dict[str, Any], dict[str, Any]] | None:
async with self._sf() as session:
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
document = (await session.execute(stmt)).scalar_one_or_none()
if document is None:
return None
version_no = int(
(await session.execute(
select(func.coalesce(func.max(GongwenDocumentVersionRow.version_no), 0)).where(
GongwenDocumentVersionRow.document_id == document_id
)
)).scalar_one()
) + 1
row = GongwenDocumentVersionRow(
id=str(version["id"]),
document_id=document_id,
version_no=version_no,
kind=str(version["kind"]),
file_name=str(version["file_name"]),
storage_key=str(version["storage_key"]),
content_type=str(version.get("content_type") or "application/octet-stream"),
size_bytes=int(version.get("size_bytes") or 0),
sha256=str(version.get("sha256") or ""),
review_snapshot=_json_dump(version.get("review_snapshot")),
)
for key, value in document_patch.items():
if key == "issue_counts":
value = _json_dump(value)
if hasattr(document, key):
setattr(document, key, value)
document.latest_version_id = row.id
if row.kind == "original" and not document.original_version_id:
document.original_version_id = row.id
session.add(row)
await session.commit()
await session.refresh(document)
await session.refresh(row)
return _as_dict(document), _as_dict(row)
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
return None
for key, value in patch.items():
if key == "issue_counts":
value = _json_dump(value)
if hasattr(row, key):
setattr(row, key, value)
await session.commit()
await session.refresh(row)
return _as_dict(row)
@@ -28,6 +28,9 @@ class LlmWikiStore(ABC):
*,
name: str | None = None,
description: str | None = None,
test_questions: list[str] | None = None,
test_questions_set: bool = False,
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None: ...
@@ -28,6 +28,8 @@ class MemoryLlmWikiStore(LlmWikiStore):
"owner_user_id": owner_user_id,
"name": name,
"description": description,
"test_questions": None,
"retrieval_settings": None,
"kb_type": kb_type,
"publication_status": "private",
"published_by": None,
@@ -74,6 +76,9 @@ class MemoryLlmWikiStore(LlmWikiStore):
*,
name: str | None = None,
description: str | None = None,
test_questions: list[str] | None = None,
test_questions_set: bool = False,
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None:
@@ -84,6 +89,10 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["name"] = name
if description is not None:
row["description"] = description
if test_questions_set:
row["test_questions"] = list(test_questions or [])
if retrieval_settings is not None:
row["retrieval_settings"] = dict(retrieval_settings)
if wiki_index_enabled is not None:
row["wiki_index_enabled"] = wiki_index_enabled
if external_search_enabled is not None:
@@ -8,7 +8,7 @@ from sqlalchemy import Boolean, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class LlmWikiKnowledgeBaseRow(Base):
@@ -19,6 +19,11 @@ class LlmWikiKnowledgeBaseRow(Base):
owner_user_id: Mapped[str] = mapped_column(String(64), index=True)
name: Mapped[str] = mapped_column(String(255), default="")
description: Mapped[str] = mapped_column(String(2048), default="")
# ``None`` means "use the product defaults derived from the knowledge-base
# name". An explicit empty list is meaningful: the owner disabled starter
# questions for this library.
test_questions: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True)
retrieval_settings: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
kb_type: Mapped[str] = mapped_column(String(32), default="document")
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
@@ -88,6 +88,9 @@ class LlmWikiRepository(LlmWikiStore):
*,
name: str | None = None,
description: str | None = None,
test_questions: list[str] | None = None,
test_questions_set: bool = False,
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
) -> dict[str, Any] | None:
@@ -99,6 +102,10 @@ class LlmWikiRepository(LlmWikiStore):
row.name = name
if description is not None:
row.description = description
if test_questions_set:
row.test_questions = list(test_questions or [])
if retrieval_settings is not None:
row.retrieval_settings = dict(retrieval_settings)
if wiki_index_enabled is not None:
row.wiki_index_enabled = wiki_index_enabled
if external_search_enabled is not None:
@@ -2,16 +2,17 @@
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.model import (
LlmWikiWikiEdgeRow,
LlmWikiWikiPageRow,
LlmWikiWikiSyncStateRow,
LlmWikiWikiVectorRow,
)
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
return (
MemoryLlmWikiIndexStore()
if session_factory is None
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
)
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
__all__ = [
@@ -19,6 +20,7 @@ __all__ = [
"MemoryLlmWikiIndexStore",
"SqlLlmWikiIndexStore",
"LlmWikiWikiPageRow",
"LlmWikiWikiEdgeRow",
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"make_llmwiki_index_store",
@@ -14,8 +14,85 @@ class LlmWikiIndexStore(ABC):
@abstractmethod
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
async def search_pages_by_keywords(
self,
mapping_ids: list[str],
tokens: list[str],
*,
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
"""Return local Wiki body pages containing any token.
The default keeps lightweight test doubles and third-party stores
backward compatible. Database and in-memory stores override it.
"""
return []
async def replace_graph_edges(
self,
mapping_id: str,
edges: list[dict[str, Any]],
*,
owner: str,
sync_id: str,
) -> dict[str, Any] | None:
"""Replace one mapping's directed Wiki graph snapshot.
The default keeps lightweight third-party/test stores compatible. An
implementation must resolve both endpoints inside ``mapping_id`` and
discard dangling or cross-mapping edges. ``None`` means the caller no
longer owns the supplied sync lease and no mutation was made.
"""
return {"edge_count": 0, "changed": False}
async def get_adjacent_pages(
self,
mapping_ids: list[str],
seed_slugs: dict[str, list[str]] | list[str],
*,
direction: str = "both",
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
"""Return one-hop neighbors within the explicitly allowed mappings.
Passing a ``{mapping_id: [slug, ...]}`` mapping is preferred because it
retains the exact seed provenance. A flat slug list is applied only to
the supplied ``mapping_ids`` and never broadens authorization.
"""
return []
@abstractmethod
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
async def touch_page(
self,
page_id: str,
sync_id: str,
*,
mapping_id: str | None = None,
owner: str | None = None,
) -> bool: ...
@abstractmethod
async def refresh_page_metadata(
self,
*,
page: dict[str, Any],
sync_id: str,
owner: str,
) -> dict[str, Any] | None:
"""Refresh a mirrored page without replacing its vectors.
The write is fenced by the mapping sync lease. ``None`` means the
caller no longer owns that lease (or the expected page disappeared),
while the returned ``changed`` flag reports whether durable page
metadata changed. The page is always marked as seen by ``sync_id``.
"""
...
@abstractmethod
async def replace_page_vectors(
@@ -25,13 +102,34 @@ class LlmWikiIndexStore(ABC):
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
) -> dict[str, Any]: ...
owner: str | None = None,
) -> dict[str, Any] | None: ...
@abstractmethod
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
async def record_page_failure(
self,
*,
page: dict[str, Any],
sync_id: str,
error: str,
owner: str | None = None,
) -> bool: ...
@abstractmethod
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
async def mark_missing_pages_deleted(
self,
mapping_id: str,
sync_id: str,
*,
owner: str,
) -> int | None:
"""Retire pages absent from a fenced full scan.
``None`` means the supplied owner/sync pair no longer owns the mapping
lease and no mutation was made.
"""
...
@abstractmethod
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
@@ -73,7 +171,17 @@ class LlmWikiIndexStore(ABC):
) -> bool: ...
@abstractmethod
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
async def finish_sync(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
success: bool,
remote_page_count: int,
error: str | None = None,
result: dict[str, Any] | None = None,
) -> bool: ...
@abstractmethod
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
@@ -17,11 +17,28 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self) -> None:
self.pages: dict[str, dict[str, Any]] = {}
self.vectors: dict[str, list[dict[str, Any]]] = {}
self.edges: dict[str, list[dict[str, Any]]] = {}
self.states: dict[str, dict[str, Any]] = {}
def _state(self, mapping_id: str) -> dict[str, Any]:
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
def _has_current_sync_lease(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
) -> bool:
state = self.states.get(mapping_id)
return bool(
state is not None
and state.get("lease_owner") == owner
and state.get("active_sync_id") == sync_id
and isinstance(state.get("lease_expires_at"), datetime)
and state["lease_expires_at"] >= _now()
)
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
@@ -30,12 +47,219 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
return dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
if page_id in self.pages:
self.pages[page_id]["last_seen_sync_id"] = sync_id
self.pages[page_id]["last_error"] = None
async def search_pages_by_keywords(
self,
mapping_ids: list[str],
tokens: list[str],
*,
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
wanted = set(mapping_ids)
needles = [str(item).strip().lower() for item in tokens if str(item).strip()]
rows = []
for page in self.pages.values():
if page.get("knowledge_base_mapping_id") not in wanted or page.get("is_remote_deleted"):
continue
if page.get("status") == "draft" and not include_drafts:
continue
if page.get("status") not in {"published", "draft"}:
continue
haystack = " ".join(str(page.get(key) or "") for key in ("title", "summary", "content_md")).lower()
if needles and any(needle in haystack for needle in needles):
rows.append(dict(page))
if len(rows) >= max(1, limit):
break
return rows
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
async def replace_graph_edges(
self,
mapping_id: str,
edges: list[dict[str, Any]],
*,
owner: str,
sync_id: str,
) -> dict[str, Any] | None:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return None
pages_by_slug = {str(page.get("slug") or ""): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") == mapping_id and not page.get("is_remote_deleted") and page.get("status") in {"published", "draft"}}
canonical = {
(
str(edge.get("source_slug") or "").strip(),
str(edge.get("target_slug") or "").strip(),
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
)
for edge in edges
if isinstance(edge, dict)
}
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
replacement = [
{
"knowledge_base_mapping_id": mapping_id,
"source_page_id": pages_by_slug[source]["id"],
"target_page_id": pages_by_slug[target]["id"],
"source_slug": source,
"target_slug": target,
"edge_type": edge_type,
}
for source, target, edge_type in sorted(canonical)
]
current = self.edges.get(mapping_id, [])
current_keys = {(row["source_slug"], row["target_slug"], row["edge_type"]) for row in current}
changed = current_keys != canonical
if changed:
self.edges[mapping_id] = replacement
self._state(mapping_id)["index_revision"] += 1
return {"edge_count": len(replacement), "changed": changed}
async def get_adjacent_pages(
self,
mapping_ids: list[str],
seed_slugs: dict[str, list[str]] | list[str],
*,
direction: str = "both",
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
allowed = set(str(item).strip() for item in mapping_ids if str(item).strip())
if direction not in {"incoming", "outgoing", "both"}:
raise ValueError("direction must be incoming, outgoing, or both")
if isinstance(seed_slugs, dict):
seeds = {mapping_id: {str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip()} for mapping_id in allowed}
else:
shared = {str(slug).strip() for slug in seed_slugs if str(slug).strip()}
seeds = {mapping_id: shared for mapping_id in allowed}
pages_by_id = {str(page["id"]): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") in allowed}
result: list[dict[str, Any]] = []
seen: set[tuple[str, str, str, str]] = set()
for mapping_id in sorted(allowed):
mapping_seeds = seeds.get(mapping_id) or set()
for edge in self.edges.get(mapping_id, []):
traversals: list[tuple[str, str, str, str]] = []
if direction in {"outgoing", "both"} and edge["source_slug"] in mapping_seeds:
traversals.append(
(
edge["source_slug"],
edge["target_slug"],
"outgoing",
edge["target_page_id"],
)
)
if direction in {"incoming", "both"} and edge["target_slug"] in mapping_seeds:
traversals.append(
(
edge["target_slug"],
edge["source_slug"],
"incoming",
edge["source_page_id"],
)
)
for seed_slug, neighbor_slug, traversal_direction, page_id in traversals:
page = pages_by_id.get(page_id)
if not page or page.get("is_remote_deleted"):
continue
if page.get("status") == "draft" and not include_drafts:
continue
if page.get("status") not in {"published", "draft"}:
continue
key = (mapping_id, seed_slug, neighbor_slug, traversal_direction)
if key in seen:
continue
seen.add(key)
result.append(
{
"knowledge_base_mapping_id": mapping_id,
"seed_slug": seed_slug,
"neighbor_slug": neighbor_slug,
"direction": traversal_direction,
"edge_type": edge["edge_type"],
"source_slug": edge["source_slug"],
"target_slug": edge["target_slug"],
"page": dict(page),
}
)
result.sort(
key=lambda row: (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
)
return result[: max(1, min(int(limit), 1000))]
async def touch_page(
self,
page_id: str,
sync_id: str,
*,
mapping_id: str | None = None,
owner: str | None = None,
) -> bool:
page = self.pages.get(page_id)
if page is None:
return False
actual_mapping_id = str(page.get("knowledge_base_mapping_id") or "")
if mapping_id is not None and mapping_id != actual_mapping_id:
return False
if owner is not None and not self._has_current_sync_lease(
actual_mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
page["last_seen_sync_id"] = sync_id
page["last_error"] = None
return True
async def refresh_page_metadata(
self,
*,
page: dict[str, Any],
sync_id: str,
owner: str,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
if not self._has_current_sync_lease(
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
existing = await self.get_page_by_slug_hash(mapping_id, str(page["slug_hash"]))
if existing is None:
return None
durable_changed = any(existing.get(key) != value for key, value in page.items())
row = self.pages[str(existing["id"])]
row.update(page)
row.update(
{
"last_seen_sync_id": sync_id,
"last_error": None,
"is_remote_deleted": False,
"updated_at": _now().isoformat(),
}
)
if durable_changed:
self._state(mapping_id)["index_revision"] += 1
return {"changed": durable_changed, "page": dict(row)}
async def replace_page_vectors(
self,
*,
page: dict[str, Any],
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
owner: str | None = None,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
if owner is not None and not self._has_current_sync_lease(
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
page_id = existing["id"] if existing else str(uuid4())
now = _now().isoformat()
@@ -59,7 +283,21 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
state["embedding_fingerprint"] = fingerprint
return dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
async def record_page_failure(
self,
*,
page: dict[str, Any],
sync_id: str,
error: str,
owner: str | None = None,
) -> bool:
mapping_id = str(page["knowledge_base_mapping_id"])
if owner is not None and not self._has_current_sync_lease(
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
if existing:
self.pages[existing["id"]].update(
@@ -70,11 +308,20 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
}
)
return
return True
page_id = str(uuid4())
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
return True
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
async def mark_missing_pages_deleted(
self,
mapping_id: str,
sync_id: str,
*,
owner: str,
) -> int | None:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return None
changed = 0
for page in self.pages.values():
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
@@ -82,6 +329,8 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
self.vectors.pop(page["id"], None)
changed += 1
if changed:
active_page_ids = {page["id"] for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")}
self.edges[mapping_id] = [edge for edge in self.edges.get(mapping_id, []) if edge["source_page_id"] in active_page_ids and edge["target_page_id"] in active_page_ids]
self._state(mapping_id)["index_revision"] += 1
return changed
@@ -131,17 +380,27 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
sync_id: str,
remote_page_count: int,
) -> bool:
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return False
state = self._state(mapping_id)
state["remote_page_count"] = max(0, remote_page_count)
state["updated_at"] = _now()
return True
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async def finish_sync(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
success: bool,
remote_page_count: int,
error: str | None = None,
result: dict[str, Any] | None = None,
) -> bool:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return False
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
return
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
was_rebuilding = state.get("state") == "rebuilding"
state.update(
@@ -158,8 +417,10 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
"last_success_at": _now() if success else state.get("last_success_at"),
"last_sync_result": dict(result) if result is not None else state.get("last_sync_result"),
}
)
return True
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
wanted = set(mapping_ids or [])
@@ -168,11 +429,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
result = []
for page in sorted(
(
page
for page in self.pages.values()
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
),
(page for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")),
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
):
result.append(
@@ -72,6 +72,53 @@ class LlmWikiWikiVectorRow(Base):
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
class LlmWikiWikiEdgeRow(Base):
"""One directed, mapping-scoped link between two mirrored Wiki pages."""
__tablename__ = "llmwiki_wiki_edges"
__table_args__ = (
UniqueConstraint(
"knowledge_base_mapping_id",
"source_page_id",
"target_page_id",
"edge_type",
name="uq_llmwiki_wiki_edge_mapping_pages_type",
),
Index(
"ix_llmwiki_wiki_edge_mapping_source",
"knowledge_base_mapping_id",
"source_page_id",
),
Index(
"ix_llmwiki_wiki_edge_mapping_target",
"knowledge_base_mapping_id",
"target_page_id",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(
String(36),
ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
source_page_id: Mapped[str] = mapped_column(
String(36),
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
target_page_id: Mapped[str] = mapped_column(
String(36),
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
edge_type: Mapped[str] = mapped_column(String(32), nullable=False, default="wiki_link")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
class LlmWikiWikiSyncStateRow(Base):
__tablename__ = "llmwiki_wiki_sync_states"
@@ -91,4 +138,5 @@ class LlmWikiWikiSyncStateRow(Base):
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_sync_result: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
@@ -7,12 +7,18 @@ from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import case, delete, func, insert, or_, select, update
from sqlalchemy import and_, case, delete, func, insert, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlalchemy.orm import aliased
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.model import (
LlmWikiWikiEdgeRow,
LlmWikiWikiPageRow,
LlmWikiWikiSyncStateRow,
LlmWikiWikiVectorRow,
)
def _now() -> datetime:
@@ -40,6 +46,32 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
await session.flush()
return row
async def _fence_current_sync_lease(
self,
session: AsyncSession,
mapping_id: str,
*,
owner: str,
sync_id: str,
) -> bool:
"""Fence destructive snapshot writes behind the current live lease."""
# Use a conditional UPDATE rather than SELECT ... FOR UPDATE. SQLite
# ignores FOR UPDATE, while this statement obtains its write lock on
# every supported backend before the page/edge mutation starts.
stmt = (
update(LlmWikiWikiSyncStateRow)
.where(
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiSyncStateRow.lease_owner == owner,
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
)
.values(updated_at=_now())
)
return bool((await session.execute(stmt)).rowcount)
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
@@ -53,18 +85,322 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
async def search_pages_by_keywords(
self,
mapping_ids: list[str],
tokens: list[str],
*,
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
normalized_ids = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
normalized_tokens = list(dict.fromkeys(str(item).strip().lower() for item in tokens if str(item).strip()))
if not normalized_ids or not normalized_tokens:
return []
conditions = []
for token in normalized_tokens[:32]:
needle = f"%{token}%"
conditions.extend(
(
func.lower(LlmWikiWikiPageRow.title).like(needle),
func.lower(LlmWikiWikiPageRow.summary).like(needle),
func.lower(LlmWikiWikiPageRow.content_md).like(needle),
)
)
statuses = ["published", "draft"] if include_drafts else ["published"]
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(statuses),
func.length(func.trim(LlmWikiWikiPageRow.content_md)) > 0,
or_(*conditions),
)
.order_by(LlmWikiWikiPageRow.updated_at.desc())
.limit(max(1, min(int(limit), 2000)))
)
async with self._sf() as session:
await session.execute(
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def replace_graph_edges(
self,
mapping_id: str,
edges: list[dict[str, Any]],
*,
owner: str,
sync_id: str,
) -> dict[str, Any] | None:
"""Atomically replace an authorized mapping's resolved edge snapshot."""
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
page_stmt = select(LlmWikiWikiPageRow).where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
pages = list((await session.execute(page_stmt)).scalars())
pages_by_slug = {page.slug: page for page in pages}
canonical = {
(
str(edge.get("source_slug") or "").strip(),
str(edge.get("target_slug") or "").strip(),
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
)
for edge in edges
if isinstance(edge, dict)
}
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
existing_stmt = select(
LlmWikiWikiEdgeRow.source_page_id,
LlmWikiWikiEdgeRow.target_page_id,
LlmWikiWikiEdgeRow.edge_type,
).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id)
existing = {(str(source_id), str(target_id), str(edge_type)) for source_id, target_id, edge_type in (await session.execute(existing_stmt)).all()}
replacement_ids = {
(
pages_by_slug[source].id,
pages_by_slug[target].id,
edge_type,
)
for source, target, edge_type in canonical
}
changed = existing != replacement_ids
if changed:
await session.execute(delete(LlmWikiWikiEdgeRow).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id))
now = _now()
edge_rows = [
{
"id": str(uuid4()),
"knowledge_base_mapping_id": mapping_id,
"source_page_id": pages_by_slug[source].id,
"target_page_id": pages_by_slug[target].id,
"edge_type": edge_type,
"created_at": now,
}
for source, target, edge_type in sorted(canonical)
]
for start in range(0, len(edge_rows), self.write_batch_size):
await session.execute(
insert(LlmWikiWikiEdgeRow),
edge_rows[start : start + self.write_batch_size],
)
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return {"edge_count": len(canonical), "changed": changed}
async def get_adjacent_pages(
self,
mapping_ids: list[str],
seed_slugs: dict[str, list[str]] | list[str],
*,
direction: str = "both",
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
allowed = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
if not allowed:
return []
if direction not in {"incoming", "outgoing", "both"}:
raise ValueError("direction must be incoming, outgoing, or both")
if isinstance(seed_slugs, dict):
seeds = {mapping_id: list(dict.fromkeys(str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip())) for mapping_id in allowed}
else:
shared = list(dict.fromkeys(str(slug).strip() for slug in seed_slugs if str(slug).strip()))
seeds = {mapping_id: shared for mapping_id in allowed}
seeds = {mapping_id: slugs for mapping_id, slugs in seeds.items() if slugs}
if not seeds:
return []
source_page = aliased(LlmWikiWikiPageRow)
target_page = aliased(LlmWikiWikiPageRow)
statuses = ["published", "draft"] if include_drafts else ["published"]
cap = max(1, min(int(limit), 1000))
async def _query(traversal_direction: str) -> list[dict[str, Any]]:
seed_page = source_page if traversal_direction == "outgoing" else target_page
neighbor_page = target_page if traversal_direction == "outgoing" else source_page
seed_conditions = or_(
*(
and_(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id,
seed_page.slug.in_(slugs),
)
for mapping_id, slugs in seeds.items()
)
)
stmt = (
select(LlmWikiWikiEdgeRow, source_page, target_page)
.join(source_page, source_page.id == LlmWikiWikiEdgeRow.source_page_id)
.join(target_page, target_page.id == LlmWikiWikiEdgeRow.target_page_id)
.where(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id.in_(allowed),
source_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
target_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
source_page.is_remote_deleted.is_(False),
target_page.is_remote_deleted.is_(False),
source_page.status.in_(statuses),
target_page.status.in_(statuses),
seed_conditions,
)
.order_by(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
seed_page.slug,
neighbor_page.slug,
)
.limit(cap)
)
async with self._sf() as session:
triples = list((await session.execute(stmt)).all())
rows: list[dict[str, Any]] = []
for edge, source, target in triples:
seed = source if traversal_direction == "outgoing" else target
neighbor = target if traversal_direction == "outgoing" else source
rows.append(
{
"knowledge_base_mapping_id": edge.knowledge_base_mapping_id,
"seed_slug": seed.slug,
"neighbor_slug": neighbor.slug,
"direction": traversal_direction,
"edge_type": edge.edge_type,
"source_slug": source.slug,
"target_slug": target.slug,
"page": _dict(neighbor),
}
)
return rows
result: list[dict[str, Any]] = []
if direction in {"outgoing", "both"}:
result.extend(await _query("outgoing"))
if direction in {"incoming", "both"}:
result.extend(await _query("incoming"))
result.sort(
key=lambda row: (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
)
deduplicated: list[dict[str, Any]] = []
seen: set[tuple[str, str, str, str]] = set()
for row in result:
key = (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
if key in seen:
continue
seen.add(key)
deduplicated.append(row)
if len(deduplicated) >= cap:
break
return deduplicated
async def touch_page(
self,
page_id: str,
sync_id: str,
*,
mapping_id: str | None = None,
owner: str | None = None,
) -> bool:
async with self._sf() as session:
if owner is not None:
if not mapping_id or not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
conditions = [LlmWikiWikiPageRow.id == page_id]
if mapping_id is not None:
conditions.append(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id)
result = await session.execute(
update(LlmWikiWikiPageRow)
.where(LlmWikiWikiPageRow.id == page_id)
.where(*conditions)
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
)
await session.commit()
return bool(result.rowcount)
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
async def refresh_page_metadata(
self,
*,
page: dict[str, Any],
sync_id: str,
owner: str,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.slug_hash == page["slug_hash"],
)
.with_for_update()
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
return None
if row.slug != page["slug"]:
raise ValueError("Wiki slug hash collision detected")
durable_changed = bool(row.is_remote_deleted or row.last_error) or any(
getattr(row, key) != value for key, value in page.items()
)
for key, value in page.items():
setattr(row, key, value)
now = _now()
row.last_seen_sync_id = sync_id
row.last_error = None
row.is_remote_deleted = False
row.updated_at = now
if durable_changed:
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return {"changed": durable_changed, "page": _dict(row)}
async def replace_page_vectors(
self,
*,
page: dict[str, Any],
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
owner: str | None = None,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
if owner is not None and not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
row = (await session.execute(stmt)).scalar_one_or_none()
if row is not None and row.slug != page["slug"]:
@@ -115,10 +451,24 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
await session.commit()
return _dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
async def record_page_failure(
self,
*,
page: dict[str, Any],
sync_id: str,
error: str,
owner: str | None = None,
) -> bool:
mapping_id = str(page["knowledge_base_mapping_id"])
safe_error = error[:4000]
async with self._sf() as session:
if owner is not None and not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
row = (await session.execute(stmt)).scalar_one_or_none()
now = _now()
@@ -136,9 +486,23 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row.updated_at = now
await self._ensure_state(session, mapping_id)
await session.commit()
return True
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
async def mark_missing_pages_deleted(
self,
mapping_id: str,
sync_id: str,
*,
owner: str,
) -> int | None:
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = (
select(LlmWikiWikiPageRow)
.where(
@@ -287,17 +651,36 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiSyncStateRow.lease_owner == owner,
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
)
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
)
await session.commit()
return bool(result.rowcount)
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async def finish_sync(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
success: bool,
remote_page_count: int,
error: str | None = None,
result: dict[str, Any] | None = None,
) -> bool:
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
return
return False
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
@@ -329,12 +712,15 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
state.ready_page_count = ready_count
state.failed_page_count = failed_count
state.vector_count = vector_count
if result is not None:
state.last_sync_result = dict(result)
if success and not error:
state.last_completed_at = now
if success:
state.last_success_at = now
state.updated_at = now
await session.commit()
return True
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
stmt = select(LlmWikiWikiSyncStateRow)
@@ -4,13 +4,14 @@ from __future__ import annotations
from deerflow.persistence.menu_overrides.base import MenuOverrideStore
from deerflow.persistence.menu_overrides.memory import MemoryMenuOverrideStore
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
from deerflow.persistence.menu_overrides.sql import MenuOverrideRepository
__all__ = [
"MemoryMenuOverrideStore",
"MenuOverrideRepository",
"MenuOverrideRow",
"UserMenuOverrideConfigRow",
"MenuOverrideStore",
"make_menu_override_store",
]
@@ -1,11 +1,11 @@
"""Abstract sidebar-menu override persistence API.
Global / shared store: no ``user_id`` scoping. Reads are open to any caller; the
router gates writes behind admin. ``updated_by`` is audit-only.
The whole override set is treated as one document: the editor computes the full
desired layout and replaces every row atomically via :meth:`replace_all`, so the
stored set never drifts out of sync with what the admin sees.
The original ``menu_overrides`` document remains the administrator's default
layout. User layouts are stored as a separate, complete snapshot so that a
user can customise the menu without changing the default for anybody else.
The router overlays a user's snapshot on top of the default and lets default
rows for node ids not present in that snapshot flow through (new administrator
menus therefore appear for existing users).
"""
from __future__ import annotations
@@ -28,3 +28,24 @@ class MenuOverrideStore(ABC):
Returns the persisted rows.
"""
raise NotImplementedError
@abstractmethod
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
"""Return one user's snapshot, or ``None`` when no snapshot exists."""
raise NotImplementedError
@abstractmethod
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
"""Replace one user's snapshot.
An empty list removes the snapshot and returns ``None``. This is the
reset-to-default operation and intentionally differs from the admin
``replace_all`` document, where an empty list means an empty default.
"""
raise NotImplementedError
@@ -23,6 +23,7 @@ def _normalize(data: dict[str, Any], updated_by: str | None) -> dict[str, Any]:
class MemoryMenuOverrideStore(MenuOverrideStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._user_rows: dict[str, list[dict[str, Any]]] = {}
async def list_overrides(self) -> list[dict[str, Any]]:
return [dict(r) for r in sorted(self._rows.values(), key=lambda r: r.get("sort_order", 0))]
@@ -35,3 +36,29 @@ class MemoryMenuOverrideStore(MenuOverrideStore):
continue
self._rows[str(node_id)] = _normalize(r, updated_by)
return [dict(r) for r in self._rows.values()]
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
rows = self._user_rows.get(str(user_id))
return [dict(r) for r in rows] if rows is not None else None
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
key = str(user_id)
if not rows:
self._user_rows.pop(key, None)
return None
by_id: dict[str, dict[str, Any]] = {}
for row in rows:
node_id = row.get("node_id")
if node_id:
by_id[str(node_id)] = _normalize(row, updated_by or key)
if not by_id:
self._user_rows.pop(key, None)
return None
self._user_rows[key] = [dict(row) for row in by_id.values()]
return [dict(row) for row in self._user_rows[key]]
@@ -1,9 +1,10 @@
"""ORM model for sidebar-menu overrides (菜单管理).
Global / shared — there is no ``user_id`` scoping; every account sees the same
menu layout. The static menu tree (icons / routes / adminOnly) lives in the
frontend code; this table only stores per-node **overrides** keyed by the static
node id (or an injected iframe light-app node id ``light-app-<appId>``):
``menu_overrides`` stores the administrator's shared default. Personal layouts
live separately in ``user_menu_override_configs`` as complete snapshots. The
static menu tree (icons / routes / adminOnly) lives in the frontend code; the
default table stores per-node **overrides** keyed by the static node id (or an
injected iframe light-app node id ``light-app-<appId>``):
- ``parent_id`` — the effective parent node id (NULL = top level). Re-parenting
a node moves its whole subtree.
@@ -23,7 +24,7 @@ from sqlalchemy import Boolean, Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class MenuOverrideRow(Base):
@@ -40,3 +41,22 @@ class MenuOverrideRow(Base):
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
class UserMenuOverrideConfigRow(Base):
"""A complete personal sidebar-menu snapshot.
Keeping the snapshot in one row is deliberate: the presence of a row is
the ``has_user_override`` bit, while the node ids in ``overrides`` are the
user's known-node set. This lets us distinguish an administrator-added
node from an old node the user intentionally left at its default value.
"""
__tablename__ = "user_menu_override_configs"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
overrides: Mapped[list[dict]] = mapped_column(PortableJSON(), nullable=False, default=list)
updated_by: Mapped[str | None] = mapped_column(String(64))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
@@ -2,14 +2,14 @@
from __future__ import annotations
from datetime import datetime
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.menu_overrides.base import MenuOverrideStore
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
def _to_dict(row: MenuOverrideRow) -> dict[str, Any]:
@@ -31,6 +31,42 @@ def _row_from(data: dict[str, Any], updated_by: str | None) -> MenuOverrideRow:
)
def _normalize_user_row(data: dict[str, Any]) -> dict[str, Any]:
"""Keep only the public override fields in the personal JSON document."""
return {
"node_id": str(data["node_id"]),
"parent_id": (str(data["parent_id"]) if data.get("parent_id") not in (None, "") else None),
"sort_order": int(data.get("sort_order", 0)),
"label": (data.get("label") or None),
"disabled": bool(data.get("disabled", False)),
}
def _user_rows(row: UserMenuOverrideConfigRow) -> list[dict[str, Any]]:
value = row.overrides
if isinstance(value, str):
# PortableJSON handles MySQL automatically, but this keeps old rows
# readable if a driver returned the raw TEXT value.
import json
try:
value = json.loads(value)
except (TypeError, ValueError):
value = []
if not isinstance(value, list):
value = []
updated_at = row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at
return [
{
**_normalize_user_row(item),
"updated_by": row.updated_by,
"updated_at": updated_at,
}
for item in value
if isinstance(item, dict) and item.get("node_id")
]
class MenuOverrideRepository(MenuOverrideStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -56,3 +92,49 @@ class MenuOverrideRepository(MenuOverrideStore):
for row in new_rows:
await session.refresh(row)
return [_to_dict(row) for row in new_rows]
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
async with self._sf() as session:
row = await session.get(UserMenuOverrideConfigRow, str(user_id))
return _user_rows(row) if row is not None else None
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
key = str(user_id)
by_id: dict[str, dict[str, Any]] = {}
for data in rows:
node_id = data.get("node_id")
if node_id:
normalized = _normalize_user_row(data)
by_id[normalized["node_id"]] = normalized
async with self._sf() as session:
row = await session.get(UserMenuOverrideConfigRow, key)
if not by_id:
if row is not None:
await session.delete(row)
await session.commit()
return None
now = datetime.now(UTC)
payload = list(by_id.values())
if row is None:
row = UserMenuOverrideConfigRow(
user_id=key,
overrides=payload,
updated_by=updated_by or key,
)
session.add(row)
else:
row.overrides = payload
row.updated_by = updated_by or key
row.updated_at = now
await session.commit()
# The shared session factory uses ``expire_on_commit=False``. Build
# the response from the writer-side instance instead of issuing a
# post-commit refresh that a read/write-splitting endpoint could
# route to a lagging replica.
return _user_rows(row)
@@ -0,0 +1,21 @@
"""Merge assistant-wiki metadata and report-collaboration heads.
Revision ID: 20260909_01
Revises: 20260903_01, 20260907_01
Create Date: 2026-09-09
"""
from __future__ import annotations
revision = "20260909_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -0,0 +1,74 @@
"""Create durable public-document library tables.
Revision ID: 20260914_01
Revises: 20260903_01, 20260907_01
Create Date: 2026-09-14
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260914_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("gongwen_documents"):
op.create_table(
"gongwen_documents",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("user_id", sa.String(length=64), nullable=True),
sa.Column("title", sa.String(length=256), nullable=False, server_default=""),
sa.Column("document_type", sa.String(length=48), nullable=False, server_default="notice"),
sa.Column("source", sa.String(length=24), nullable=False, server_default="upload"),
sa.Column("status", sa.String(length=32), nullable=False, server_default="checking"),
sa.Column("template_key", sa.String(length=96), nullable=True),
sa.Column("research_session_id", sa.String(length=64), nullable=True),
sa.Column("original_version_id", sa.String(length=64), nullable=True),
sa.Column("latest_version_id", sa.String(length=64), nullable=True),
sa.Column("issue_counts", sa.Text(), nullable=True),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_gongwen_documents_user_id", "gongwen_documents", ["user_id"])
op.create_index("ix_gongwen_documents_document_type", "gongwen_documents", ["document_type"])
op.create_index("ix_gongwen_documents_source", "gongwen_documents", ["source"])
op.create_index("ix_gongwen_documents_status", "gongwen_documents", ["status"])
op.create_index("ix_gongwen_documents_research_session_id", "gongwen_documents", ["research_session_id"])
if not inspector.has_table("gongwen_document_versions"):
op.create_table(
"gongwen_document_versions",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("document_id", sa.String(length=64), nullable=False),
sa.Column("version_no", sa.Integer(), nullable=False),
sa.Column("kind", sa.String(length=32), nullable=False),
sa.Column("file_name", sa.String(length=512), nullable=False),
sa.Column("storage_key", sa.String(length=1024), nullable=False),
sa.Column("content_type", sa.String(length=128), nullable=False, server_default="application/octet-stream"),
sa.Column("size_bytes", sa.Integer(), nullable=False, server_default="0"),
sa.Column("sha256", sa.String(length=64), nullable=False, server_default=""),
sa.Column("review_snapshot", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
)
op.create_index("ix_gongwen_document_versions_document_id", "gongwen_document_versions", ["document_id"])
op.create_index(
"ix_gongwen_document_versions_document_created",
"gongwen_document_versions",
["document_id", "created_at"],
)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if inspector.has_table("gongwen_document_versions"):
op.drop_table("gongwen_document_versions")
if inspector.has_table("gongwen_documents"):
op.drop_table("gongwen_documents")
@@ -0,0 +1,88 @@
"""Persist mapping-scoped links for local LLMWiki graph expansion.
Revision ID: 20260916_01
Revises: 20260903_01, 20260907_01
Create Date: 2026-09-16
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260916_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if inspector.has_table("llmwiki_wiki_edges"):
return
op.create_table(
"llmwiki_wiki_edges",
sa.Column("id", sa.String(length=36), primary_key=True),
sa.Column(
"knowledge_base_mapping_id",
sa.String(length=36),
sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"source_page_id",
sa.String(length=36),
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"target_page_id",
sa.String(length=36),
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"edge_type",
sa.String(length=32),
nullable=False,
server_default="wiki_link",
),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint(
"knowledge_base_mapping_id",
"source_page_id",
"target_page_id",
"edge_type",
name="uq_llmwiki_wiki_edge_mapping_pages_type",
),
)
op.create_index(
"ix_llmwiki_wiki_edges_knowledge_base_mapping_id",
"llmwiki_wiki_edges",
["knowledge_base_mapping_id"],
)
op.create_index(
"ix_llmwiki_wiki_edges_source_page_id",
"llmwiki_wiki_edges",
["source_page_id"],
)
op.create_index(
"ix_llmwiki_wiki_edges_target_page_id",
"llmwiki_wiki_edges",
["target_page_id"],
)
op.create_index(
"ix_llmwiki_wiki_edge_mapping_source",
"llmwiki_wiki_edges",
["knowledge_base_mapping_id", "source_page_id"],
)
op.create_index(
"ix_llmwiki_wiki_edge_mapping_target",
"llmwiki_wiki_edges",
["knowledge_base_mapping_id", "target_page_id"],
)
def downgrade() -> None:
if sa.inspect(op.get_bind()).has_table("llmwiki_wiki_edges"):
op.drop_table("llmwiki_wiki_edges")
@@ -0,0 +1,27 @@
"""Merge the public-document and assistant-knowledge migration heads.
Revision ID: 20260916_01
Revises: 20260909_01, 20260914_01
Create Date: 2026-09-16
The feedback-scoring migration was added as ``20260916_02`` and references
this merge revision. Keeping the merge as an explicit no-op makes the
Alembic graph linear again without changing either existing schema branch.
"""
from __future__ import annotations
revision = "20260916_01"
down_revision = ("20260909_01", "20260914_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
"""Record the merge; both parent revisions already own their DDL."""
def downgrade() -> None:
"""Splitting the migration history requires no schema change."""
@@ -0,0 +1,240 @@
"""Add answer-level feedback reasons and agent score dimensions.
Revision ID: 20260916_02
Revises: 20260916_01
Create Date: 2026-09-16
"""
from __future__ import annotations
import hashlib
import sqlalchemy as sa
from alembic import op
revision = "20260916_02"
down_revision = "20260916_01"
branch_labels = None
depends_on = None
_TABLE = "feedback"
_OLD_INDEX = "uq_feedback_thread_run_user"
_TARGET_INDEX = "uq_feedback_target_key"
_AGENT_INDEX = "ix_feedback_agent_created"
def _columns() -> set[str]:
return {
str(column["name"])
for column in sa.inspect(op.get_bind()).get_columns(_TABLE)
}
def _indexes() -> set[str]:
return {
str(index["name"])
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE)
if index.get("name")
}
def _json_type() -> sa.types.TypeEngine:
return sa.Text() if op.get_bind().dialect.name == "mysql" else sa.JSON()
def _empty_reasons_value() -> str | list[str]:
# PortableJSON is TEXT on MySQL, where binding a Python list directly to
# a raw migration Text column is invalid. SQLite/PostgreSQL JSON columns
# should receive the native list so their type processors serialize it.
return "[]" if op.get_bind().dialect.name == "mysql" else []
def _target_key(
*,
user_id: str | None,
thread_id: str,
run_id: str,
message_id: str | None,
agent_id: str,
) -> str:
message_key = message_id or f"run:{run_id}"
raw = "\x1f".join(
(user_id or "", thread_id, message_key, agent_id or "lead_agent")
)
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def _backfill() -> None:
bind = op.get_bind()
feedback = sa.table(
_TABLE,
sa.column("feedback_id", sa.String(64)),
sa.column("run_id", sa.String(64)),
sa.column("thread_id", sa.String(64)),
sa.column("user_id", sa.String(64)),
sa.column("message_id", sa.String(64)),
sa.column("agent_id", sa.String(128)),
sa.column("target_key", sa.String(64)),
sa.column("reasons", _json_type()),
sa.column("created_at", sa.DateTime(timezone=True)),
sa.column("updated_at", sa.DateTime(timezone=True)),
)
rows = list(
bind.execute(
sa.select(
feedback.c.feedback_id,
feedback.c.run_id,
feedback.c.thread_id,
feedback.c.user_id,
feedback.c.message_id,
feedback.c.agent_id,
feedback.c.created_at,
)
).mappings()
)
run_agents: dict[str, str] = {}
inspector = sa.inspect(bind)
if rows and inspector.has_table("runs"):
run_columns = {str(column["name"]) for column in inspector.get_columns("runs")}
if "assistant_id" in run_columns:
runs = sa.table(
"runs",
sa.column("run_id", sa.String(64)),
sa.column("assistant_id", sa.String(128)),
)
run_ids = sorted({str(row["run_id"]) for row in rows})
run_agents = {
str(row.run_id): str(row.assistant_id)
for row in bind.execute(
sa.select(runs.c.run_id, runs.c.assistant_id).where(
runs.c.run_id.in_(run_ids)
)
)
if row.assistant_id
}
seen_target_keys: set[str] = set()
for row in rows:
agent_id = str(
run_agents.get(str(row["run_id"]))
or row.get("agent_id")
or "lead_agent"
)[:128]
target_key = _target_key(
user_id=row.get("user_id"),
thread_id=str(row["thread_id"]),
run_id=str(row["run_id"]),
message_id=row.get("message_id"),
agent_id=agent_id,
)
# SQL unique indexes historically allowed several NULL-user rows.
# Keep those legacy records without blocking the new unique digest;
# one canonical row retains the deterministic future-upsert key.
if target_key in seen_target_keys:
legacy_raw = f"{target_key}\x1flegacy:{row['feedback_id']}"
target_key = hashlib.sha256(legacy_raw.encode("utf-8")).hexdigest()
else:
seen_target_keys.add(target_key)
bind.execute(
sa.update(feedback)
.where(feedback.c.feedback_id == row["feedback_id"])
.values(
agent_id=agent_id,
target_key=target_key,
reasons=_empty_reasons_value(),
updated_at=row.get("created_at"),
)
)
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TABLE):
return
columns = _columns()
if "agent_id" not in columns:
op.add_column(
_TABLE,
sa.Column(
"agent_id",
sa.String(length=128),
nullable=True,
server_default="lead_agent",
),
)
if "target_key" not in columns:
op.add_column(
_TABLE,
sa.Column("target_key", sa.String(length=64), nullable=True),
)
if "reasons" not in columns:
op.add_column(
_TABLE,
sa.Column("reasons", _json_type(), nullable=True),
)
if "updated_at" not in columns:
op.add_column(
_TABLE,
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
)
_backfill()
# SQLite needs a batch table rebuild for nullability changes; Alembic uses
# native ALTER statements on MySQL/PostgreSQL where possible.
with op.batch_alter_table(_TABLE) as batch:
batch.alter_column(
"agent_id",
existing_type=sa.String(length=128),
nullable=False,
server_default=None,
)
batch.alter_column(
"target_key",
existing_type=sa.String(length=64),
nullable=False,
)
batch.alter_column(
"reasons",
existing_type=_json_type(),
nullable=False,
)
batch.alter_column(
"updated_at",
existing_type=sa.DateTime(timezone=True),
nullable=False,
)
indexes = _indexes()
if _OLD_INDEX in indexes:
op.drop_index(_OLD_INDEX, table_name=_TABLE)
indexes = _indexes()
if _TARGET_INDEX not in indexes:
op.create_index(_TARGET_INDEX, _TABLE, ["target_key"], unique=True)
if _AGENT_INDEX not in indexes:
op.create_index(_AGENT_INDEX, _TABLE, ["agent_id", "created_at"])
def downgrade() -> None:
if not sa.inspect(op.get_bind()).has_table(_TABLE):
return
indexes = _indexes()
if _AGENT_INDEX in indexes:
op.drop_index(_AGENT_INDEX, table_name=_TABLE)
if _TARGET_INDEX in indexes:
op.drop_index(_TARGET_INDEX, table_name=_TABLE)
if _OLD_INDEX not in _indexes():
op.create_index(
_OLD_INDEX,
_TABLE,
["thread_id", "run_id", "user_id"],
unique=True,
mysql_length={"user_id": 63},
)
columns = _columns()
with op.batch_alter_table(_TABLE) as batch:
for name in ("updated_at", "reasons", "target_key", "agent_id"):
if name in columns:
batch.drop_column(name)
@@ -0,0 +1,45 @@
"""Persist the latest LLMWiki incremental-sync result.
Revision ID: 20260920_01
Revises: 20260916_02
Create Date: 2026-09-20
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20260920_01"
down_revision = "20260916_02"
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("llmwiki_wiki_sync_states"):
return
columns = {
str(column["name"])
for column in inspector.get_columns("llmwiki_wiki_sync_states")
}
if "last_sync_result" not in columns:
op.add_column(
"llmwiki_wiki_sync_states",
sa.Column("last_sync_result", PortableJSON(), nullable=True),
)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("llmwiki_wiki_sync_states"):
return
columns = {
str(column["name"])
for column in inspector.get_columns("llmwiki_wiki_sync_states")
}
if "last_sync_result" in columns:
op.drop_column("llmwiki_wiki_sync_states", "last_sync_result")
@@ -0,0 +1,36 @@
"""Add editable starter questions and retrieval settings to LLMWiki mappings."""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260920_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("llmwiki_knowledge_bases"):
return set()
return {str(column["name"]) for column in inspector.get_columns("llmwiki_knowledge_bases")}
def upgrade() -> None:
columns = _columns()
if not columns:
return
if "test_questions" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("test_questions", sa.JSON(), nullable=True))
if "retrieval_settings" not in columns:
op.add_column("llmwiki_knowledge_bases", sa.Column("retrieval_settings", sa.JSON(), nullable=True))
def downgrade() -> None:
columns = _columns()
if "retrieval_settings" in columns:
op.drop_column("llmwiki_knowledge_bases", "retrieval_settings")
if "test_questions" in columns:
op.drop_column("llmwiki_knowledge_bases", "test_questions")
@@ -35,6 +35,7 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.analysis_projects import AnalysisProjectConfigRow, AnalysisProjectRow, AnalysisProjectThreadRow
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
@@ -48,7 +49,7 @@ from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow, UserCompactMenuOverrideConfigRow
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
@@ -60,13 +61,19 @@ from deerflow.persistence.document_rewrite_versions.model import DocumentRewrite
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.model import (
LlmWikiWikiEdgeRow,
LlmWikiWikiPageRow,
LlmWikiWikiSyncStateRow,
LlmWikiWikiVectorRow,
)
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.notifications.model import NotificationRow
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
@@ -133,6 +140,7 @@ from deerflow.persistence.taskcop_tasks.model import TaskCopTaskRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.upload_policy.model import UploadPolicyRow
from deerflow.persistence.user.model import UserRow
from deerflow.persistence.user_preferences.model import UserPreferencesRow
from deerflow.persistence.user_prompts.model import UserCustomPromptRow
@@ -150,8 +158,11 @@ from deerflow.persistence.workflow_runs.model import (
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
__all__ = [
"AnalysisProjectConfigRow",
"FeedbackRow",
"FixedQuestionRow",
"GongwenDocumentRow",
"GongwenDocumentVersionRow",
"AgentExtrasRow",
"AgentFavoriteRow",
"AgentRow",
@@ -160,6 +171,7 @@ __all__ = [
"BusinessMappingRow",
"AssistantKnowledgeBaseRow",
"AssistantKnowledgeSourceRow",
"AssistantKnowledgeVectorRow",
"AssistantKnowledgeImportJobRow",
"AssistantKnowledgeWikiPageRow",
"AssistantKnowledgeWikiRevisionRow",
@@ -168,6 +180,7 @@ __all__ = [
"AssistantKnowledgeRelationRow",
"AssistantKnowledgeRelationContributionRow",
"CompactMenuOverrideRow",
"UserCompactMenuOverrideConfigRow",
"ConcurrencySampleRow",
"DashboardSessionRow",
"DocumentRewriteVersionRow",
@@ -201,9 +214,11 @@ __all__ = [
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
"LlmWikiWikiPageRow",
"LlmWikiWikiEdgeRow",
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"MenuOverrideRow",
"UserMenuOverrideConfigRow",
"NotificationRow",
"ParallelAgentTaskRow",
"PositionRoundtableNodeRow",
@@ -259,6 +274,8 @@ __all__ = [
"TagAssignmentRow",
"TagRow",
"TaskButtonRow",
"AnalysisProjectRow",
"AnalysisProjectThreadRow",
"TaskCopTaskRow",
"TaskReportRow",
"EmbedSessionRow",
@@ -266,6 +283,7 @@ __all__ = [
"ThreadMetaRow",
"ThreadShareRow",
"ToolCallMetricRow",
"UploadPolicyRow",
"UserCustomPromptRow",
"UserMemoryConfigRow",
"UserPreferencesRow",
@@ -57,12 +57,40 @@ class ParallelAgentTaskRepository:
data["messages"] = _loads(row.messages) or []
return data
async def list_tasks(self, user_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
"""列出当前用户的任务(轻量元信息,不含 messages,供历史列表)。"""
stmt = select(ParallelAgentTaskRow).where(ParallelAgentTaskRow.user_id == user_id).order_by(ParallelAgentTaskRow.updated_at.desc()).limit(limit)
async def list_tasks(
self,
user_id: str,
*,
limit: int = 200,
offset: int = 0,
id_prefix: str | None = None,
include_messages: bool = False,
) -> list[dict[str, Any]]:
"""List the current user's tasks, optionally including saved snapshots.
History-list callers keep the lightweight default. Server-side
projections such as the artifact library can opt into ``messages`` in
one query instead of issuing an N+1 ``get_task`` sequence. ``id_prefix``
and ``offset`` allow those projections to page only their own task
namespace without loading unrelated, potentially large transcripts.
"""
stmt = select(ParallelAgentTaskRow).where(ParallelAgentTaskRow.user_id == user_id)
if id_prefix:
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix))
stmt = (
stmt.order_by(
ParallelAgentTaskRow.updated_at.desc(),
ParallelAgentTaskRow.id.asc(),
)
.offset(offset)
.limit(limit)
)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._to_dict(row, with_messages=False) for row in result.scalars()]
return [
self._to_dict(row, with_messages=include_messages)
for row in result.scalars()
]
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
@@ -0,0 +1,48 @@
"""External-task multi-agent scheduled analysis persistence (任务分析定时)."""
from __future__ import annotations
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
from deerflow.persistence.task_analysis.data_blocks import (
DEFAULT_DATA_BLOCK_KEYS,
DEFAULT_DATA_BLOCKS,
merge_payload,
)
from deerflow.persistence.task_analysis.memory import MemoryTaskAnalysisStore
from deerflow.persistence.task_analysis.model import (
TaskAnalysisBindingRow,
TaskAnalysisContentVersionRow,
TaskAnalysisRunMessageRow,
TaskAnalysisRunRow,
TaskAnalysisTaskRow,
TaskDataBlockDefRow,
TaskDataBlockRow,
TaskDataPeriodRow,
TaskDataSubmissionRow,
)
from deerflow.persistence.task_analysis.sql import TaskAnalysisRepository
__all__ = [
"DEFAULT_DATA_BLOCK_KEYS",
"DEFAULT_DATA_BLOCKS",
"MemoryTaskAnalysisStore",
"TaskAnalysisBindingRow",
"TaskAnalysisContentVersionRow",
"TaskAnalysisRepository",
"TaskAnalysisRunMessageRow",
"TaskAnalysisRunRow",
"TaskAnalysisStore",
"TaskAnalysisTaskRow",
"TaskDataBlockDefRow",
"TaskDataBlockRow",
"TaskDataPeriodRow",
"TaskDataSubmissionRow",
"make_task_analysis_store",
"merge_payload",
]
def make_task_analysis_store(session_factory):
if session_factory is None:
return MemoryTaskAnalysisStore()
return TaskAnalysisRepository(session_factory)
@@ -0,0 +1,165 @@
"""Abstract store API for the external-task scheduled analysis feature."""
from __future__ import annotations
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any
class TaskAnalysisStore(ABC):
"""Tasks / bindings / runs / run-messages / content versions."""
# ── external tasks ──────────────────────────────────────────────────────
@abstractmethod
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
"""Create a task. Raises ValueError when source_task_id already exists."""
@abstractmethod
async def get_task(self, task_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def list_tasks(self) -> list[dict[str, Any]]:
"""All tasks newest first, each augmented with its binding (if any)."""
@abstractmethod
async def delete_task_cascade(self, task_id: str) -> bool:
"""Delete task + binding + runs + messages + versions. True if existed."""
# ── bindings ────────────────────────────────────────────────────────────
@abstractmethod
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
"""Create-or-update the 1:1 binding for a task (refresh chain snapshot)."""
@abstractmethod
async def delete_binding(self, external_task_id: str) -> bool: ...
@abstractmethod
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
"""Enabled bindings with next_run_at <= now (execution candidates)."""
@abstractmethod
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
"""Patch mutable fields (next_run_at / last_status / direction_notes...)."""
@abstractmethod
async def last_bound_chain_id(self) -> str | None:
"""Most recently updated binding's chain id (default option for create API)."""
# ── runs ────────────────────────────────────────────────────────────────
@abstractmethod
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
"""Idempotent create: same binding_id + scheduled_for returns existing row."""
@abstractmethod
async def get_run(self, run_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]: ...
@abstractmethod
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None: ...
@abstractmethod
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
"""Runs still pending/running past ``older_than`` (watchdog recovery)."""
# ── run messages ────────────────────────────────────────────────────────
@abstractmethod
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
"""Append batch of {seat_agent_id, stage_index, event_type, content}."""
@abstractmethod
async def list_run_messages(
self, run_id: str, *, seat: str | None = None, limit: int = 500
) -> list[dict[str, Any]]: ...
# ── content versions (timeline) ─────────────────────────────────────────
@abstractmethod
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
"""One version per run (idempotent on run_id: replaces existing)."""
@abstractmethod
async def list_versions(self, external_task_id: str, *, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:
"""Timeline versions newest first (对方详情接口数据源)."""
# ── data delivery: block defs / periods / submissions ────────────────────
@abstractmethod
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
"""Seed the 5 default block defs when absent (create/legacy tasks); return ordered."""
@abstractmethod
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
"""Block defs ordered by sort_order (empty list when never seeded)."""
@abstractmethod
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Patch title/sort_order per block_key. ValueError on unknown block_key."""
@abstractmethod
async def open_period(
self,
task_id: str,
*,
source: str,
run_id: str | None = None,
opened_at: datetime | None = None,
) -> dict[str, Any]:
"""Append a new timeline period (period_index = max+1). Idempotent on run_id.
``opened_at`` 默认取当前时间;演示/回填场景可显式指定刻度时刻。
"""
@abstractmethod
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None: ...
@abstractmethod
async def current_period(self, task_id: str) -> dict[str, Any] | None:
"""Latest period (submissions always land here)."""
@abstractmethod
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
"""Periods ascending by period_index (timeline ticks)."""
@abstractmethod
async def append_submission(
self,
task_id: str,
period_index: int,
block_key: str,
payload: dict[str, Any],
*,
submitted_by: str | None = None,
) -> dict[str, Any]:
"""Append to the submission log (seq per task+period+block)."""
@abstractmethod
async def list_submissions(
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
) -> list[dict[str, Any]]:
"""Submission log for one block of one period ordered by seq (merge source)."""
@abstractmethod
async def upsert_period_block(
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
) -> dict[str, Any]:
"""Write the merged payload for (task, period, block); bumps updated_at."""
@abstractmethod
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
"""All merged block rows of a task ({block_key, period_index, payload, updated_at})."""
@@ -0,0 +1,45 @@
"""Constants and pure helpers for the scheduled data-delivery blocks.
块 key 定死 ``block_1``~``block_6``(已定稿口径,不做席位↔块映射);payload 不做
schema 校验(schema 会变),只要求是 JSON 对象。合并规则:顶层 key 合并、同 key
后写覆盖 —— 多个智能体可各写一个块的不同部分,一个智能体也可写多个块。
"""
from __future__ import annotations
from typing import Any
# (block_key, 默认标题);建任务时自动种 6 条,标题/排序后续可经管理端修改。
DEFAULT_DATA_BLOCKS: tuple[tuple[str, str], ...] = (
("block_1", "数据块一"),
("block_2", "数据块二"),
("block_3", "数据块三"),
("block_4", "数据块四"),
("block_5", "数据块五"),
("block_6", "数据块六"),
)
DEFAULT_DATA_BLOCK_KEYS: tuple[str, ...] = tuple(key for key, _title in DEFAULT_DATA_BLOCKS)
# 期(时间轴刻度)触发来源。
PERIOD_TRIGGER_CRON = "cron"
PERIOD_TRIGGER_MANUAL = "manual"
PERIOD_TRIGGER_EXTERNAL = "external"
def default_block_defs() -> list[dict[str, Any]]:
return [
{"block_key": key, "title": title, "sort_order": index}
for index, (key, title) in enumerate(DEFAULT_DATA_BLOCKS, start=1)
]
def is_default_block_key(block_key: str) -> bool:
return block_key in DEFAULT_DATA_BLOCK_KEYS
def merge_payload(base: dict[str, Any], incoming: dict[str, Any]) -> dict[str, Any]:
"""顶层 key 合并、同 key 后写覆盖(纯函数,返回新 dict)。"""
merged = dict(base)
merged.update(incoming)
return merged
@@ -0,0 +1,458 @@
"""In-memory task-analysis store (no DB configured / tests).
Also serves as the behavioural reference for the SQL repository: field
normalisation (JSON encode/decode, ISO datetime) lives in ``_normalize_*``
helpers that the SQL layer mirrors.
"""
from __future__ import annotations
import json
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
def _iso(value: datetime | str | None) -> str | None:
if value is None:
return None
if isinstance(value, datetime):
return value.isoformat()
return str(value)
def _parse_dt(value: Any) -> datetime | None:
if value is None or isinstance(value, datetime):
return value
try:
return datetime.fromisoformat(str(value))
except ValueError:
return None
def _dump(value: Any) -> str | None:
return None if value is None else json.dumps(value, ensure_ascii=False)
def _load(value: str | None, default: Any) -> Any:
if value is None or value == "":
return default
try:
return json.loads(value)
except (TypeError, ValueError):
return default
class MemoryTaskAnalysisStore(TaskAnalysisStore):
def __init__(self) -> None:
self._tasks: dict[str, dict[str, Any]] = {}
self._bindings: dict[str, dict[str, Any]] = {} # keyed by external_task_id
self._runs: dict[str, dict[str, Any]] = {}
self._messages: list[dict[str, Any]] = []
self._versions: dict[str, dict[str, Any]] = {} # keyed by run_id
# 数据交付:块定义 / 期刻度 / 合并块 / 提交流水。
self._block_defs: dict[str, dict[str, dict[str, Any]]] = {} # task_id -> block_key -> def
self._periods: dict[str, list[dict[str, Any]]] = {} # task_id -> ordered by period_index
self._task_blocks: dict[tuple[str, int, str], dict[str, Any]] = {} # (task, period, block)
self._submissions: dict[tuple[str, int, str], list[dict[str, Any]]] = {}
self._seq = 0
# 绑定最近写入序号(时钟精度不足时仍能稳定判定“最近一次绑定”)。
self._binding_rev = 0
def _next_id(self, prefix: str) -> str:
self._seq += 1
return f"{prefix}-{self._seq:06d}"
# ── data delivery helpers ───────────────────────────────────────────────
def _seed_block_defs(self, task_id: str) -> None:
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
existing = self._block_defs.setdefault(task_id, {})
now = datetime.now(UTC).isoformat()
for definition in default_block_defs():
if definition["block_key"] in existing:
continue
existing[definition["block_key"]] = {
"id": self._next_id("tdbd"),
"task_id": task_id,
**definition,
"created_at": now,
"updated_at": now,
}
def _ordered_defs(self, task_id: str) -> list[dict[str, Any]]:
rows = list(self._block_defs.get(task_id, {}).values())
rows.sort(key=lambda d: (d.get("sort_order", 0), d.get("block_key", "")))
return [dict(row) for row in rows]
def _period_rows(self, task_id: str) -> list[dict[str, Any]]:
rows = list(self._periods.get(task_id, []))
rows.sort(key=lambda p: p["period_index"])
return rows
# ── tasks ───────────────────────────────────────────────────────────────
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
source = str(data["source_task_id"])
if await self.get_task_by_source(source) is not None:
raise ValueError(f"source_task_id already exists: {source}")
now = datetime.now(UTC).isoformat()
task = {
"id": self._next_id("ta"),
"source_task_id": source,
"name": str(data.get("name") or ""),
"description": data.get("description"),
"created_at": now,
"updated_at": now,
}
self._tasks[task["id"]] = dict(task)
# 建任务即种 6 条数据块定义(块 key 定死,标题后续可改)。
self._seed_block_defs(task["id"])
return dict(task)
async def get_task(self, task_id: str) -> dict[str, Any] | None:
task = self._tasks.get(task_id)
return dict(task) if task else None
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None:
for task in self._tasks.values():
if task["source_task_id"] == source_task_id:
return dict(task)
return None
async def list_tasks(self) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for task in sorted(self._tasks.values(), key=lambda t: t["created_at"], reverse=True):
row = dict(task)
row["binding"] = dict(self._bindings[task["id"]]) if task["id"] in self._bindings else None
out.append(row)
return out
async def delete_task_cascade(self, task_id: str) -> bool:
if task_id not in self._tasks:
return False
del self._tasks[task_id]
self._bindings.pop(task_id, None)
run_ids = [run_id for run_id, run in self._runs.items() if run["external_task_id"] == task_id]
for run_id in run_ids:
del self._runs[run_id]
self._versions.pop(run_id, None)
self._messages = [m for m in self._messages if m["run_id"] not in set(run_ids)]
self._block_defs.pop(task_id, None)
self._periods.pop(task_id, None)
self._task_blocks = {
key: row for key, row in self._task_blocks.items() if key[0] != task_id
}
self._submissions = {
key: rows for key, rows in self._submissions.items() if key[0] != task_id
}
return True
# ── bindings ────────────────────────────────────────────────────────────
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None:
binding = self._bindings.get(external_task_id)
return dict(binding) if binding else None
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None:
for binding in self._bindings.values():
if binding["id"] == binding_id:
return dict(binding)
return None
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
existing = self._bindings.get(external_task_id)
now = datetime.now(UTC).isoformat()
self._binding_rev += 1
if existing is None:
row = {
"id": self._next_id("tab"),
"external_task_id": external_task_id,
"chain_id": str(data["chain_id"]),
"chain_snapshot": data.get("chain_snapshot") or {},
"cron_expr": str(data.get("cron_expr") or ""),
"timezone": str(data.get("timezone") or "Asia/Shanghai"),
"schedule_type": str(data.get("schedule_type") or "cron"),
"enabled": bool(data.get("enabled", True)),
"next_run_at": _iso(data.get("next_run_at")),
"coordinator_prompt": data.get("coordinator_prompt"),
"direction_notes": list(data.get("direction_notes") or []),
"last_run_at": None,
"last_status": None,
"model": data.get("model"),
"created_by": data.get("created_by"),
"created_at": now,
"updated_at": now,
"_rev": self._binding_rev,
}
else:
row = dict(existing)
row["_rev"] = self._binding_rev
row.update({k: v for k, v in data.items() if k in {
"chain_id", "chain_snapshot", "cron_expr", "timezone", "schedule_type", "enabled",
"next_run_at", "coordinator_prompt", "model",
}})
if "direction_notes" in data:
row["direction_notes"] = list(data["direction_notes"])
row["updated_at"] = now
self._bindings[external_task_id] = row
return dict(row)
async def delete_binding(self, external_task_id: str) -> bool:
return self._bindings.pop(external_task_id, None) is not None
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
due = []
for binding in self._bindings.values():
if not binding.get("enabled"):
continue
next_at = _parse_dt(binding.get("next_run_at"))
if next_at is not None and next_at <= now:
due.append(binding)
due.sort(key=lambda b: b.get("next_run_at") or "")
return [dict(b) for b in due[:limit]]
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
for key, binding in self._bindings.items():
if binding["id"] == binding_id:
binding.update({k: v for k, v in patch.items() if k != "id"})
binding["updated_at"] = datetime.now(UTC).isoformat()
return dict(binding)
return None
async def last_bound_chain_id(self) -> str | None:
if not self._bindings:
return None
latest = max(self._bindings.values(), key=lambda b: b.get("_rev", 0))
return latest.get("chain_id")
# ── runs ────────────────────────────────────────────────────────────────
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
binding_id = data["binding_id"]
scheduled_for = _parse_dt(data["scheduled_for"]) or datetime.now(UTC)
for run in self._runs.values():
if run["binding_id"] == binding_id and run["scheduled_for"] == scheduled_for.isoformat():
return dict(run)
now = datetime.now(UTC).isoformat()
row = {
"id": self._next_id("tar"),
"binding_id": binding_id,
"external_task_id": str(data["external_task_id"]),
"trigger": str(data.get("trigger") or "schedule"),
"scheduled_for": scheduled_for.isoformat(),
"started_at": now,
"finished_at": None,
"status": str(data.get("status") or "pending"),
"stages_snapshot": data.get("stages_snapshot") or [],
"stage_goals_snapshot": data.get("stage_goals_snapshot") or [],
"seat_names_snapshot": data.get("seat_names_snapshot") or {},
"seat_results": data.get("seat_results") or {},
"error": data.get("error"),
"retry_count": 0,
"created_at": now,
"updated_at": now,
}
self._runs[row["id"]] = row
return dict(row)
async def get_run(self, run_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
return dict(run) if run else None
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
rows = [r for r in self._runs.values() if r["external_task_id"] == external_task_id]
rows.sort(key=lambda r: r["started_at"], reverse=True)
return [dict(r) for r in rows[:limit]]
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None:
return None
run.update({k: v for k, v in patch.items() if k not in {"id", "binding_id"}})
run["updated_at"] = datetime.now(UTC).isoformat()
return dict(run)
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
out = []
for run in self._runs.values():
if run["status"] in {"pending", "running"}:
started = _parse_dt(run["started_at"])
if started is not None and started < older_than:
out.append(run)
out.sort(key=lambda r: r["started_at"])
return [dict(r) for r in out[:limit]]
# ── messages ────────────────────────────────────────────────────────────
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
base_seq = sum(1 for m in self._messages if m["run_id"] == run_id)
added = 0
for index, message in enumerate(messages):
self._messages.append({
"id": self._next_id("tam"),
"run_id": run_id,
"seq": base_seq + index,
"seat_agent_id": str(message.get("seat_agent_id") or ""),
"stage_index": int(message.get("stage_index") or 0),
"event_type": str(message.get("event_type") or "llm.ai"),
"content": message.get("content"),
"created_at": datetime.now(UTC).isoformat(),
})
added += 1
return added
async def list_run_messages(
self, run_id: str, *, seat: str | None = None, limit: int = 500
) -> list[dict[str, Any]]:
rows = [m for m in self._messages if m["run_id"] == run_id]
if seat is not None:
rows = [m for m in rows if m["seat_agent_id"] == seat]
rows.sort(key=lambda m: m["seq"])
return [dict(m) for m in rows[:limit]]
# ── versions ────────────────────────────────────────────────────────────
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
run_id = str(data["run_id"])
row = {
"id": self._next_id("tav"),
"external_task_id": str(data["external_task_id"]),
"run_id": run_id,
"blocks": data.get("blocks") or {},
"created_at": datetime.now(UTC).isoformat(),
}
self._versions[run_id] = row
return dict(row)
async def list_versions(
self, external_task_id: str, *, limit: int = 50, offset: int = 0
) -> list[dict[str, Any]]:
rows = [v for v in self._versions.values() if v["external_task_id"] == external_task_id]
rows.sort(key=lambda v: v["created_at"], reverse=True)
return [dict(v) for v in rows[offset : offset + limit]]
# ── data delivery: block defs / periods / submissions ────────────────────
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
self._seed_block_defs(task_id)
return self._ordered_defs(task_id)
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
return self._ordered_defs(task_id)
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
definitions = self._block_defs.setdefault(task_id, {})
now = datetime.now(UTC).isoformat()
for update in updates:
block_key = str(update.get("block_key") or "")
row = definitions.get(block_key)
if row is None:
raise ValueError(f"未知数据块:{block_key}")
if update.get("title") is not None:
row["title"] = str(update["title"])
if update.get("sort_order") is not None:
row["sort_order"] = int(update["sort_order"])
row["updated_at"] = now
return self._ordered_defs(task_id)
async def open_period(
self,
task_id: str,
*,
source: str,
run_id: str | None = None,
opened_at: datetime | None = None,
) -> dict[str, Any]:
if run_id:
for period in self._periods.get(task_id, []):
if period.get("run_id") == run_id:
return dict(period)
rows = self._periods.setdefault(task_id, [])
index = (rows[-1]["period_index"] if rows else 0) + 1
now = datetime.now(UTC).isoformat()
row = {
"id": self._next_id("tdp"),
"task_id": task_id,
"period_index": index,
"opened_at": _iso(opened_at) or now,
"trigger": str(source or "cron"),
"run_id": run_id,
"created_at": now,
}
rows.append(row)
return dict(row)
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None:
for period in self._period_rows(task_id):
if period["period_index"] == period_index:
return dict(period)
return None
async def current_period(self, task_id: str) -> dict[str, Any] | None:
rows = self._period_rows(task_id)
return dict(rows[-1]) if rows else None
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
rows = self._period_rows(task_id)
return [dict(p) for p in rows[-limit:]]
async def append_submission(
self,
task_id: str,
period_index: int,
block_key: str,
payload: dict[str, Any],
*,
submitted_by: str | None = None,
) -> dict[str, Any]:
key = (task_id, period_index, block_key)
rows = self._submissions.setdefault(key, [])
row = {
"id": self._next_id("tds"),
"task_id": task_id,
"period_index": period_index,
"block_key": block_key,
"payload": dict(payload),
"submitted_by": submitted_by,
"seq": len(rows),
"created_at": datetime.now(UTC).isoformat(),
}
rows.append(row)
return dict(row)
async def list_submissions(
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
) -> list[dict[str, Any]]:
rows = list(self._submissions.get((task_id, period_index, block_key), []))
rows.sort(key=lambda s: s["seq"])
return [dict(s) for s in rows[:limit]]
async def upsert_period_block(
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
) -> dict[str, Any]:
key = (task_id, period_index, block_key)
now = datetime.now(UTC).isoformat()
row = self._task_blocks.get(key)
if row is None:
row = {
"id": self._next_id("tdb"),
"task_id": task_id,
"period_index": period_index,
"block_key": block_key,
"payload": dict(payload),
"updated_at": now,
"created_at": now,
}
self._task_blocks[key] = row
else:
row["payload"] = dict(payload)
row["updated_at"] = now
return dict(row)
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
rows = [dict(row) for key, row in self._task_blocks.items() if key[0] == task_id]
rows.sort(key=lambda r: (r["period_index"], r["block_key"]))
return rows[:limit]
@@ -0,0 +1,207 @@
"""ORM models for the external-task multi-agent scheduled analysis (任务分析定时).
Feature summary (design doc in the frontend ``official-task-analysis`` module):
an external system pushes tasks via our create API; each task binds 1:1 to a
business-chain orchestration plus a cron schedule; the scheduler runs the chain
stage-by-stage (seats within a stage in parallel) and persists per-seat process
messages plus a content "timeline" version per run. Manual runs wrap the
roundtable Step-2 coordinator Q&A and are stored with the same message table
(``seat_agent_id`` distinguishes seats; the coordinator uses ``__coordinator__``).
Tables:
- ``task_analysis_tasks`` — external tasks (source_task_id unique)
- ``task_analysis_bindings`` — 1:1 binding (chain snapshot + cron + notes)
- ``task_analysis_runs`` — executions (trigger: schedule | manual)
- ``task_analysis_run_messages`` — per-seat process messages for a run
- ``task_analysis_content_versions`` — one timeline version per run
- ``task_data_block_defs`` — the 5 fixed data blocks per task (title/sort editable)
- ``task_data_periods`` — data-delivery timeline ticks (period_index per task)
- ``task_data_blocks`` — merged payload per (task, period, block)
- ``task_data_submissions`` — append-only submission log feeding the merge
JSON payloads are stored as LONGTEXT-backed JSON strings (PortableLongText),
mirroring the sibling roundtable persistence modules for headroom.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
def _utcnow() -> datetime:
return datetime.now(UTC)
class TaskAnalysisTaskRow(Base):
__tablename__ = "task_analysis_tasks"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# 对方系统任务号(创建接口传入;唯一)。
source_task_id: Mapped[str] = mapped_column(String(128), nullable=False, unique=True, index=True)
name: Mapped[str] = mapped_column(String(512), nullable=False, default="")
# 任务简介:创建时落库的存量即执行时用的版本(不回拉对方接口)。
description: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
class TaskAnalysisBindingRow(Base):
__tablename__ = "task_analysis_bindings"
__table_args__ = (UniqueConstraint("external_task_id", name="uq_task_analysis_binding_task"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# 引用 roundtable_chains.id;执行前校验,链缺失/停用 → 跳过并标记 chain_missing。
chain_id: Mapped[str] = mapped_column(String(64), nullable=False)
# 执行编排快照 JSON:{seats, stages, stage_goals, coordinator_prompt}。
# 每次执行前刷新(跟随链最新版),历史 run 用自己的 stages_snapshot 回放。
chain_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
cron_expr: Mapped[str] = mapped_column(String(64), nullable=False, default="")
timezone: Mapped[str] = mapped_column(String(64), nullable=False, default="Asia/Shanghai")
# 执行方式:cron = 按频率定时;once = 只执行一次(创建即执行,next_run_at 恒空)。
schedule_type: Mapped[str] = mapped_column(String(16), nullable=False, default="cron", server_default="cron")
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="1")
next_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
# 绑定级总控提示(与链内 coordinator_prompt 拼接使用)。
coordinator_prompt: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# 问答沉淀的「执行方向备注」JSON 数组(逐条追加,注入后续每次定时执行)。
direction_notes: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
last_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
# 执行模型(席位 run 的 model_name;NULL = 系统/席位默认)。
model: Mapped[str | None] = mapped_column(String(128), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
class TaskAnalysisRunRow(Base):
__tablename__ = "task_analysis_runs"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
binding_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# schedule = cron 定时执行;manual = 总控问答(对方详情页发起)。
trigger: Mapped[str] = mapped_column(String(16), nullable=False, default="schedule")
# 幂等键:同 binding + scheduled_for 只建一条(对齐 scheduled_tasks.create_run_once)。
scheduled_for: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False)
started_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
# pending | running | succeeded | failed | partial
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
# 本次编排快照 JSON:[[seat_id,...],...](回放不受链后续修改影响)。
stages_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
stage_goals_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
seat_names_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# 各席位结果 JSON:{agent_id: {state, duration_ms, thread_id, error}}。
seat_results: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
class TaskAnalysisRunMessageRow(Base):
__tablename__ = "task_analysis_run_messages"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# 消息归属席位(agent_id);总控消息用固定标识 __coordinator__。
seat_agent_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
stage_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# input | llm.ai | tool.result | artifact(对齐现有 run messages 事件分类)。
event_type: Mapped[str] = mapped_column(String(32), nullable=False, default="llm.ai")
content: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
class TaskAnalysisContentVersionRow(Base):
__tablename__ = "task_analysis_content_versions"
__table_args__ = (UniqueConstraint("run_id", name="uq_task_analysis_version_run"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# 一次执行 = 一期时间线版本(1:1)。
run_id: Mapped[str] = mapped_column(String(64), nullable=False)
# JSON:{key: {name, seat_agent_id, content, format}} —— 对方详情的各块内容。
blocks: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
class TaskDataBlockDefRow(Base):
"""数据块定义:块 key 定死 block_1~block_6,标题默认「数据块一~六」可改。"""
__tablename__ = "task_data_block_defs"
__table_args__ = (UniqueConstraint("task_id", "block_key", name="uq_task_data_block_def_key"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# 关联 task_analysis_tasks.id(内部任务 id)。
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
title: Mapped[str] = mapped_column(String(128), nullable=False, default="")
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
class TaskDataPeriodRow(Base):
"""数据交付时间轴刻度:每个任务 period_index 从 1 递增;run_id 作幂等锚点。"""
__tablename__ = "task_data_periods"
__table_args__ = (
UniqueConstraint("task_id", "period_index", name="uq_task_data_period_index"),
UniqueConstraint("run_id", name="uq_task_data_period_run"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
opened_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
# cron = 定时执行开期;manual = 手动触发/总控问答开期;external = 首次提交兜底开期。
trigger: Mapped[str] = mapped_column(String(16), nullable=False, default="cron")
# 触发本期的执行记录(问答/定时执行共用 run 表);提交兜底开的期没有 run。
run_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
class TaskDataBlockRow(Base):
"""每期每块的合并结果(由 task_data_submissions 流水按序合并得出)。"""
__tablename__ = "task_data_blocks"
__table_args__ = (
UniqueConstraint("task_id", "period_index", "block_key", name="uq_task_data_block_tpb"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
# JSON 对象(顶层 key 合并、同 key 后写覆盖)。
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
class TaskDataSubmissionRow(Base):
"""入库流水(追加只读):块 payload 的一致性来源。"""
__tablename__ = "task_data_submissions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
# 本次提交的 JSON 对象片段(合并语义:顶层 key 合并、同 key 后写覆盖)。
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# 提交方(席位 agent id / 用户 id,仅审计)。
submitted_by: Mapped[str | None] = mapped_column(String(128), nullable=True)
# 同 (task, period, block) 内的提交序号,从 0 递增 —— 合并顺序依据。
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
@@ -0,0 +1,785 @@
"""SQLAlchemy-backed task-analysis store.
Mirrors the memory implementation's normalisation: JSON payloads
(chain_snapshot / stages / seat_results / blocks / direction_notes) are stored
as LONGTEXT JSON strings; datetimes are ISO strings in dict form.
"""
from __future__ import annotations
import json
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
from deerflow.persistence.task_analysis.model import (
TaskAnalysisBindingRow,
TaskAnalysisContentVersionRow,
TaskAnalysisRunMessageRow,
TaskAnalysisRunRow,
TaskAnalysisTaskRow,
TaskDataBlockDefRow,
TaskDataBlockRow,
TaskDataPeriodRow,
TaskDataSubmissionRow,
)
def _iso(value: datetime | str | None) -> str | None:
if value is None:
return None
if isinstance(value, datetime):
return value.isoformat()
return str(value)
def _utcnow() -> datetime:
return datetime.now(UTC)
def _parse_dt(value: Any) -> datetime | None:
if value is None or isinstance(value, datetime):
return value
try:
return datetime.fromisoformat(str(value))
except ValueError:
return None
def _dump(value: Any) -> str | None:
return None if value is None else json.dumps(value, ensure_ascii=False)
def _load(value: str | None, default: Any) -> Any:
if value is None or value == "":
return default
try:
return json.loads(value)
except (TypeError, ValueError):
return default
def _task_dict(row: TaskAnalysisTaskRow) -> dict[str, Any]:
data = row.to_dict()
data["created_at"] = _iso(row.created_at)
data["updated_at"] = _iso(row.updated_at)
return data
def _binding_dict(row: TaskAnalysisBindingRow) -> dict[str, Any]:
data = row.to_dict()
data["chain_snapshot"] = _load(row.chain_snapshot, {})
data["direction_notes"] = _load(row.direction_notes, [])
for key in ("next_run_at", "last_run_at", "created_at", "updated_at"):
data[key] = _iso(getattr(row, key))
return data
def _run_dict(row: TaskAnalysisRunRow) -> dict[str, Any]:
data = row.to_dict()
data["scheduled_for"] = _iso(row.scheduled_for)
data["started_at"] = _iso(row.started_at)
data["finished_at"] = _iso(row.finished_at)
data["created_at"] = _iso(row.created_at)
data["updated_at"] = _iso(row.updated_at)
for key, default in (
("stages_snapshot", []),
("stage_goals_snapshot", []),
("seat_names_snapshot", {}),
("seat_results", {}),
):
data[key] = _load(getattr(row, key), default)
return data
def _message_dict(row: TaskAnalysisRunMessageRow) -> dict[str, Any]:
data = row.to_dict()
data["created_at"] = _iso(row.created_at)
return data
def _version_dict(row: TaskAnalysisContentVersionRow) -> dict[str, Any]:
data = row.to_dict()
data["blocks"] = _load(row.blocks, {})
data["created_at"] = _iso(row.created_at)
return data
def _block_def_dict(row: TaskDataBlockDefRow) -> dict[str, Any]:
data = row.to_dict()
data["created_at"] = _iso(row.created_at)
data["updated_at"] = _iso(row.updated_at)
return data
def _period_dict(row: TaskDataPeriodRow) -> dict[str, Any]:
data = row.to_dict()
data["opened_at"] = _iso(row.opened_at)
data["created_at"] = _iso(row.created_at)
return data
def _task_block_dict(row: TaskDataBlockRow) -> dict[str, Any]:
data = row.to_dict()
data["payload"] = _load(row.payload, {})
data["updated_at"] = _iso(row.updated_at)
data["created_at"] = _iso(row.created_at)
return data
def _submission_dict(row: TaskDataSubmissionRow) -> dict[str, Any]:
data = row.to_dict()
data["payload"] = _load(row.payload, {})
data["created_at"] = _iso(row.created_at)
return data
class TaskAnalysisRepository(TaskAnalysisStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# ── tasks ───────────────────────────────────────────────────────────────
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
source = str(data["source_task_id"])
async with self._sf() as session:
existing = await session.execute(
select(TaskAnalysisTaskRow).where(TaskAnalysisTaskRow.source_task_id == source)
)
if existing.scalar_one_or_none() is not None:
raise ValueError(f"source_task_id already exists: {source}")
row = TaskAnalysisTaskRow(
id=str(data.get("id") or f"ta-{source}"),
source_task_id=source,
name=str(data.get("name") or ""),
description=data.get("description"),
)
session.add(row)
# 建任务即种 6 条数据块定义(块 key 定死,标题后续可改)。
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
for definition in default_block_defs():
session.add(
TaskDataBlockDefRow(
id=f"tdbd-{row.id}-{definition['block_key']}",
task_id=row.id,
block_key=definition["block_key"],
title=definition["title"],
sort_order=definition["sort_order"],
)
)
await session.commit()
await session.refresh(row)
return _task_dict(row)
async def get_task(self, task_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(TaskAnalysisTaskRow, task_id)
return _task_dict(row) if row else None
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisTaskRow).where(TaskAnalysisTaskRow.source_task_id == source_task_id)
)
row = result.scalar_one_or_none()
return _task_dict(row) if row else None
async def list_tasks(self) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisTaskRow).order_by(TaskAnalysisTaskRow.created_at.desc())
)
tasks = [_task_dict(row) for row in result.scalars()]
bindings = await session.execute(select(TaskAnalysisBindingRow))
by_task: dict[str, dict[str, Any]] = {}
for row in bindings.scalars():
by_task[row.external_task_id] = _binding_dict(row)
for task in tasks:
task["binding"] = by_task.get(task["id"])
return tasks
async def delete_task_cascade(self, task_id: str) -> bool:
async with self._sf() as session:
row = await session.get(TaskAnalysisTaskRow, task_id)
if row is None:
return False
run_ids = (
(
await session.execute(
select(TaskAnalysisRunRow.id).where(TaskAnalysisRunRow.external_task_id == task_id)
)
)
.scalars()
.all()
)
await session.execute(
delete(TaskAnalysisBindingRow).where(TaskAnalysisBindingRow.external_task_id == task_id)
)
await session.execute(
delete(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
)
await session.execute(
delete(TaskDataPeriodRow).where(TaskDataPeriodRow.task_id == task_id)
)
await session.execute(
delete(TaskDataBlockRow).where(TaskDataBlockRow.task_id == task_id)
)
await session.execute(
delete(TaskDataSubmissionRow).where(TaskDataSubmissionRow.task_id == task_id)
)
if run_ids:
await session.execute(
delete(TaskAnalysisRunMessageRow).where(TaskAnalysisRunMessageRow.run_id.in_(run_ids))
)
await session.execute(
delete(TaskAnalysisContentVersionRow).where(
TaskAnalysisContentVersionRow.run_id.in_(run_ids)
)
)
await session.execute(
delete(TaskAnalysisRunRow).where(TaskAnalysisRunRow.external_task_id == task_id)
)
await session.delete(row)
await session.commit()
return True
# ── bindings ────────────────────────────────────────────────────────────
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisBindingRow).where(
TaskAnalysisBindingRow.external_task_id == external_task_id
)
)
row = result.scalar_one_or_none()
return _binding_dict(row) if row else None
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(TaskAnalysisBindingRow, binding_id)
return _binding_dict(row) if row else None
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisBindingRow).where(
TaskAnalysisBindingRow.external_task_id == external_task_id
)
)
row = result.scalar_one_or_none()
if row is None:
row = TaskAnalysisBindingRow(
id=str(data.get("id") or f"tab-{external_task_id}"),
external_task_id=external_task_id,
chain_id=str(data["chain_id"]),
created_by=data.get("created_by"),
)
session.add(row)
row.chain_id = str(data.get("chain_id") or row.chain_id)
row.chain_snapshot = _dump(data.get("chain_snapshot") or {})
row.cron_expr = str(data.get("cron_expr") or row.cron_expr or "")
row.timezone = str(data.get("timezone") or row.timezone or "Asia/Shanghai")
if "schedule_type" in data:
row.schedule_type = str(data.get("schedule_type") or "cron")
row.model = data.get("model")
if "enabled" in data:
row.enabled = bool(data["enabled"])
if "next_run_at" in data:
row.next_run_at = _parse_dt(data.get("next_run_at"))
row.coordinator_prompt = data.get("coordinator_prompt")
if "direction_notes" in data:
row.direction_notes = _dump(list(data["direction_notes"] or []))
await session.commit()
await session.refresh(row)
return _binding_dict(row)
async def delete_binding(self, external_task_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(TaskAnalysisBindingRow).where(
TaskAnalysisBindingRow.external_task_id == external_task_id
)
)
await session.commit()
return bool(result.rowcount)
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisBindingRow)
.where(
TaskAnalysisBindingRow.enabled.is_(True),
TaskAnalysisBindingRow.next_run_at.is_not(None),
TaskAnalysisBindingRow.next_run_at <= now,
)
.order_by(TaskAnalysisBindingRow.next_run_at.asc())
.limit(limit)
)
return [_binding_dict(row) for row in result.scalars()]
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
allowed = {
"chain_id", "cron_expr", "timezone", "schedule_type", "enabled", "next_run_at",
"last_run_at", "last_status", "coordinator_prompt", "model",
}
values: dict[str, Any] = {k: v for k, v in patch.items() if k in allowed}
if "next_run_at" in values:
values["next_run_at"] = _parse_dt(values["next_run_at"])
if "last_run_at" in values:
values["last_run_at"] = _parse_dt(values["last_run_at"])
if "direction_notes" in patch:
values["direction_notes"] = _dump(list(patch["direction_notes"] or []))
if not values:
row = await self._get_binding_row(binding_id)
return _binding_dict(row) if row else None
async with self._sf() as session:
await session.execute(
update(TaskAnalysisBindingRow).where(TaskAnalysisBindingRow.id == binding_id).values(**values)
)
await session.commit()
row = await session.get(TaskAnalysisBindingRow, binding_id)
return _binding_dict(row) if row else None
async def _get_binding_row(self, binding_id: str) -> TaskAnalysisBindingRow | None:
async with self._sf() as session:
return await session.get(TaskAnalysisBindingRow, binding_id)
async def last_bound_chain_id(self) -> str | None:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisBindingRow.chain_id)
.order_by(TaskAnalysisBindingRow.updated_at.desc())
.limit(1)
)
return result.scalar_one_or_none()
# ── runs ────────────────────────────────────────────────────────────────
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
binding_id = str(data["binding_id"])
scheduled_for = _parse_dt(data["scheduled_for"]) or datetime.now(UTC)
import uuid as _uuid
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisRunRow).where(
TaskAnalysisRunRow.binding_id == binding_id,
TaskAnalysisRunRow.scheduled_for == scheduled_for,
)
)
existing = result.scalar_one_or_none()
if existing is not None:
return _run_dict(existing)
row = TaskAnalysisRunRow(
id=str(data.get("id") or f"tar-{_uuid.uuid4()}"),
binding_id=binding_id,
external_task_id=str(data["external_task_id"]),
trigger=str(data.get("trigger") or "schedule"),
scheduled_for=scheduled_for,
status=str(data.get("status") or "pending"),
stages_snapshot=_dump(data.get("stages_snapshot") or []),
stage_goals_snapshot=_dump(data.get("stage_goals_snapshot") or []),
seat_names_snapshot=_dump(data.get("seat_names_snapshot") or {}),
seat_results=_dump(data.get("seat_results") or {}),
)
session.add(row)
await session.commit()
await session.refresh(row)
return _run_dict(row)
async def get_run(self, run_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(TaskAnalysisRunRow, run_id)
return _run_dict(row) if row else None
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisRunRow)
.where(TaskAnalysisRunRow.external_task_id == external_task_id)
.order_by(TaskAnalysisRunRow.started_at.desc())
.limit(limit)
)
return [_run_dict(row) for row in result.scalars()]
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
allowed = {
"trigger", "status", "finished_at", "error", "retry_count",
"stages_snapshot", "stage_goals_snapshot", "seat_names_snapshot", "seat_results",
}
values: dict[str, Any] = {}
for key, value in patch.items():
if key not in allowed:
continue
if key == "finished_at":
values[key] = _parse_dt(value)
elif key in {"stages_snapshot", "stage_goals_snapshot", "seat_names_snapshot", "seat_results"}:
values[key] = _dump(value)
else:
values[key] = value
if not values:
row = await self._get_run_row(run_id)
return _run_dict(row) if row else None
async with self._sf() as session:
await session.execute(
update(TaskAnalysisRunRow).where(TaskAnalysisRunRow.id == run_id).values(**values)
)
await session.commit()
row = await session.get(TaskAnalysisRunRow, run_id)
return _run_dict(row) if row else None
async def _get_run_row(self, run_id: str) -> TaskAnalysisRunRow | None:
async with self._sf() as session:
return await session.get(TaskAnalysisRunRow, run_id)
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisRunRow)
.where(
TaskAnalysisRunRow.status.in_(("pending", "running")),
TaskAnalysisRunRow.started_at < older_than,
)
.order_by(TaskAnalysisRunRow.started_at.asc())
.limit(limit)
)
return [_run_dict(row) for row in result.scalars()]
# ── messages ────────────────────────────────────────────────────────────
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
import uuid as _uuid
async with self._sf() as session:
existing = (
(
await session.execute(
select(TaskAnalysisRunMessageRow.id).where(
TaskAnalysisRunMessageRow.run_id == run_id
)
)
)
.scalars()
.all()
)
base_seq = len(existing)
rows = [
TaskAnalysisRunMessageRow(
id=str(_uuid.uuid4()),
run_id=run_id,
seq=base_seq + index,
seat_agent_id=str(message.get("seat_agent_id") or ""),
stage_index=int(message.get("stage_index") or 0),
event_type=str(message.get("event_type") or "llm.ai"),
content=message.get("content"),
)
for index, message in enumerate(messages)
]
session.add_all(rows)
await session.commit()
return len(rows)
async def list_run_messages(
self, run_id: str, *, seat: str | None = None, limit: int = 500
) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = (
select(TaskAnalysisRunMessageRow)
.where(TaskAnalysisRunMessageRow.run_id == run_id)
.order_by(TaskAnalysisRunMessageRow.seq.asc())
.limit(limit)
)
if seat is not None:
stmt = stmt.where(TaskAnalysisRunMessageRow.seat_agent_id == seat)
result = await session.execute(stmt)
return [_message_dict(row) for row in result.scalars()]
# ── versions ────────────────────────────────────────────────────────────
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
import uuid as _uuid
run_id = str(data["run_id"])
async with self._sf() as session:
await session.execute(
delete(TaskAnalysisContentVersionRow).where(
TaskAnalysisContentVersionRow.run_id == run_id
)
)
row = TaskAnalysisContentVersionRow(
id=str(_uuid.uuid4()),
external_task_id=str(data["external_task_id"]),
run_id=run_id,
blocks=_dump(data.get("blocks") or {}),
)
session.add(row)
await session.commit()
await session.refresh(row)
return _version_dict(row)
async def list_versions(
self, external_task_id: str, *, limit: int = 50, offset: int = 0
) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisContentVersionRow)
.where(TaskAnalysisContentVersionRow.external_task_id == external_task_id)
.order_by(TaskAnalysisContentVersionRow.created_at.desc())
.offset(offset)
.limit(limit)
)
return [_version_dict(row) for row in result.scalars()]
# ── data delivery: block defs / periods / submissions ────────────────────
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
async with self._sf() as session:
result = await session.execute(
select(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
)
existing_keys = {row.block_key for row in result.scalars()}
missing = [
definition
for definition in default_block_defs()
if definition["block_key"] not in existing_keys
]
if not missing:
return await self._ordered_defs(session, task_id)
for definition in missing:
session.add(
TaskDataBlockDefRow(
id=f"tdbd-{task_id}-{definition['block_key']}",
task_id=task_id,
block_key=definition["block_key"],
title=definition["title"],
sort_order=definition["sort_order"],
)
)
try:
await session.commit()
except IntegrityError:
# 并发补种:另一个请求先种好了,沿用其结果。
await session.rollback()
return await self._ordered_defs(session, task_id)
@staticmethod
async def _ordered_defs(session: AsyncSession, task_id: str) -> list[dict[str, Any]]:
result = await session.execute(
select(TaskDataBlockDefRow)
.where(TaskDataBlockDefRow.task_id == task_id)
.order_by(TaskDataBlockDefRow.sort_order.asc(), TaskDataBlockDefRow.block_key.asc())
)
return [_block_def_dict(row) for row in result.scalars()]
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
async with self._sf() as session:
return await self._ordered_defs(session, task_id)
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
)
rows = {row.block_key: row for row in result.scalars()}
touched = False
for item in updates:
block_key = str(item.get("block_key") or "")
row = rows.get(block_key)
if row is None:
raise ValueError(f"未知数据块:{block_key}")
patch: dict[str, Any] = {}
if item.get("title") is not None:
patch["title"] = str(item["title"])
if item.get("sort_order") is not None:
patch["sort_order"] = int(item["sort_order"])
if patch:
await session.execute(
update(TaskDataBlockDefRow)
.where(TaskDataBlockDefRow.id == row.id)
.values(**patch)
)
touched = True
if touched:
await session.commit()
return await self._ordered_defs(session, task_id)
async def open_period(
self,
task_id: str,
*,
source: str,
run_id: str | None = None,
opened_at: datetime | None = None,
) -> dict[str, Any]:
import uuid as _uuid
for _attempt in range(3):
async with self._sf() as session:
if run_id:
existing = await session.execute(
select(TaskDataPeriodRow).where(TaskDataPeriodRow.run_id == run_id)
)
row = existing.scalar_one_or_none()
if row is not None:
return _period_dict(row)
max_index = (
await session.execute(
select(func.max(TaskDataPeriodRow.period_index)).where(
TaskDataPeriodRow.task_id == task_id
)
)
).scalar()
row = TaskDataPeriodRow(
id=f"tdp-{_uuid.uuid4()}",
task_id=task_id,
period_index=(max_index or 0) + 1,
opened_at=opened_at or _utcnow(),
trigger=str(source or "cron"),
run_id=run_id,
)
session.add(row)
try:
await session.commit()
except IntegrityError:
# 并发开期撞了期号唯一键(或 run_id 幂等锚点)——重查/重试。
await session.rollback()
continue
await session.refresh(row)
return _period_dict(row)
raise RuntimeError(f"open_period 连续冲突,task={task_id} source={source}")
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(TaskDataPeriodRow).where(
TaskDataPeriodRow.task_id == task_id,
TaskDataPeriodRow.period_index == period_index,
)
)
row = result.scalar_one_or_none()
return _period_dict(row) if row else None
async def current_period(self, task_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(TaskDataPeriodRow)
.where(TaskDataPeriodRow.task_id == task_id)
.order_by(TaskDataPeriodRow.period_index.desc())
.limit(1)
)
row = result.scalar_one_or_none()
return _period_dict(row) if row else None
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskDataPeriodRow)
.where(TaskDataPeriodRow.task_id == task_id)
.order_by(TaskDataPeriodRow.period_index.desc())
.limit(limit)
)
rows = [_period_dict(row) for row in result.scalars()]
rows.reverse() # 时间轴语义:按期号升序返回
return rows
async def append_submission(
self,
task_id: str,
period_index: int,
block_key: str,
payload: dict[str, Any],
*,
submitted_by: str | None = None,
) -> dict[str, Any]:
import uuid as _uuid
async with self._sf() as session:
base_seq = (
(
await session.execute(
select(func.count(TaskDataSubmissionRow.id)).where(
TaskDataSubmissionRow.task_id == task_id,
TaskDataSubmissionRow.period_index == period_index,
TaskDataSubmissionRow.block_key == block_key,
)
)
)
.scalar_one()
)
row = TaskDataSubmissionRow(
id=f"tds-{_uuid.uuid4()}",
task_id=task_id,
period_index=period_index,
block_key=block_key,
payload=_dump(payload),
submitted_by=submitted_by,
seq=int(base_seq),
)
session.add(row)
await session.commit()
await session.refresh(row)
return _submission_dict(row)
async def list_submissions(
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskDataSubmissionRow)
.where(
TaskDataSubmissionRow.task_id == task_id,
TaskDataSubmissionRow.period_index == period_index,
TaskDataSubmissionRow.block_key == block_key,
)
.order_by(TaskDataSubmissionRow.seq.asc())
.limit(limit)
)
return [_submission_dict(row) for row in result.scalars()]
async def upsert_period_block(
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
) -> dict[str, Any]:
import uuid as _uuid
async with self._sf() as session:
result = await session.execute(
select(TaskDataBlockRow).where(
TaskDataBlockRow.task_id == task_id,
TaskDataBlockRow.period_index == period_index,
TaskDataBlockRow.block_key == block_key,
)
)
row = result.scalar_one_or_none()
if row is None:
row = TaskDataBlockRow(
id=f"tdb-{_uuid.uuid4()}",
task_id=task_id,
period_index=period_index,
block_key=block_key,
payload=_dump(payload),
)
session.add(row)
else:
row.payload = _dump(payload)
row.updated_at = _utcnow()
await session.commit()
await session.refresh(row)
return _task_block_dict(row)
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskDataBlockRow)
.where(TaskDataBlockRow.task_id == task_id)
.order_by(TaskDataBlockRow.period_index.asc(), TaskDataBlockRow.block_key.asc())
.limit(limit)
)
return [_task_block_dict(row) for row in result.scalars()]
@@ -30,6 +30,18 @@ EXCLUDED_SYSTEM_THREAD_TYPES = frozenset(
"deep_research",
"writing_setup",
"report_structure_setup",
"task-analysis",
"task-analysis-qa",
}
)
# 这些系统线程 id 只存在于编排内部,不进「最近的对话」;loopback 派活必须能跨
# 「点击触发的 HTTP 用户」和「绑定 created_by 签发的 JWT」读写,否则会 404 not accessible。
SHARED_ACCESS_THREAD_TYPES = frozenset(
{
"roundtable",
"task-analysis",
"task-analysis-qa",
}
)
@@ -43,6 +55,15 @@ def is_excluded_system_thread(metadata: dict[str, Any] | None) -> bool:
return isinstance(thread_type, str) and thread_type in EXCLUDED_SYSTEM_THREAD_TYPES
def is_shared_access_thread(metadata: dict[str, Any] | None) -> bool:
"""Return True when any authenticated caller may read/write this thread."""
md = metadata or {}
if md.get("taskId"):
return True
thread_type = md.get("thread_type")
return isinstance(thread_type, str) and thread_type in SHARED_ACCESS_THREAD_TYPES
class ThreadMetaStore(abc.ABC):
@abc.abstractmethod
async def create(
@@ -11,7 +11,11 @@ from typing import Any
from langgraph.store.base import BaseStore
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread
from deerflow.persistence.thread_meta.base import (
ThreadMetaStore,
is_excluded_system_thread,
is_shared_access_thread,
)
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
from deerflow.utils.time import coerce_iso, now_iso
@@ -183,8 +187,8 @@ class MemoryThreadMetaStore(ThreadMetaStore):
item = await self._store.aget(THREADS_NS, thread_id)
if item is None:
return not require_existing
# 任务深链对话(metadata.taskId)全局共享、不区分用户:任何登录用户都可读写/续聊。
if (item.value.get("metadata") or {}).get("taskId"):
# 任务深链 / 圆桌 / 任务分析系统线程:全局共享(与 SQL store 同口径)。
if is_shared_access_thread(item.value.get("metadata") or {}):
return True
record_user_id = item.value.get("user_id")
if record_user_id is None:
@@ -8,7 +8,7 @@ from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
@@ -108,10 +108,10 @@ class ThreadMetaRepository(ThreadMetaStore):
md = row.metadata_json or {}
# 全局共享、不区分用户(任何登录用户都可读写/续聊):
# ① 任务深链对话(metadata.taskId);
# ② 圆桌会商系统线程(metadata.thread_type=="roundtable")——其 Step2/Step3
# 沙箱产物(md/流程图/报告)需让历史草稿或其他账号打开同一会商时也能查看,
# 与任务深链对话一致(线程 id 仅存在于草稿中,不会被无关账号发现)。
if md.get("taskId") or md.get("thread_type") == "roundtable":
# ② 圆桌 / 任务分析等系统编排线程(thread_type 见 SHARED_ACCESS_THREAD_TYPES)
# —— loopback 派活签发的 JWT 可能与建 thread 时 HTTP 上下文用户不是同一人,
# 若按属主拦截会把刚建好的总控 thread 打成 404 not accessible。
if is_shared_access_thread(md):
return True
if row.user_id is None:
return True
@@ -0,0 +1,22 @@
"""Global database-backed upload policy persistence."""
from __future__ import annotations
from deerflow.persistence.upload_policy.base import UploadPolicyStore
from deerflow.persistence.upload_policy.memory import MemoryUploadPolicyStore
from deerflow.persistence.upload_policy.model import UploadPolicyRow
from deerflow.persistence.upload_policy.sql import UploadPolicyRepository
__all__ = [
"MemoryUploadPolicyStore",
"UploadPolicyRepository",
"UploadPolicyRow",
"UploadPolicyStore",
"make_upload_policy_store",
]
def make_upload_policy_store(session_factory):
if session_factory is None:
return MemoryUploadPolicyStore()
return UploadPolicyRepository(session_factory)
@@ -0,0 +1,26 @@
"""Persistence contract for the global upload policy document."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class UploadPolicyStore(ABC):
@abstractmethod
async def ensure_seed(self, policy: dict[str, Any]) -> dict[str, Any]:
"""Create the default row if missing without overwriting saved edits."""
raise NotImplementedError
@abstractmethod
async def get_policy(self) -> dict[str, Any] | None:
"""Return the stored policy, or ``None`` before the first save."""
raise NotImplementedError
@abstractmethod
async def set_policy(self, policy: dict[str, Any], *, updated_by: str | None = None) -> dict[str, Any]:
"""Atomically replace the global policy document."""
raise NotImplementedError
@@ -0,0 +1,39 @@
"""In-memory upload-policy store used without a configured SQL database."""
from __future__ import annotations
from copy import deepcopy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.upload_policy.base import UploadPolicyStore
class MemoryUploadPolicyStore(UploadPolicyStore):
def __init__(self) -> None:
self._policy: dict[str, Any] | None = None
self._updated_by: str | None = None
self._updated_at: str | None = None
self._revision = 0
async def ensure_seed(self, policy: dict[str, Any]) -> dict[str, Any]:
if self._policy is None:
return await self.set_policy(policy, updated_by=None)
return (await self.get_policy()) or {}
async def get_policy(self) -> dict[str, Any] | None:
if self._policy is None:
return None
return {
"policy": deepcopy(self._policy),
"updated_by": self._updated_by,
"updated_at": self._updated_at,
"revision": self._revision,
}
async def set_policy(self, policy: dict[str, Any], *, updated_by: str | None = None) -> dict[str, Any]:
self._policy = deepcopy(policy)
self._updated_by = updated_by
self._updated_at = datetime.now(UTC).isoformat()
self._revision += 1
return (await self.get_policy()) or {}
@@ -0,0 +1,25 @@
"""ORM row for the global, database-backed upload policy."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class UploadPolicyRow(Base):
__tablename__ = "upload_policy"
# A named singleton keeps the schema extensible if scoped policies are
# introduced later without requiring a table redesign.
scope: Mapped[str] = mapped_column(String(64), primary_key=True, default="global")
policy_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
revision: Mapped[int] = mapped_column(Integer, default=1, server_default="1")
updated_by: Mapped[str | None] = mapped_column(String(64))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
@@ -0,0 +1,84 @@
"""SQLAlchemy-backed upload-policy store."""
from __future__ import annotations
from copy import deepcopy
from datetime import datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.upload_policy.base import UploadPolicyStore
from deerflow.persistence.upload_policy.model import UploadPolicyRow
_GLOBAL_SCOPE = "global"
def _to_dict(row: UploadPolicyRow) -> dict[str, Any]:
updated_at: datetime | str | None = row.updated_at
if isinstance(updated_at, datetime):
updated_at = updated_at.isoformat()
return {
"policy": deepcopy(row.policy_json or {}),
"updated_by": row.updated_by,
"updated_at": updated_at,
"revision": int(row.revision or 1),
}
class UploadPolicyRepository(UploadPolicyStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def ensure_seed(self, policy: dict[str, Any]) -> dict[str, Any]:
existing = await self.get_policy()
if existing is not None:
return existing
async with self._sf() as session:
row = UploadPolicyRow(
scope=_GLOBAL_SCOPE,
policy_json=deepcopy(policy),
revision=1,
updated_by=None,
)
session.add(row)
try:
await session.commit()
except IntegrityError:
# Another worker seeded the singleton concurrently.
await session.rollback()
winner = await self.get_policy()
if winner is None:
raise
return winner
await session.refresh(row)
return _to_dict(row)
async def get_policy(self) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(UploadPolicyRow, _GLOBAL_SCOPE)
return _to_dict(row) if row is not None else None
async def set_policy(self, policy: dict[str, Any], *, updated_by: str | None = None) -> dict[str, Any]:
async with self._sf() as session:
result = await session.execute(
select(UploadPolicyRow).where(UploadPolicyRow.scope == _GLOBAL_SCOPE).with_for_update()
)
row = result.scalar_one_or_none()
if row is None:
row = UploadPolicyRow(
scope=_GLOBAL_SCOPE,
policy_json=deepcopy(policy),
revision=1,
updated_by=updated_by,
)
session.add(row)
else:
row.policy_json = deepcopy(policy)
row.updated_by = updated_by
row.revision = int(row.revision or 0) + 1
await session.commit()
await session.refresh(row)
return _to_dict(row)