sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -0,0 +1,34 @@
"""Project-scoped cross-session memory persistence."""
from deerflow.persistence.analysis_project_memory.model import (
AnalysisProjectMemoryItemRow,
AnalysisProjectMemoryJobRow,
AnalysisProjectMemoryStateRow,
)
from deerflow.persistence.analysis_project_memory.store import (
ACTIVE_ITEM_STATUS,
ACTIVE_JOB_STATUSES,
ITEM_KINDS,
ITEM_STATUSES,
TERMINAL_JOB_STATUSES,
AnalysisProjectMemoryStore,
)
def make_analysis_project_memory_store(session_factory=None) -> AnalysisProjectMemoryStore:
"""Create the SQL store, or its compatible memory fallback when factory is None."""
return AnalysisProjectMemoryStore(session_factory)
__all__ = [
"ACTIVE_ITEM_STATUS",
"ACTIVE_JOB_STATUSES",
"AnalysisProjectMemoryItemRow",
"AnalysisProjectMemoryJobRow",
"AnalysisProjectMemoryStateRow",
"AnalysisProjectMemoryStore",
"ITEM_KINDS",
"ITEM_STATUSES",
"TERMINAL_JOB_STATUSES",
"make_analysis_project_memory_store",
]
@@ -0,0 +1,179 @@
"""ORM rows for durable, project-scoped cross-session memory.
The SQL rows are deliberately independent from the generic user/agent memory
tables. Their authoritative scope is ``(project_id, owner_user_id)`` and every
child row carries the owner as a defence-in-depth filter for repository reads.
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Boolean, Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class AnalysisProjectMemoryStateRow(Base):
"""One materialized memory snapshot per analysis project."""
__tablename__ = "analysis_project_memory_states"
project_id: Mapped[str] = mapped_column(
ForeignKey("analysis_projects.id", ondelete="CASCADE"), primary_key=True
)
owner_user_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
enabled: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=True, server_default=sa.true()
)
settings_json: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
snapshot_json: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
revision: Mapped[int] = mapped_column(
Integer, nullable=False, default=0, server_default=sa.text("0")
)
item_count: Mapped[int] = mapped_column(
Integer, nullable=False, default=0, server_default=sa.text("0")
)
# ready / building / rebuilding / recovering / error
status: Mapped[str] = mapped_column(
String(24), nullable=False, default="ready", server_default="ready", index=True
)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
rebuild_requested_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
rebuilt_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=_now, onupdate=_now
)
class AnalysisProjectMemoryItemRow(Base):
"""An atomic fact/decision/constraint/episode with auditable provenance."""
__tablename__ = "analysis_project_memory_items"
__table_args__ = (
# ``active_key`` is populated only for the active version of a logical
# item. NULL values do not conflict, so superseded/deleted versions
# remain as an audit trail while one active version is enforced.
UniqueConstraint("active_key", name="uq_analysis_project_memory_item_active_key"),
Index(
"ix_analysis_project_memory_item_scope_status",
"project_id",
"owner_user_id",
"status",
),
Index(
"ix_analysis_project_memory_item_source_thread",
"project_id",
"source_thread_id",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
project_id: Mapped[str] = mapped_column(
ForeignKey("analysis_projects.id", ondelete="CASCADE"), nullable=False, index=True
)
owner_user_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
logical_key: Mapped[str] = mapped_column(String(512), nullable=False)
key_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
active_key: Mapped[str | None] = mapped_column(String(64), nullable=True)
kind: Mapped[str] = mapped_column(String(32), nullable=False, default="fact", index=True)
content: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
# active / superseded / deleted
status: Mapped[str] = mapped_column(
String(16), nullable=False, default="active", server_default="active", index=True
)
confidence: Mapped[float] = mapped_column(Float, nullable=False, default=1.0)
importance: Mapped[float] = mapped_column(Float, nullable=False, default=0.5)
pinned: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=sa.false()
)
source_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
source_message_ids: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
source_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
source_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
extractor_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
superseded_by: Mapped[str | None] = mapped_column(
ForeignKey("analysis_project_memory_items.id", ondelete="SET NULL"), nullable=True
)
revision: Mapped[int] = mapped_column(
Integer, nullable=False, default=1, server_default=sa.text("1")
)
# Phase-two foundation: the encoder remains optional. JSON keeps this
# portable across SQLite/MySQL/Postgres/Gauss; retrieval validates dimensions.
embedding_json: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True)
embedding_model: Mapped[str | None] = mapped_column(String(255), nullable=True)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=_now, onupdate=_now
)
class AnalysisProjectMemoryJobRow(Base):
"""Durable, idempotent extraction/rebuild work with an expiring lease."""
__tablename__ = "analysis_project_memory_jobs"
__table_args__ = (
UniqueConstraint("idempotency_key", name="uq_analysis_project_memory_job_idempotency"),
Index(
"ix_analysis_project_memory_job_claim",
"status",
"available_at",
"lease_until",
),
Index(
"ix_analysis_project_memory_job_scope",
"project_id",
"owner_user_id",
"status",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
project_id: Mapped[str] = mapped_column(
ForeignKey("analysis_projects.id", ondelete="CASCADE"), nullable=False, index=True
)
owner_user_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
source_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
source_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
source_hash: Mapped[str] = mapped_column(String(64), nullable=False)
extractor_version: Mapped[str] = mapped_column(String(64), nullable=False)
idempotency_key: Mapped[str] = mapped_column(String(64), nullable=False)
payload_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
result_json: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
# queued / running / completed / failed / cancelled
status: Mapped[str] = mapped_column(
String(24), nullable=False, default="queued", server_default="queued", index=True
)
attempt: Mapped[int] = mapped_column(
Integer, nullable=False, default=0, server_default=sa.text("0")
)
max_attempts: Mapped[int] = mapped_column(
Integer, nullable=False, default=3, server_default=sa.text("3")
)
available_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=_now, onupdate=_now
)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
__all__ = [
"AnalysisProjectMemoryItemRow",
"AnalysisProjectMemoryJobRow",
"AnalysisProjectMemoryStateRow",
]
@@ -0,0 +1,202 @@
"""User-owned analysis projects and immutable project/thread bindings."""
from __future__ import annotations
from copy import deepcopy
from datetime import UTC, datetime
from uuid import uuid4
from sqlalchemy import ForeignKey, String, Text, delete, select
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
MODES = frozenset({"qa", "strategy", "solution"})
CONFIG_FIELDS = frozenset({"agent_id", "report_config", "strategy_config"})
EDITABLE_FIELDS = frozenset({"name", "background", "knowledge_base_ids", "assistant_knowledge_base_ids"}) | CONFIG_FIELDS
class AnalysisProjectConfigRow(Base):
"""Additive table: upgrades do not alter existing project/thread tables."""
__tablename__ = "analysis_project_configs"
key: Mapped[str] = mapped_column(String(80), primary_key=True)
config: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
class AnalysisProjectRow(Base):
__tablename__ = "analysis_projects"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
user_id: Mapped[str] = mapped_column(String(255), index=True)
name: Mapped[str] = mapped_column(String(160))
mode: Mapped[str] = mapped_column(String(16))
background: Mapped[str] = mapped_column(Text, default="")
knowledge_base_ids: Mapped[list] = mapped_column(PortableJSON(), default=list)
assistant_knowledge_base_ids: Mapped[list] = mapped_column(PortableJSON(), default=list)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime())
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime())
class AnalysisProjectThreadRow(Base):
__tablename__ = "analysis_project_threads"
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
project_id: Mapped[str] = mapped_column(ForeignKey("analysis_projects.id"), index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime())
def _dict(row):
data = row.to_dict()
for key in ("created_at", "updated_at"):
if isinstance(data.get(key), datetime):
data[key] = data[key].isoformat()
return deepcopy(data)
class AnalysisProjectStore:
"""SQL in persistent deployments; isolated in-memory fallback for memory mode.
Every public project operation requires an explicit owner. The unscoped
thread lookup is only for the Gateway's ownership and run-policy checks.
Mode has no update path, and bindings are separate from client-editable metadata.
"""
def __init__(self, session_factory=None):
self._sf = session_factory
self._projects: dict[str, dict] = {}
self._threads: dict[str, dict] = {}
self._configs: dict[str, dict] = {}
async def config_for(self, key: str) -> dict:
if not self._sf:
return deepcopy(self._configs.get(key, {}))
async with self._sf() as session:
row = await session.get(AnalysisProjectConfigRow, key)
return deepcopy(row.config) if row else {}
async def _enrich(self, project: dict | None) -> dict | None:
if project is None:
return None
return {
**project,
"agent_id": None,
"report_config": {},
"strategy_config": {},
**await self.config_for(f"project:{project['id']}"),
}
async def create(self, user_id: str, *, mode: str, **fields) -> dict:
if mode not in MODES or set(fields) - EDITABLE_FIELDS:
raise ValueError("Invalid project fields")
now = datetime.now(UTC)
row = AnalysisProjectRow(
id=str(uuid4()),
user_id=user_id,
mode=mode,
name=fields["name"],
background=fields.get("background", ""),
knowledge_base_ids=list(fields.get("knowledge_base_ids", [])),
assistant_knowledge_base_ids=list(fields.get("assistant_knowledge_base_ids", [])),
created_at=now,
updated_at=now,
)
if self._sf:
async with self._sf() as session:
session.add(row)
session.add(AnalysisProjectConfigRow(key=f"project:{row.id}", config={key: deepcopy(fields[key]) for key in CONFIG_FIELDS if key in fields}))
await session.commit()
else:
self._projects[row.id] = _dict(row)
self._configs[f"project:{row.id}"] = {key: deepcopy(fields[key]) for key in CONFIG_FIELDS if key in fields}
return await self._enrich(_dict(row))
async def list(self, user_id: str) -> list[dict]:
if not self._sf:
return [await self._enrich(deepcopy(p)) for p in sorted(self._projects.values(), key=lambda p: p["updated_at"], reverse=True) if p["user_id"] == user_id]
async with self._sf() as session:
result = await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.user_id == user_id).order_by(AnalysisProjectRow.updated_at.desc()))
rows = [_dict(row) for row in result.scalars()]
return [await self._enrich(row) for row in rows]
async def get(self, project_id: str, user_id: str) -> dict | None:
if not self._sf:
row = self._projects.get(project_id)
return await self._enrich(deepcopy(row)) if row and row["user_id"] == user_id else None
async with self._sf() as session:
row = (await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.id == project_id, AnalysisProjectRow.user_id == user_id))).scalar_one_or_none()
data = _dict(row) if row else None
return await self._enrich(data)
async def update(self, project_id: str, user_id: str, **fields) -> dict | None:
if set(fields) - EDITABLE_FIELDS:
raise ValueError("项目模式创建后不可更改")
if not self._sf:
row = self._projects.get(project_id)
if not row or row["user_id"] != user_id:
return None
row.update({key: deepcopy(value) for key, value in fields.items() if key not in CONFIG_FIELDS}, updated_at=datetime.now(UTC).isoformat())
self._configs.setdefault(f"project:{project_id}", {}).update({key: deepcopy(value) for key, value in fields.items() if key in CONFIG_FIELDS})
return await self._enrich(deepcopy(row))
async with self._sf() as session:
row = (await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.id == project_id, AnalysisProjectRow.user_id == user_id).with_for_update())).scalar_one_or_none()
if row is None:
return None
for key, value in fields.items():
if key not in CONFIG_FIELDS:
setattr(row, key, deepcopy(value))
options = await session.get(AnalysisProjectConfigRow, f"project:{project_id}")
config = {**(options.config if options else {}), **{key: deepcopy(value) for key, value in fields.items() if key in CONFIG_FIELDS}}
await session.merge(AnalysisProjectConfigRow(key=f"project:{project_id}", config=config))
row.updated_at = datetime.now(UTC)
await session.commit()
data = _dict(row)
return await self._enrich(data)
async def bind_thread(self, project_id: str, user_id: str, thread_id: str, *, agent_id: str | None = None) -> None:
if await self.get(project_id, user_id) is None:
raise ValueError("Project not found")
row = AnalysisProjectThreadRow(thread_id=thread_id, project_id=project_id, created_at=datetime.now(UTC))
if not self._sf:
if thread_id in self._threads:
raise ValueError("Thread already bound")
self._threads[thread_id] = _dict(row)
self._configs[f"thread:{thread_id}"] = {"agent_id": agent_id}
return
async with self._sf() as session:
session.add(row)
session.add(AnalysisProjectConfigRow(key=f"thread:{thread_id}", config={"agent_id": agent_id}))
await session.commit()
async def project_for_thread(self, thread_id: str) -> dict | None:
if not self._sf:
binding = self._threads.get(thread_id)
return await self._enrich(deepcopy(self._projects.get(binding["project_id"]))) if binding else None
async with self._sf() as session:
row = (await session.execute(select(AnalysisProjectRow).join(AnalysisProjectThreadRow).where(AnalysisProjectThreadRow.thread_id == thread_id))).scalar_one_or_none()
data = _dict(row) if row else None
return await self._enrich(data)
async def thread_ids(self, project_id: str, user_id: str) -> list[str]:
if await self.get(project_id, user_id) is None:
return []
if not self._sf:
return [key for key, value in reversed(list(self._threads.items())) if value["project_id"] == project_id]
async with self._sf() as session:
return list((await session.execute(select(AnalysisProjectThreadRow.thread_id).where(AnalysisProjectThreadRow.project_id == project_id).order_by(AnalysisProjectThreadRow.created_at.desc()))).scalars())
async def unbind_failed_creation(self, thread_id: str) -> None:
if not self._sf:
self._threads.pop(thread_id, None)
self._configs.pop(f"thread:{thread_id}", None)
return
async with self._sf() as session:
await session.execute(delete(AnalysisProjectThreadRow).where(AnalysisProjectThreadRow.thread_id == thread_id))
await session.execute(delete(AnalysisProjectConfigRow).where(AnalysisProjectConfigRow.key == f"thread:{thread_id}"))
await session.commit()
async def unbind_thread(self, thread_id: str) -> None:
"""Remove a durable project binding when its ordinary thread is deleted."""
await self.unbind_failed_creation(thread_id)
@@ -0,0 +1,26 @@
"""Persisted read index for the generated-file library."""
from __future__ import annotations
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
from deerflow.persistence.artifact_library.memory import MemoryArtifactLibraryStore
from deerflow.persistence.artifact_library.model import (
ArtifactLibraryItemRow,
ArtifactLibrarySyncStateRow,
)
from deerflow.persistence.artifact_library.sql import SqlArtifactLibraryStore
__all__ = [
"ArtifactLibraryItemRow",
"ArtifactLibraryStore",
"ArtifactLibrarySyncStateRow",
"MemoryArtifactLibraryStore",
"SqlArtifactLibraryStore",
"make_artifact_library_store",
]
def make_artifact_library_store(session_factory):
if session_factory is None:
return MemoryArtifactLibraryStore()
return SqlArtifactLibraryStore(session_factory)
@@ -0,0 +1,61 @@
"""Storage contract for the generated-file library index."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class ArtifactLibraryStore(ABC):
"""A user-scoped, replace-on-sync artifact metadata index."""
@abstractmethod
async def claim_sync(
self,
user_id: str,
*,
force: bool = False,
stale_after_seconds: float = 15.0,
lease_seconds: float = 300.0,
) -> str | None:
"""Claim one background refresh and return its generation token.
``None`` means another worker owns a live lease or the persisted index
is still fresh enough. Implementations must make this operation safe
when multiple Gateway workers receive the same request concurrently.
"""
raise NotImplementedError
@abstractmethod
async def finish_sync(
self,
user_id: str,
generation: str,
items: list[dict[str, Any]],
) -> bool:
"""Atomically publish ``items`` when ``generation`` still owns the lease."""
raise NotImplementedError
@abstractmethod
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
"""Record a failed refresh when ``generation`` still owns the lease."""
raise NotImplementedError
@abstractmethod
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
"""Return the persisted refresh state for one user."""
raise NotImplementedError
@abstractmethod
async def list_items(
self,
user_id: str,
*,
query: str | None = None,
file_kind: str = "all",
source_type: str = "all",
offset: int = 0,
limit: int = 50,
) -> tuple[int, list[dict[str, Any]]]:
"""Return an indexed, newest-first page and its filtered total."""
raise NotImplementedError
@@ -0,0 +1,158 @@
"""In-memory artifact-library index used by memory-mode deployments and tests."""
from __future__ import annotations
import asyncio
import hashlib
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _artifact_id(user_id: str, item: dict[str, Any]) -> str:
identity = "\0".join((user_id, str(item.get("thread_id") or ""), str(item.get("path") or "")))
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
class MemoryArtifactLibraryStore(ArtifactLibraryStore):
def __init__(self) -> None:
self._items: dict[str, dict[str, Any]] = {}
self._states: dict[str, dict[str, Any]] = {}
self._lock = asyncio.Lock()
async def claim_sync(
self,
user_id: str,
*,
force: bool = False,
stale_after_seconds: float = 15.0,
lease_seconds: float = 300.0,
) -> str | None:
now = datetime.now(UTC)
async with self._lock:
state = self._states.get(user_id)
if state is not None:
lease = state.get("lease_expires_at")
if state.get("status") == "syncing" and isinstance(lease, datetime) and lease > now:
return None
freshness_anchor = state.get("completed_at")
if state.get("status") == "error":
freshness_anchor = state.get("started_at")
if not force and isinstance(freshness_anchor, datetime) and freshness_anchor + timedelta(seconds=stale_after_seconds) > now:
return None
generation = str(uuid4())
self._states[user_id] = {
"status": "syncing",
"generation": generation,
"started_at": now,
"lease_expires_at": now + timedelta(seconds=lease_seconds),
"completed_at": state.get("completed_at") if state else None,
"last_error": None,
}
return generation
async def finish_sync(
self,
user_id: str,
generation: str,
items: list[dict[str, Any]],
) -> bool:
now = datetime.now(UTC)
async with self._lock:
state = self._states.get(user_id)
if state is None or state.get("generation") != generation or state.get("status") != "syncing":
return False
self._items = {key: value for key, value in self._items.items() if value.get("user_id") != user_id}
for item in items:
row = dict(item)
row["artifact_id"] = _artifact_id(user_id, item)
row["user_id"] = user_id
row["sync_generation"] = generation
row["indexed_at"] = now.isoformat()
self._items[row["artifact_id"]] = row
state.update(
status="ready",
completed_at=now,
lease_expires_at=None,
last_error=None,
)
return True
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
async with self._lock:
state = self._states.get(user_id)
if state is None or state.get("generation") != generation or state.get("status") != "syncing":
return False
state.update(
status="error",
lease_expires_at=None,
last_error=(error or "Artifact library refresh failed")[:1024],
)
return True
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
async with self._lock:
state = self._states.get(user_id)
if state is None:
return {
"status": "idle",
"generation": None,
"started_at": None,
"completed_at": None,
"last_error": None,
}
return {
"status": str(state.get("status") or "idle"),
"generation": state.get("generation"),
"started_at": _iso(state.get("started_at")),
"completed_at": _iso(state.get("completed_at")),
"last_error": state.get("last_error"),
}
async def list_items(
self,
user_id: str,
*,
query: str | None = None,
file_kind: str = "all",
source_type: str = "all",
offset: int = 0,
limit: int = 50,
) -> tuple[int, list[dict[str, Any]]]:
normalized = (query or "").strip().casefold()
async with self._lock:
rows = [dict(row) for row in self._items.values() if row.get("user_id") == user_id]
def matches(row: dict[str, Any]) -> bool:
if file_kind != "all" and row.get("file_kind") != file_kind:
return False
if source_type != "all" and row.get("source_type") != source_type:
return False
if not normalized:
return True
return any(
normalized in str(row.get(field) or "").casefold()
for field in (
"name",
"thread_title",
"agent_name",
"strategy_name",
"strategy_project_id",
)
)
filtered = [row for row in rows if matches(row)]
filtered.sort(
key=lambda row: (
str(row.get("modified_at") or ""),
str(row.get("artifact_id") or ""),
),
reverse=True,
)
return len(filtered), filtered[offset : offset + limit]
@@ -0,0 +1,78 @@
"""ORM rows for the generated-file library read index."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ArtifactLibraryItemRow(Base):
"""One generated file projected into the fast library listing."""
__tablename__ = "artifact_library_items"
__table_args__ = (
Index(
"ix_artifact_library_user_modified",
"user_id",
"modified_at",
"artifact_id",
),
Index(
"ix_artifact_library_user_kind",
"user_id",
"file_kind",
"modified_at",
"artifact_id",
),
Index(
"ix_artifact_library_user_source",
"user_id",
"source_type",
"modified_at",
"artifact_id",
),
Index("ix_artifact_library_user_generation", "user_id", "sync_generation"),
)
# A deterministic SHA-256 of (user_id, thread_id, virtual path) avoids a
# long composite primary key that would exceed older MySQL index limits.
artifact_id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False)
thread_id: Mapped[str] = mapped_column(String(128), nullable=False)
thread_title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
agent_name: Mapped[str | None] = mapped_column(String(512), nullable=True)
thread_type: Mapped[str | None] = mapped_column(String(64), nullable=True)
source_type: Mapped[str] = mapped_column(String(32), nullable=False)
strategy_project_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
strategy_name: Mapped[str | None] = mapped_column(String(512), nullable=True)
strategy_stale: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
strategy_source_kind: Mapped[str | None] = mapped_column(String(32), nullable=True)
download_url: Mapped[str | None] = mapped_column(Text, nullable=True)
path: Mapped[str] = mapped_column(Text, nullable=False)
name: Mapped[str] = mapped_column(String(512), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
mime_type: Mapped[str | None] = mapped_column(String(255), nullable=True)
modified_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False)
file_kind: Mapped[str] = mapped_column(String(32), nullable=False)
sync_generation: Mapped[str] = mapped_column(String(36), nullable=False)
indexed_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class ArtifactLibrarySyncStateRow(Base):
"""Per-user lease and last-published generation metadata."""
__tablename__ = "artifact_library_sync_states"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
status: Mapped[str] = mapped_column(String(16), nullable=False, default="idle")
generation: Mapped[str | None] = mapped_column(String(36), nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error: Mapped[str | None] = mapped_column(String(1024), nullable=True)
@@ -0,0 +1,346 @@
"""SQLAlchemy-backed generated-file library index."""
from __future__ import annotations
import hashlib
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import and_, delete, func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
from deerflow.persistence.artifact_library.model import (
ArtifactLibraryItemRow,
ArtifactLibrarySyncStateRow,
)
_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
def _artifact_id(user_id: str, item: dict[str, Any]) -> str:
identity = "\0".join((user_id, str(item.get("thread_id") or ""), str(item.get("path") or "")))
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
def _parse_datetime(value: object) -> datetime:
if isinstance(value, datetime):
parsed = value
else:
text = str(value or "").strip()
if not text:
return _EPOCH
try:
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError:
return _EPOCH
if parsed.tzinfo is None:
return parsed.replace(tzinfo=UTC)
return parsed.astimezone(UTC)
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _state_dict(row: ArtifactLibrarySyncStateRow | None) -> dict[str, Any]:
if row is None:
return {
"status": "idle",
"generation": None,
"started_at": None,
"completed_at": None,
"last_error": None,
}
return {
"status": row.status,
"generation": row.generation,
"started_at": _iso(row.started_at),
"completed_at": _iso(row.completed_at),
"last_error": row.last_error,
}
def _item_dict(row: ArtifactLibraryItemRow) -> dict[str, Any]:
return {
"thread_id": row.thread_id,
"thread_title": row.thread_title,
"agent_id": row.agent_id,
"agent_name": row.agent_name,
"thread_type": row.thread_type,
"source_type": row.source_type,
"strategy_project_id": row.strategy_project_id,
"strategy_name": row.strategy_name,
"strategy_stale": row.strategy_stale,
"strategy_source_kind": row.strategy_source_kind,
"download_url": row.download_url,
"path": row.path,
"name": row.name,
"size_bytes": row.size_bytes,
"mime_type": row.mime_type,
"modified_at": row.modified_at.isoformat(),
"file_kind": row.file_kind,
}
def _apply_item(
row: ArtifactLibraryItemRow,
*,
user_id: str,
generation: str,
item: dict[str, Any],
indexed_at: datetime,
) -> None:
row.user_id = user_id
row.thread_id = str(item.get("thread_id") or "")[:128]
row.thread_title = str(item.get("thread_title") or "")[:512]
row.agent_id = str(item.get("agent_id"))[:128] if item.get("agent_id") else None
row.agent_name = str(item.get("agent_name"))[:512] if item.get("agent_name") else None
row.thread_type = str(item.get("thread_type"))[:64] if item.get("thread_type") else None
row.source_type = str(item.get("source_type") or "general_qa")[:32]
row.strategy_project_id = str(item.get("strategy_project_id"))[:128] if item.get("strategy_project_id") else None
row.strategy_name = str(item.get("strategy_name"))[:512] if item.get("strategy_name") else None
row.strategy_stale = item.get("strategy_stale") is True
row.strategy_source_kind = str(item.get("strategy_source_kind"))[:32] if item.get("strategy_source_kind") else None
row.download_url = str(item.get("download_url")) if item.get("download_url") else None
row.path = str(item.get("path") or "")
row.name = str(item.get("name") or "")[:512]
row.size_bytes = max(0, int(item.get("size_bytes") or 0))
row.mime_type = str(item.get("mime_type"))[:255] if item.get("mime_type") else None
row.modified_at = _parse_datetime(item.get("modified_at"))
row.file_kind = str(item.get("file_kind") or "other")[:32]
row.sync_generation = generation
row.indexed_at = indexed_at
class SqlArtifactLibraryStore(ArtifactLibraryStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def claim_sync(
self,
user_id: str,
*,
force: bool = False,
stale_after_seconds: float = 15.0,
lease_seconds: float = 300.0,
) -> str | None:
now = datetime.now(UTC)
generation = str(uuid4())
try:
async with self._sf() as session:
async with session.begin():
claim_filters = [
ArtifactLibrarySyncStateRow.user_id == user_id,
# An expired (or legacy missing) lease may be reclaimed.
# A live lease remains exclusive even for force=True.
or_(
ArtifactLibrarySyncStateRow.status != "syncing",
ArtifactLibrarySyncStateRow.lease_expires_at.is_(None),
ArtifactLibrarySyncStateRow.lease_expires_at <= now,
),
]
if not force:
stale_before = now - timedelta(seconds=stale_after_seconds)
claim_filters.append(
or_(
and_(
ArtifactLibrarySyncStateRow.status == "error",
or_(
ArtifactLibrarySyncStateRow.started_at.is_(None),
ArtifactLibrarySyncStateRow.started_at <= stale_before,
),
),
and_(
ArtifactLibrarySyncStateRow.status != "error",
or_(
ArtifactLibrarySyncStateRow.completed_at.is_(None),
ArtifactLibrarySyncStateRow.completed_at <= stale_before,
),
),
)
)
# A conditional UPDATE is the portable compare-and-swap.
# SQLite ignores SELECT ... FOR UPDATE, whereas this write
# serializes claimers there and also works on MySQL,
# PostgreSQL, and openGauss.
result = await session.execute(
update(ArtifactLibrarySyncStateRow)
.where(*claim_filters)
.values(
status="syncing",
generation=generation,
started_at=now,
lease_expires_at=now + timedelta(seconds=lease_seconds),
last_error=None,
)
)
if result.rowcount == 1:
return generation
row = await session.get(ArtifactLibrarySyncStateRow, user_id)
if row is not None:
return None
session.add(
ArtifactLibrarySyncStateRow(
user_id=user_id,
status="syncing",
generation=generation,
started_at=now,
lease_expires_at=now + timedelta(seconds=lease_seconds),
last_error=None,
)
)
return generation
except IntegrityError:
# Two workers can race while inserting a user's first state row.
# The winner owns the lease; the loser simply serves the same
# persisted index without starting a duplicate filesystem scan.
async with self._sf() as session:
if await session.get(ArtifactLibrarySyncStateRow, user_id) is not None:
return None
raise
async def finish_sync(
self,
user_id: str,
generation: str,
items: list[dict[str, Any]],
) -> bool:
now = datetime.now(UTC)
deduped = {_artifact_id(user_id, item): item for item in items}
async with self._sf() as session:
async with session.begin():
# Write first so SQLite also serializes this publisher against
# a lease takeover. Requiring status=syncing makes completion
# one-shot even if the same callback is accidentally invoked
# twice with an old generation token.
fence = await session.execute(
update(ArtifactLibrarySyncStateRow)
.where(
ArtifactLibrarySyncStateRow.user_id == user_id,
ArtifactLibrarySyncStateRow.generation == generation,
ArtifactLibrarySyncStateRow.status == "syncing",
)
.values(lease_expires_at=now + timedelta(seconds=300))
)
if fence.rowcount != 1:
return False
existing_result = await session.execute(select(ArtifactLibraryItemRow).where(ArtifactLibraryItemRow.user_id == user_id))
existing = {row.artifact_id: row for row in existing_result.scalars()}
for artifact_id, item in deduped.items():
row = existing.get(artifact_id)
if row is None:
row = ArtifactLibraryItemRow(
artifact_id=artifact_id,
user_id=user_id,
thread_id="",
thread_title="",
source_type="general_qa",
path="",
name="",
size_bytes=0,
modified_at=_EPOCH,
file_kind="other",
sync_generation=generation,
indexed_at=now,
)
session.add(row)
_apply_item(
row,
user_id=user_id,
generation=generation,
item=item,
indexed_at=now,
)
await session.flush()
await session.execute(
delete(ArtifactLibraryItemRow).where(
ArtifactLibraryItemRow.user_id == user_id,
ArtifactLibraryItemRow.sync_generation != generation,
)
)
await session.execute(
update(ArtifactLibrarySyncStateRow)
.where(
ArtifactLibrarySyncStateRow.user_id == user_id,
ArtifactLibrarySyncStateRow.generation == generation,
ArtifactLibrarySyncStateRow.status == "syncing",
)
.values(
status="ready",
completed_at=now,
lease_expires_at=None,
last_error=None,
)
)
return True
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
async with self._sf() as session:
async with session.begin():
result = await session.execute(
update(ArtifactLibrarySyncStateRow)
.where(
ArtifactLibrarySyncStateRow.user_id == user_id,
ArtifactLibrarySyncStateRow.generation == generation,
ArtifactLibrarySyncStateRow.status == "syncing",
)
.values(
status="error",
lease_expires_at=None,
last_error=(error or "Artifact library refresh failed")[:1024],
)
)
return result.rowcount == 1
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(ArtifactLibrarySyncStateRow, user_id)
return _state_dict(row)
async def list_items(
self,
user_id: str,
*,
query: str | None = None,
file_kind: str = "all",
source_type: str = "all",
offset: int = 0,
limit: int = 50,
) -> tuple[int, list[dict[str, Any]]]:
filters = [ArtifactLibraryItemRow.user_id == user_id]
if file_kind != "all":
filters.append(ArtifactLibraryItemRow.file_kind == file_kind)
if source_type != "all":
filters.append(ArtifactLibraryItemRow.source_type == source_type)
normalized = (query or "").strip().casefold()
if normalized:
searchable = (
ArtifactLibraryItemRow.name,
ArtifactLibraryItemRow.thread_title,
ArtifactLibraryItemRow.agent_name,
ArtifactLibraryItemRow.strategy_name,
ArtifactLibraryItemRow.strategy_project_id,
)
filters.append(or_(*(func.lower(func.coalesce(column, "")).contains(normalized, autoescape=True) for column in searchable)))
count_stmt = select(func.count()).select_from(ArtifactLibraryItemRow).where(*filters)
page_stmt = (
select(ArtifactLibraryItemRow)
.where(*filters)
.order_by(
ArtifactLibraryItemRow.modified_at.desc(),
ArtifactLibraryItemRow.artifact_id.desc(),
)
.offset(offset)
.limit(limit)
)
async with self._sf() as session:
total = int((await session.execute(count_stmt)).scalar_one())
rows = (await session.execute(page_stmt)).scalars().all()
return total, [_item_dict(row) for row in rows]
@@ -1,4 +1,21 @@
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
_assistant_knowledge_store = None
def register_assistant_knowledge_store(store) -> None:
global _assistant_knowledge_store
_assistant_knowledge_store = store
def get_assistant_knowledge_store():
return _assistant_knowledge_store
__all__ = [
name
for name in globals()
if name.startswith("AssistantKnowledge")
or name in {"make_assistant_knowledge_store", "register_assistant_knowledge_store", "get_assistant_knowledge_store"}
]
@@ -1,7 +1,7 @@
"""ORM model for the global business → agent / chain mapping (业务链条管理).
A "business mapping" row binds a fixed business code (3Q / 6BF / 6-1…6-4 / 7BF /
8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
A "business mapping" row binds a fixed business code (3Q / 6BF / 3q-0…3q-6 /
6-1…6-4 / 7BF / 7-1…7-4 / 8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
account can resolve it). Used to auto-pick the business chain when a task page
launches the multi-agent roundtable.
@@ -21,7 +21,7 @@ _UNSET = object()
# Fixed business rows (业务码 → 展示名). Seeded on first boot; agent/chain left empty.
# 8BF 页面尚未建好,先占位保留行。顺序即管理表的展示顺序。
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
# 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
# 3q-0…3q-6 / 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
{"business_code": "3Q", "label": "3QFX"},
{"business_code": "6BF", "label": "任务FX"},
@@ -29,6 +29,13 @@ DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
{"business_code": "8BF", "label": "8BF"},
{"business_code": "AFX", "label": "AGENTFX"},
# 独立配置位,与上方业务码无关;固定排在表尾。
{"business_code": "3q-0", "label": "3q-0"},
{"business_code": "3q-1", "label": "3q-1"},
{"business_code": "3q-2", "label": "3q-2"},
{"business_code": "3q-3", "label": "3q-3"},
{"business_code": "3q-4", "label": "3q-4"},
{"business_code": "3q-5", "label": "3q-5"},
{"business_code": "3q-6", "label": "3q-6"},
{"business_code": "6-1", "label": "6-1"},
{"business_code": "6-2", "label": "6-2"},
{"business_code": "6-3", "label": "6-3"},
@@ -9,13 +9,17 @@ from deerflow.persistence.compact_menu_overrides.keys import (
normalize_compact_menu_type,
)
from deerflow.persistence.compact_menu_overrides.memory import MemoryCompactMenuOverrideStore
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.compact_menu_overrides.model import (
CompactMenuOverrideRow,
UserCompactMenuOverrideConfigRow,
)
from deerflow.persistence.compact_menu_overrides.sql import CompactMenuOverrideRepository
__all__ = [
"ALLOWED_COMPACT_MENU_TYPES",
"CompactMenuOverrideRepository",
"CompactMenuOverrideRow",
"UserCompactMenuOverrideConfigRow",
"CompactMenuOverrideStore",
"DEFAULT_COMPACT_MENU_TYPE",
"MemoryCompactMenuOverrideStore",
@@ -1,4 +1,10 @@
"""Abstract compact-sidebar menu override persistence API."""
"""Abstract compact-sidebar menu override persistence API.
The regular rows are the administrator's default for a compact-menu variant.
Personal layouts live in a separate complete snapshot per user and variant;
the router overlays that snapshot while allowing newly-added default node ids
to flow through.
"""
from __future__ import annotations
@@ -22,3 +28,20 @@ class CompactMenuOverrideStore(ABC):
) -> list[dict[str, Any]]:
"""Atomically replace the compact-menu override set for one type."""
raise NotImplementedError
@abstractmethod
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
"""Return one user's snapshot for a type, or ``None`` if absent."""
raise NotImplementedError
@abstractmethod
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
menu_type: str | None = None,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
"""Replace one user's snapshot; an empty list resets to defaults."""
raise NotImplementedError
@@ -38,6 +38,7 @@ def _public(row: dict[str, Any]) -> dict[str, Any]:
class MemoryCompactMenuOverrideStore(CompactMenuOverrideStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._user_rows: dict[tuple[str, str], list[dict[str, Any]]] = {}
async def list_overrides(self, menu_type: str | None = None) -> list[dict[str, Any]]:
t = normalize_compact_menu_type(menu_type)
@@ -62,3 +63,36 @@ class MemoryCompactMenuOverrideStore(CompactMenuOverrideStore):
stored = _normalize(r, t, updated_by)
self._rows[str(stored["node_id"])] = stored
return await self.list_overrides(t)
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
t = normalize_compact_menu_type(menu_type)
rows = self._user_rows.get((str(user_id), t))
return [dict(r) for r in rows] if rows is not None else None
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
menu_type: str | None = None,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
t = normalize_compact_menu_type(menu_type)
key = (str(user_id), t)
if not rows:
self._user_rows.pop(key, None)
return None
by_id: dict[str, dict[str, Any]] = {}
for row in rows:
node_id = row.get("node_id")
if not node_id:
continue
stored = _normalize(row, t, updated_by or str(user_id))
# Expose the public id in a personal document just like the API.
public = _public(stored)
by_id[str(public["node_id"])] = public
if not by_id:
self._user_rows.pop(key, None)
return None
self._user_rows[key] = [dict(r) for r in by_id.values()]
return [dict(r) for r in self._user_rows[key]]
@@ -12,7 +12,7 @@ from sqlalchemy import Index, Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class CompactMenuOverrideRow(Base):
@@ -28,3 +28,17 @@ class CompactMenuOverrideRow(Base):
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
class UserCompactMenuOverrideConfigRow(Base):
"""A user's complete snapshot for one compact-menu variant."""
__tablename__ = "user_compact_menu_override_configs"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
menu_type: Mapped[str] = mapped_column(String(32), primary_key=True)
overrides: Mapped[list[dict]] = mapped_column(PortableJSON(), nullable=False, default=list)
updated_by: Mapped[str | None] = mapped_column(String(64))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
@@ -2,7 +2,7 @@
from __future__ import annotations
from datetime import datetime
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, or_, select
@@ -15,7 +15,10 @@ from deerflow.persistence.compact_menu_overrides.keys import (
public_node_id,
storage_node_id,
)
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.compact_menu_overrides.model import (
CompactMenuOverrideRow,
UserCompactMenuOverrideConfigRow,
)
def _to_dict(row: CompactMenuOverrideRow) -> dict[str, Any]:
@@ -42,6 +45,41 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C
)
def _normalize_user_row(data: dict[str, Any], menu_type: str) -> dict[str, Any]:
placement = str(data.get("placement") or "sidebar")
return {
"node_id": public_node_id(str(data["node_id"]), menu_type),
"menu_type": menu_type,
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
"sort_order": int(data.get("sort_order", 0)),
"label": (data.get("label") or None),
}
def _user_rows(row: UserCompactMenuOverrideConfigRow) -> list[dict[str, Any]]:
value = row.overrides
if isinstance(value, str):
import json
try:
value = json.loads(value)
except (TypeError, ValueError):
value = []
if not isinstance(value, list):
value = []
menu_type = normalize_compact_menu_type(row.menu_type)
updated_at = row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at
return [
{
**_normalize_user_row(item, menu_type),
"updated_by": row.updated_by,
"updated_at": updated_at,
}
for item in value
if isinstance(item, dict) and item.get("node_id")
]
class CompactMenuOverrideRepository(CompactMenuOverrideStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -76,6 +114,54 @@ class CompactMenuOverrideRepository(CompactMenuOverrideStore):
await session.refresh(row)
return [_to_dict(row) for row in new_rows]
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
t = normalize_compact_menu_type(menu_type)
async with self._sf() as session:
row = await session.get(UserCompactMenuOverrideConfigRow, (str(user_id), t))
return _user_rows(row) if row is not None else None
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
menu_type: str | None = None,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
t = normalize_compact_menu_type(menu_type)
key = str(user_id)
by_id: dict[str, dict[str, Any]] = {}
for data in rows:
node_id = data.get("node_id")
if node_id:
normalized = _normalize_user_row(data, t)
by_id[normalized["node_id"]] = normalized
async with self._sf() as session:
row = await session.get(UserCompactMenuOverrideConfigRow, (key, t))
if not by_id:
if row is not None:
await session.delete(row)
await session.commit()
return None
now = datetime.now(UTC)
payload = list(by_id.values())
if row is None:
row = UserCompactMenuOverrideConfigRow(
user_id=key,
menu_type=t,
overrides=payload,
updated_by=updated_by or key,
)
session.add(row)
else:
row.overrides = payload
row.updated_by = updated_by or key
row.updated_at = now
await session.commit()
# ``expire_on_commit=False`` keeps this writer-side row usable and
# avoids a replica-lag-prone post-commit refresh.
return _user_rows(row)
def _menu_type_filter(menu_type: str):
if menu_type == DEFAULT_COMPACT_MENU_TYPE:
@@ -215,7 +215,7 @@ async def _auto_create_mysql_db(url: str) -> None:
await maint_engine.dispose()
def _ensure_orm_columns_sync(sync_conn) -> None:
def _ensure_orm_columns_sync(sync_conn) -> None:
"""Add ORM-declared columns that are missing from already-existing tables.
``Base.metadata.create_all`` creates brand-new tables but never patches
@@ -226,8 +226,8 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
This sweep compares each existing table against the ORM metadata and issues
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
safely (nullable, or carrying a server_default). Indexes covering a newly
added column are created as well. Table-level ``UniqueConstraint`` objects
safely (nullable, or carrying a server_default). Missing ORM indexes are
created as well. Table-level ``UniqueConstraint`` objects
that are declared on the model but missing from the table are also created
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
@@ -261,8 +261,7 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
if not missing:
continue
added: set[str] = set()
for col in missing:
for col in missing:
if not col.nullable and col.server_default is None:
logger.warning(
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
@@ -271,24 +270,21 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
)
continue
try:
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
added.add(col.name)
logger.info("Auto-added missing column %s.%s", table.name, col.name)
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
logger.info("Auto-added missing column %s.%s", table.name, col.name)
except Exception:
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
if not added:
continue
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
continue
try:
idx.create(sync_conn)
try:
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
except Exception:
existing_idx = set()
for idx in table.indexes:
if idx.name in existing_idx:
continue
try:
idx.create(sync_conn)
logger.info("Auto-created index %s on %s", idx.name, table.name)
except Exception:
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
@@ -332,15 +328,111 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
f"ON {preparer.quote(table.name)} ({cols})"
)
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
except Exception:
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
Alembic auto-upgrade is intentionally disabled in deployed Gateway
processes, while the ORM column synchronizer is always run. The latter
can add the task lookup columns and indexes but cannot derive values from
JSON metadata. Backfill task rows once at startup so an upgrade never
temporarily hides every historical task conversation from ``/threads/search``.
"""
from sqlalchemy import inspect as sa_inspect
table_name = "threads_meta"
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
try:
inspector = sa_inspect(sync_conn)
if table_name not in inspector.get_table_names():
return
columns = {column["name"] for column in inspector.get_columns(table_name)}
if not required_columns <= columns:
return
dialect = sync_conn.dialect.name
if dialect == "sqlite":
task_id = "json_extract(metadata_json, '$.taskId')"
agent_id = "json_extract(metadata_json, '$.agent_id')"
action_id = "json_extract(metadata_json, '$.actionId')"
valid_json = "json_valid(metadata_json)"
elif dialect == "mysql":
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
valid_json = "JSON_VALID(metadata_json)"
elif dialect in {"postgresql", "opengauss"}:
task_id = "metadata_json ->> 'taskId'"
agent_id = "metadata_json ->> 'agent_id'"
action_id = "metadata_json ->> 'actionId'"
valid_json = "TRUE"
else:
return
result = sync_conn.execute(
text(
"UPDATE threads_meta "
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
)
)
if result.rowcount:
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
except Exception:
# A malformed legacy JSON row or a backend-specific JSON function must
# not prevent startup. The generic metadata search remains available,
# and the failure is visible to the operator for remediation.
logger.exception("Failed to backfill thread task lookup projections")
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
message = str(exc).lower()
return "unknown database" in message or "1049" in message
def _acquire_gauss_mysql_schema_bootstrap_lock():
"""Serialize Dolphin schema bootstrap across workers on this host."""
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
if not is_gauss_mysql_product():
return None
try:
import fcntl
except ImportError:
# Production offline deployments are Linux. Keeping a no-lock fallback
# makes imports and unit tests work on Windows without changing MySQL.
logger.warning("fcntl is unavailable; openGauss schema bootstrap is not cross-process locked")
return None
try:
from deerflow.config.runtime_paths import runtime_home
home = runtime_home()
except Exception:
home = Path(os.getenv("DEER_FLOW_HOME", ".deer-flow")).resolve()
lock_path = home / "data" / "gauss-schema-bootstrap.lock"
lock_path.parent.mkdir(parents=True, exist_ok=True)
handle = lock_path.open("a+", encoding="utf-8")
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
logger.info("Acquired openGauss schema bootstrap lock: %s", lock_path)
return handle
def _release_gauss_mysql_schema_bootstrap_lock(handle) -> None: # noqa: ANN001
if handle is None:
return
import fcntl
try:
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
finally:
handle.close()
async def init_engine(
backend: str,
*,
@@ -371,6 +463,10 @@ async def init_engine(
logger.info("Persistence backend=memory -- ORM engine not initialized")
return
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
gauss_mysql = backend == "mysql" and is_gauss_mysql_product()
# Remember the loop that owns the engine's connection pool, so off-loop
# best-effort writes (metrics from worker threads, etc.) can be routed here
# instead of reusing pooled connections cross-loop.
@@ -394,6 +490,10 @@ async def init_engine(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "gauss":
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
install_gauss_dialect_compatibility()
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
@@ -403,6 +503,10 @@ async def init_engine(
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
)
if gauss_mysql:
from deerflow.persistence.gauss_mysql_compat import install_gauss_mysql_dialect_compatibility
install_gauss_mysql_dialect_compatibility(driver)
if backend == "sqlite":
import os
@@ -474,48 +578,98 @@ async def init_engine(
# This is expected during initial scaffolding or minimal installs.
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
async def _create_all_with_product_integrity(conn) -> None: # noqa: ANN001
await conn.run_sync(Base.metadata.create_all)
if gauss_mysql:
from deerflow.persistence.gauss_schema import ensure_gauss_schema_sync
await conn.run_sync(ensure_gauss_schema_sync, Base.metadata)
# Schema bootstrap is deliberately best-effort. A single incompatible or
# permission-restricted table must not make the whole Gateway unavailable:
# routes backed by healthy existing tables can still work, while a route
# whose table is missing will report its own request-time error. Keep the
# auto-create-database retries, but contain failures from both the initial
# create_all and the retry path.
schema_bootstrap_lock = None
if gauss_mysql:
try:
schema_bootstrap_lock = _acquire_gauss_mysql_schema_bootstrap_lock()
except Exception:
# A lock-file permission problem is operationally important, but
# the global startup invariant still requires the Gateway to come
# up and report schema errors at the affected routes.
logger.exception("Could not acquire openGauss schema bootstrap lock; continuing without the host lock")
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database
await _engine.dispose()
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, pool_recycle=pool_recycle, json_serializer=_json_serializer)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
try:
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
else:
raise
await _create_all_with_product_integrity(conn)
except Exception as exc:
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database.
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await _create_all_with_product_integrity(conn)
elif backend == "mysql" and not gauss_mysql and _is_mysql_unknown_database_error(exc):
# Database not yet created. Create it, rebuild the app engine
# against the target DB, then let SQLAlchemy create all tables.
await _auto_create_mysql_db(url)
await _engine.dispose()
_engine = create_async_engine(
url,
echo=echo,
pool_size=pool_size,
max_overflow=max_overflow,
pool_timeout=pool_timeout,
pool_pre_ping=True,
pool_recycle=pool_recycle,
json_serializer=_json_serializer,
)
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
async with _engine.begin() as conn:
await _create_all_with_product_integrity(conn)
else:
raise
except Exception:
logger.exception(
"Automatic ORM schema creation failed; continuing Gateway startup "
"with the existing database schema"
)
finally:
try:
_release_gauss_mysql_schema_bootstrap_lock(schema_bootstrap_lock)
except Exception:
logger.exception("Could not release openGauss schema bootstrap lock")
# Patch existing tables with ORM-declared columns that ``create_all``
# cannot add (it only creates brand-new tables). Best-effort: a failure is
# logged and startup continues on the current schema.
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
except Exception:
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
if gauss_mysql:
# Dolphin returns PostgreSQL-style DDL from SHOW CREATE TABLE. The
# stock MySQL inspector misreads it and may issue duplicate ALTERs,
# aborting the transaction with error 322. The native-catalog helper
# above is the only schema repair path for this product.
logger.info("Skipping MySQL reflection and JSON backfill for openGauss Dolphin")
else:
try:
async with _engine.begin() as conn:
await conn.run_sync(_ensure_orm_columns_sync)
await conn.run_sync(_backfill_thread_meta_task_projections_sync)
except Exception:
logger.exception("Auto-updating ORM columns or task lookup projections failed; continuing with current schema")
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
# cannot do on its own. Failures don't abort startup — see the helper.
@@ -1,38 +1,52 @@
"""ORM model for user feedback on runs."""
"""ORM model for user feedback on assistant answers."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from sqlalchemy import Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class FeedbackRow(Base):
__tablename__ = "feedback"
# Prefix-unique instead of UniqueConstraint: a plain unique over three
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
# mysql_length is ignored on SQLite/Postgres.
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
class FeedbackRow(Base):
__tablename__ = "feedback"
# ``target_key`` is a SHA-256 digest of user/thread/message/agent. A
# digest keeps the idempotency index well below MySQL's legacy utf8mb4
# key-size limit; indexing all four source strings would exceed it.
__table_args__ = (
Index("uq_feedback_target_key", "target_key", unique=True),
Index("ix_feedback_agent_created", "agent_id", "created_at"),
)
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
message_id: Mapped[str | None] = mapped_column(String(64))
# message_id is an optional RunEventStore event identifier —
# allows feedback to target a specific message or the entire run
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
message_id: Mapped[str | None] = mapped_column(String(64))
# ``message_id`` is the LangChain/checkpoint message id serialized inside
# a RunEventStore ``llm.ai.response`` event. It stays nullable solely for
# legacy run-level feedback rows created before answer-level feedback.
agent_id: Mapped[str] = mapped_column(String(128), nullable=False, default="lead_agent")
target_key: Mapped[str] = mapped_column(String(64), nullable=False)
rating: Mapped[int] = mapped_column(nullable=False)
# +1 (thumbs-up) or -1 (thumbs-down)
comment: Mapped[str | None] = mapped_column(Text)
# Optional text feedback from the user
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
comment: Mapped[str | None] = mapped_column(Text)
# Optional user detail. The API exposes this as ``detail`` and retains
# ``comment`` as a compatibility alias for older clients.
reasons: Mapped[list[str]] = mapped_column(PortableJSON(), default=list)
# Stable machine-readable reason codes for negative feedback.
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,225 @@
"""Transparent user-feedback scoring for assistant answers.
This module deliberately uses no model or heuristic content grading. It
summarizes explicit thumbs feedback and selected problem reasons only. The
result is an answer-quality signal, not a complete evaluation of an agent's
planning, reasoning trajectory, tool use, latency, reliability, or safety.
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from typing import Any
FEEDBACK_REASONS = (
"incorrect_or_incomplete",
"unmet_expectations",
"slow_or_problematic",
"style_or_tone",
"safety_or_legal",
"other",
)
# The five response-quality dimensions and weights are fixed product policy.
# A negative reason is evidence only for the explicitly mapped dimensions;
# operational, communication, safety, and uncategorized complaints remain in
# the problem breakdown instead of being mislabeled as task-quality evidence.
DIMENSION_CONFIG: dict[str, dict[str, Any]] = {
"correctness": {
"weight": 0.30,
"mapped_reasons": ["incorrect_or_incomplete"],
},
"completeness": {
"weight": 0.20,
"mapped_reasons": ["incorrect_or_incomplete"],
},
"evidence_grounding": {
"weight": 0.20,
"mapped_reasons": ["incorrect_or_incomplete"],
},
"relevance": {
"weight": 0.15,
"mapped_reasons": ["unmet_expectations"],
},
"actionability": {
"weight": 0.15,
"mapped_reasons": ["unmet_expectations"],
},
}
# Beta(8, 2): prior mean 80/100 with ten equivalent observations. This keeps
# one or two votes from producing an extreme public score while real evidence
# steadily dominates. With no real sample the API returns ``score=null`` so
# the prior is never presented as earned performance.
PRIOR_POSITIVE = 8
PRIOR_NEGATIVE = 2
PRIOR_EQUIVALENT_SAMPLES = PRIOR_POSITIVE + PRIOR_NEGATIVE
PRIOR_MEAN = PRIOR_POSITIVE / PRIOR_EQUIVALENT_SAMPLES
def _posterior_score(successes: int, failures: int) -> float | None:
evidence_count = successes + failures
if evidence_count == 0:
return None
value = 100.0 * (successes + PRIOR_POSITIVE) / (
evidence_count + PRIOR_EQUIVALENT_SAMPLES
)
return round(value, 1)
def _score_band(score: float | None) -> dict[str, Any]:
if score is None:
return {
"key": "insufficient_data",
"label": "数据不足",
"min": 0,
"max": 100,
}
if score >= 90:
return {
"key": "excellent",
"label": "优秀",
"min": 90,
"max": 100,
}
if score >= 80:
return {
"key": "good",
"label": "良好",
"min": 80,
"max": 89.9,
}
if score >= 70:
return {
"key": "needs_attention",
"label": "需关注",
"min": 70,
"max": 79.9,
}
return {
"key": "poor",
"label": "较差",
"min": 0,
"max": 69.9,
}
def _confidence_level(sample_count: int) -> str:
if sample_count == 0:
return "none"
if sample_count < 10:
return "low"
if sample_count < 30:
return "medium"
return "high"
def calculate_answer_quality_score(
rows: Iterable[Mapping[str, Any]],
*,
agent_id: str,
) -> dict[str, Any]:
"""Aggregate explicit answer feedback into a transparent 0-100 score."""
records = list(rows)
positive_count = sum(1 for row in records if int(row.get("rating") or 0) == 1)
negative_rows = [row for row in records if int(row.get("rating") or 0) == -1]
negative_count = len(negative_rows)
sample_count = positive_count + negative_count
reason_counts = {reason: 0 for reason in FEEDBACK_REASONS}
negative_reason_sets: list[set[str]] = []
for row in negative_rows:
raw_reasons = row.get("reasons")
if not isinstance(raw_reasons, list):
raw_reasons = []
reason_set = {
str(reason)
for reason in raw_reasons
if str(reason) in reason_counts
}
negative_reason_sets.append(reason_set)
for reason in reason_set:
reason_counts[reason] += 1
dimensions: dict[str, dict[str, Any]] = {}
weighted_total = 0.0
available_weight = 0.0
for name, config in DIMENSION_CONFIG.items():
mapped = set(config["mapped_reasons"])
failures = sum(1 for reasons in negative_reason_sets if reasons & mapped)
evidence_count = positive_count + failures
dimension_score = _posterior_score(positive_count, failures)
weight = float(config["weight"])
dimensions[name] = {
"weight": weight,
"score": dimension_score,
"evidence_count": evidence_count,
"positive_count": positive_count,
"negative_count": failures,
"mapped_reasons": list(config["mapped_reasons"]),
}
if dimension_score is not None:
weighted_total += weight * dimension_score
available_weight += weight
task_score = (
round(weighted_total / available_weight, 1)
if available_weight > 0
else None
)
score = _posterior_score(positive_count, negative_count)
task_success_rate = (
round(positive_count / sample_count, 4) if sample_count else None
)
reason_breakdown = {
reason: {
"count": count,
"rate": round(count / negative_count, 4) if negative_count else 0.0,
}
for reason, count in reason_counts.items()
}
return {
"agent_id": agent_id,
"score": score,
"score_band": _score_band(score),
"confidence": round(sample_count / (sample_count + 10), 4),
"confidence_level": _confidence_level(sample_count),
"sample_count": sample_count,
"positive_count": positive_count,
"negative_count": negative_count,
"task_success_rate": task_success_rate,
"task_score": task_score,
"dimensions": dimensions,
"reason_breakdown": reason_breakdown,
"prior": {
"positive": PRIOR_POSITIVE,
"negative": PRIOR_NEGATIVE,
"mean": PRIOR_MEAN,
"equivalent_samples": PRIOR_EQUIVALENT_SAMPLES,
},
"formula": {
"scope": "user_feedback_answer_quality",
"overall": (
"100 * (positive_count + 8) / (sample_count + 10); "
"null when sample_count is zero"
),
"confidence": "sample_count / (sample_count + 10)",
"dimension": (
"100 * (positive_count + 8) / "
"(positive_count + mapped_negative_count + 10); "
"null when dimension evidence_count is zero"
),
"task_score": (
"weighted mean of evidenced dimensions using weights "
"30/20/20/15/15, renormalized over available evidence"
),
"reason_rate_denominator": "negative_count",
"limitations": (
"This is not a complete agent-capability score and does not "
"grade hidden reasoning, planning, skill/tool use, latency, "
"reliability, or safety without separate evidence."
),
},
}
@@ -1,17 +1,23 @@
"""SQLAlchemy-backed feedback storage.
"""SQLAlchemy-backed assistant-answer feedback storage.
Each method acquires its own short-lived session.
Each method acquires its own short-lived session. Writes are idempotent on a
SHA-256 target key derived from user/thread/message/agent; the digest avoids
oversized compound indexes on legacy MySQL utf8mb4 deployments.
"""
from __future__ import annotations
import hashlib
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import case, func, select
from sqlalchemy import case, delete, func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.feedback.scoring import FEEDBACK_REASONS, calculate_answer_quality_score
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
@@ -20,12 +26,51 @@ class FeedbackRepository:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: FeedbackRow) -> dict:
d = row.to_dict()
val = d.get("created_at")
if isinstance(val, datetime):
d["created_at"] = val.isoformat()
return d
def _row_to_dict(row: FeedbackRow) -> dict[str, Any]:
data = row.to_dict()
for key in ("created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
data["reasons"] = list(data.get("reasons") or [])
data["detail"] = data.get("comment")
return data
@staticmethod
def make_target_key(
*,
user_id: str | None,
thread_id: str,
run_id: str,
message_id: str | None,
agent_id: str,
) -> str:
"""Build the portable idempotency identity for one answer rating.
Historical run-level callers have no message id. Their deterministic
``run:<id>`` surrogate preserves the old one-rating-per-run behavior.
"""
message_key = message_id or f"run:{run_id}"
raw = "\x1f".join(
(
user_id or "",
thread_id,
message_key,
agent_id or "lead_agent",
)
)
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
@staticmethod
def _normalize_reasons(reasons: list[str] | None) -> list[str]:
allowed = set(FEEDBACK_REASONS)
normalized: list[str] = []
for reason in reasons or []:
value = str(reason).strip()
if value in allowed and value not in normalized:
normalized.append(value)
return normalized
async def create(
self,
@@ -35,34 +80,29 @@ class FeedbackRepository:
rating: int,
user_id: str | None | _AutoSentinel = AUTO,
message_id: str | None = None,
agent_id: str = "lead_agent",
reasons: list[str] | None = None,
comment: str | None = None,
) -> dict:
"""Create a feedback record. rating must be +1 or -1."""
if rating not in (1, -1):
raise ValueError(f"rating must be +1 or -1, got {rating}")
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.create")
row = FeedbackRow(
feedback_id=str(uuid.uuid4()),
) -> dict[str, Any]:
"""Create or update the same answer target idempotently."""
return await self.upsert(
run_id=run_id,
thread_id=thread_id,
user_id=resolved_user_id,
message_id=message_id,
rating=rating,
user_id=user_id,
message_id=message_id,
agent_id=agent_id,
reasons=reasons,
comment=comment,
created_at=datetime.now(UTC),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def get(
self,
feedback_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict | None:
) -> dict[str, Any] | None:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.get")
async with self._sf() as session:
row = await session.get(FeedbackRow, feedback_id)
@@ -72,38 +112,101 @@ class FeedbackRepository:
return None
return self._row_to_dict(row)
async def get_by_target(
self,
*,
thread_id: str,
run_id: str,
message_id: str | None,
agent_id: str,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict[str, Any] | None:
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.get_by_target",
)
target_key = self.make_target_key(
user_id=resolved_user_id,
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=agent_id,
)
async with self._sf() as session:
row = await session.scalar(
select(FeedbackRow).where(FeedbackRow.target_key == target_key)
)
return self._row_to_dict(row) if row is not None else None
async def list_by_run(
self,
thread_id: str,
run_id: str,
*,
limit: int = 100,
message_id: str | None = None,
user_id: str | None | _AutoSentinel = AUTO,
) -> list[dict]:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_run")
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
) -> list[dict[str, Any]]:
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.list_by_run",
)
stmt = select(FeedbackRow).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
)
if message_id is not None:
stmt = stmt.where(FeedbackRow.message_id == message_id)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
return [self._row_to_dict(row) for row in result.scalars()]
async def list_by_thread(
self,
thread_id: str,
*,
limit: int = 100,
limit: int = 1000,
user_id: str | None | _AutoSentinel = AUTO,
) -> list[dict]:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread")
) -> list[dict[str, Any]]:
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.list_by_thread",
)
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
return [self._row_to_dict(row) for row in result.scalars()]
async def list_by_message_ids(
self,
*,
thread_id: str,
message_ids: list[str],
user_id: str | None | _AutoSentinel = AUTO,
) -> list[dict[str, Any]]:
"""Load current-user feedback for a bounded visible-message batch."""
if not message_ids:
return []
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.list_by_message_ids",
)
stmt = select(FeedbackRow).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.message_id.in_(message_ids),
)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(row) for row in result.scalars()]
async def delete(
self,
@@ -111,7 +214,10 @@ class FeedbackRepository:
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete")
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.delete",
)
async with self._sf() as session:
row = await session.get(FeedbackRow, feedback_id)
if row is None:
@@ -129,84 +235,152 @@ class FeedbackRepository:
thread_id: str,
rating: int,
user_id: str | None | _AutoSentinel = AUTO,
message_id: str | None = None,
agent_id: str = "lead_agent",
reasons: list[str] | None = None,
comment: str | None = None,
) -> dict:
"""Create or update feedback for (thread_id, run_id, user_id). rating must be +1 or -1."""
) -> dict[str, Any]:
"""Create or update feedback for one user/thread/message/agent target."""
if rating not in (1, -1):
raise ValueError(f"rating must be +1 or -1, got {rating}")
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.upsert")
async with self._sf() as session:
stmt = select(FeedbackRow).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
FeedbackRow.user_id == resolved_user_id,
)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
if row is not None:
row.rating = rating
row.comment = comment
row.created_at = datetime.now(UTC)
else:
row = FeedbackRow(
feedback_id=str(uuid.uuid4()),
run_id=run_id,
thread_id=thread_id,
user_id=resolved_user_id,
rating=rating,
comment=comment,
created_at=datetime.now(UTC),
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.upsert",
)
normalized_agent_id = str(agent_id or "lead_agent")[:128]
normalized_reasons = self._normalize_reasons(reasons)
target_key = self.make_target_key(
user_id=resolved_user_id,
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=normalized_agent_id,
)
# The unique digest handles cross-worker races. On a simultaneous
# insert, retry as an update in a fresh transaction.
for attempt in range(2):
now = datetime.now(UTC)
async with self._sf() as session:
row = await session.scalar(
select(FeedbackRow).where(FeedbackRow.target_key == target_key)
)
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
if row is None:
row = FeedbackRow(
feedback_id=str(uuid.uuid4()),
run_id=run_id,
thread_id=thread_id,
user_id=resolved_user_id,
message_id=message_id,
agent_id=normalized_agent_id,
target_key=target_key,
rating=rating,
reasons=normalized_reasons,
comment=comment,
created_at=now,
updated_at=now,
)
session.add(row)
else:
row.run_id = run_id
row.rating = rating
row.reasons = normalized_reasons
row.comment = comment
row.updated_at = now
try:
await session.commit()
except IntegrityError:
await session.rollback()
if attempt == 0:
continue
raise
# Runtime factories use expire_on_commit=False. Avoid a
# post-commit refresh, unsafe behind a read replica.
return self._row_to_dict(row)
raise RuntimeError("unreachable feedback upsert retry state")
async def delete_by_run(
self,
*,
thread_id: str,
run_id: str,
message_id: str | None = None,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
"""Delete the current user's feedback for a run. Returns True if a record was deleted."""
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete_by_run")
"""Delete this user's feedback for a run or one message in the run."""
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.delete_by_run",
)
conditions = [
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
FeedbackRow.user_id == resolved_user_id,
]
if message_id is not None:
conditions.append(FeedbackRow.message_id == message_id)
async with self._sf() as session:
stmt = select(FeedbackRow).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
FeedbackRow.user_id == resolved_user_id,
)
result = await session.execute(stmt)
row = result.scalar_one_or_none()
if row is None:
return False
await session.delete(row)
result = await session.execute(delete(FeedbackRow).where(*conditions))
await session.commit()
return True
return bool(result.rowcount)
async def delete_by_target(
self,
*,
thread_id: str,
run_id: str,
message_id: str,
agent_id: str,
user_id: str | None | _AutoSentinel = AUTO,
) -> bool:
resolved_user_id = resolve_user_id(
user_id,
method_name="FeedbackRepository.delete_by_target",
)
target_key = self.make_target_key(
user_id=resolved_user_id,
thread_id=thread_id,
run_id=run_id,
message_id=message_id,
agent_id=agent_id,
)
async with self._sf() as session:
result = await session.execute(
delete(FeedbackRow).where(FeedbackRow.target_key == target_key)
)
await session.commit()
return bool(result.rowcount)
async def list_by_thread_grouped(
self,
thread_id: str,
*,
user_id: str | None | _AutoSentinel = AUTO,
) -> dict[str, dict]:
"""Return feedback grouped by run_id for a thread: {run_id: feedback_dict}."""
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread_grouped")
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
if resolved_user_id is not None:
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
async with self._sf() as session:
result = await session.execute(stmt)
return {row.run_id: self._row_to_dict(row) for row in result.scalars()}
) -> dict[str, dict[str, Any]]:
"""Legacy run-keyed view, retaining the newest row for each run."""
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict:
records = await self.list_by_thread(thread_id, user_id=user_id)
return {record["run_id"]: record for record in records}
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict[str, Any]:
"""Aggregate feedback stats for a run using database-side counting."""
stmt = select(
func.count().label("total"),
func.coalesce(func.sum(case((FeedbackRow.rating == 1, 1), else_=0)), 0).label("positive"),
func.coalesce(func.sum(case((FeedbackRow.rating == -1, 1), else_=0)), 0).label("negative"),
).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
func.coalesce(
func.sum(case((FeedbackRow.rating == 1, 1), else_=0)),
0,
).label("positive"),
func.coalesce(
func.sum(case((FeedbackRow.rating == -1, 1), else_=0)),
0,
).label("negative"),
).where(
FeedbackRow.thread_id == thread_id,
FeedbackRow.run_id == run_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).one()
return {
@@ -215,3 +389,20 @@ class FeedbackRepository:
"positive": row.positive,
"negative": row.negative,
}
async def aggregate_by_agent(self, agent_id: str) -> dict[str, Any]:
"""Return the documented user-feedback answer-quality score."""
normalized_agent_id = str(agent_id or "lead_agent")[:128]
stmt = select(FeedbackRow.rating, FeedbackRow.reasons).where(
FeedbackRow.agent_id == normalized_agent_id
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
return calculate_answer_quality_score(
(
{"rating": row.rating, "reasons": list(row.reasons or [])}
for row in rows
),
agent_id=normalized_agent_id,
)
@@ -0,0 +1,26 @@
"""Force-situation task detail persistence(力量态势任务详情)."""
from __future__ import annotations
from deerflow.persistence.force_situation.base import ForceSituationStore
from deerflow.persistence.force_situation.memory import MemoryForceSituationStore
from deerflow.persistence.force_situation.model import (
ForceSituationDetailRow,
ForceSituationDetailVersionRow,
)
from deerflow.persistence.force_situation.sql import ForceSituationRepository
__all__ = [
"ForceSituationDetailRow",
"ForceSituationDetailVersionRow",
"ForceSituationRepository",
"ForceSituationStore",
"MemoryForceSituationStore",
"make_force_situation_store",
]
def make_force_situation_store(session_factory):
if session_factory is None:
return MemoryForceSituationStore()
return ForceSituationRepository(session_factory)
@@ -0,0 +1,44 @@
"""Persistence contract for force-situation task details(力量态势任务详情).
Each task owns one shared JSON document; the store never inspects the payload
structure — fields are owned by the front-end and may evolve freely.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class ForceSituationStore(ABC):
"""Shared force-situation detail store (no user-level partitioning)."""
@abstractmethod
async def get_detail(self, task_id: str) -> dict[str, Any] | None:
"""Return ``{task_id, detail, updated_at, ...}`` or ``None`` when absent."""
raise NotImplementedError
@abstractmethod
async def upsert_detail(
self,
task_id: str,
detail: dict[str, Any],
*,
updated_by: str | None = None,
) -> dict[str, Any]:
"""Replace the task's detail document wholesale; create when absent.
Every write also appends one version row(时间轴刻度).
"""
raise NotImplementedError
@abstractmethod
async def list_versions(self, task_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
"""Version list for the timeline, ascending by version
(``[{version, created_at, updated_by}]``), newest last."""
raise NotImplementedError
@abstractmethod
async def get_version(self, task_id: str, version: int) -> dict[str, Any] | None:
"""Return one historical version record or ``None`` when absent."""
raise NotImplementedError
@@ -0,0 +1,63 @@
"""In-memory force-situation detail store (tests / no-database mode)."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.force_situation.base import ForceSituationStore
def _now_iso() -> str:
return datetime.now(UTC).astimezone().isoformat(timespec="seconds")
class MemoryForceSituationStore(ForceSituationStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._versions: dict[str, list[dict[str, Any]]] = {}
async def get_detail(self, task_id: str) -> dict[str, Any] | None:
row = self._rows.get(task_id)
return dict(row) if row else None
async def upsert_detail(
self,
task_id: str,
detail: dict[str, Any],
*,
updated_by: str | None = None,
) -> dict[str, Any]:
now = _now_iso()
row = self._rows.get(task_id)
if row is None:
row = {"task_id": task_id, "created_at": now}
self._rows[task_id] = row
row.update(
{
"detail": detail,
"updated_by": updated_by,
"updated_at": now,
}
)
versions = self._versions.setdefault(task_id, [])
versions.append(
{
"version": len(versions) + 1,
"detail": detail,
"updated_by": updated_by,
"created_at": now,
}
)
return dict(row)
async def list_versions(self, task_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
versions = self._versions.get(task_id, [])
trimmed = versions[-limit:] if limit else versions
return [{key: row.get(key) for key in ("version", "created_at", "updated_by")} for row in trimmed]
async def get_version(self, task_id: str, version: int) -> dict[str, Any] | None:
for row in self._versions.get(task_id, []):
if row.get("version") == version:
return dict(row)
return None
@@ -0,0 +1,48 @@
"""ORM model for force-situation task details(力量态势任务详情).
新类型任务的详情文档:一个任务一份自由 JSON(含时间轴、任务衔接、页面单块内容等),
结构由前端与产出方约定、后续可变,数据库不定死字段(整体存 ``detail_json``)。
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class ForceSituationDetailRow(Base):
"""One free-form JSON detail document per external task id (current state)."""
__tablename__ = "force_situation_details"
task_id: Mapped[str] = mapped_column(String(128), primary_key=True)
detail_json: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="{}")
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
class ForceSituationDetailVersionRow(Base):
"""Version history for one task's detail document(时间轴 = 每次写入的版本).
Every PUT appends one version row; ``version`` is a 1-based sequence per task.
"""
__tablename__ = "force_situation_detail_versions"
id: Mapped[int] = mapped_column(autoincrement=True, primary_key=True)
task_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
version: Mapped[int] = mapped_column(nullable=False, default=1)
detail_json: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="{}")
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
@@ -0,0 +1,120 @@
"""SQL repository for force-situation task details."""
from __future__ import annotations
import json
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.force_situation.base import ForceSituationStore
from deerflow.persistence.force_situation.model import (
ForceSituationDetailRow,
ForceSituationDetailVersionRow,
)
def _to_record(row: ForceSituationDetailRow) -> dict[str, Any]:
try:
detail = json.loads(row.detail_json or "{}")
except json.JSONDecodeError:
detail = {}
return {
"task_id": row.task_id,
"detail": detail,
"updated_by": row.updated_by,
"created_at": row.created_at.isoformat() if row.created_at else None,
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
}
def _version_record(row: ForceSituationDetailVersionRow, *, with_detail: bool = False) -> dict[str, Any]:
record: dict[str, Any] = {
"version": row.version,
"created_at": row.created_at.isoformat() if row.created_at else None,
"updated_by": row.updated_by,
}
if with_detail:
try:
record["detail"] = json.loads(row.detail_json or "{}")
except json.JSONDecodeError:
record["detail"] = {}
return record
class ForceSituationRepository(ForceSituationStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._session_factory = session_factory
async def get_detail(self, task_id: str) -> dict[str, Any] | None:
async with self._session_factory() as session:
row = await session.get(ForceSituationDetailRow, task_id)
return _to_record(row) if row else None
async def upsert_detail(
self,
task_id: str,
detail: dict[str, Any],
*,
updated_by: str | None = None,
) -> dict[str, Any]:
async with self._session_factory() as session:
row = await session.get(ForceSituationDetailRow, task_id)
if row is None:
row = ForceSituationDetailRow(task_id=task_id, updated_by=updated_by)
session.add(row)
detail_json = json.dumps(detail, ensure_ascii=False)
row.detail_json = detail_json
row.updated_by = updated_by
# 版本历史:每次写入追加一条(时间轴刻度)。
max_version = (
await session.execute(
select(func.max(ForceSituationDetailVersionRow.version)).where(
ForceSituationDetailVersionRow.task_id == task_id
)
)
).scalar() or 0
session.add(
ForceSituationDetailVersionRow(
task_id=task_id,
version=int(max_version) + 1,
detail_json=detail_json,
updated_by=updated_by,
)
)
await session.commit()
await session.refresh(row)
return _to_record(row)
async def list_versions(self, task_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
async with self._session_factory() as session:
rows = (
(
await session.execute(
select(ForceSituationDetailVersionRow)
.where(ForceSituationDetailVersionRow.task_id == task_id)
.order_by(ForceSituationDetailVersionRow.version.desc())
.limit(limit)
)
)
.scalars()
.all()
)
return [_version_record(row) for row in reversed(rows)]
async def get_version(self, task_id: str, version: int) -> dict[str, Any] | None:
async with self._session_factory() as session:
row = (
(
await session.execute(
select(ForceSituationDetailVersionRow).where(
ForceSituationDetailVersionRow.task_id == task_id,
ForceSituationDetailVersionRow.version == version,
)
)
)
.scalars()
.first()
)
return _version_record(row, with_detail=True) if row else None
@@ -0,0 +1,29 @@
"""Runtime compatibility fixes for the official openGauss SQLAlchemy dialect."""
from __future__ import annotations
from sqlalchemy.dialects.postgresql.base import PGTypeCompiler
class DeerFlowOpenGaussTypeCompiler(PGTypeCompiler):
"""Render variable strings without a length for stricter GaussDB variants.
Some GaussDB PostgreSQL-compatible deployments reject ``VARCHAR(n)`` at
the numeric length token even though ordinary PostgreSQL accepts it. A
length-less ``VARCHAR`` keeps the same variable-length string semantics on
openGauss and avoids coupling the schema to that parser difference.
"""
def visit_string(self, type_, **kw): # noqa: ANN001
return "VARCHAR"
def visit_VARCHAR(self, type_, **kw): # noqa: N802, ANN001
return "VARCHAR"
def install_gauss_dialect_compatibility() -> None:
"""Install the compiler only on the openGauss async dialect class."""
from opengauss_sqlalchemy.asyncpg import OpenGaussDialect_asyncpg
OpenGaussDialect_asyncpg.type_compiler_cls = DeerFlowOpenGaussTypeCompiler
@@ -0,0 +1,218 @@
"""Compatibility helpers for openGauss/Dolphin's MySQL protocol.
The Dolphin endpoint deliberately presents itself as MySQL, but a few pieces
of server metadata and DDL reflection differ from real MySQL. These patches
are installed only when the deployment explicitly selects the ``gauss_mysql``
database product. A normal MySQL deployment never imports or installs them.
"""
from __future__ import annotations
import ast
import json
import logging
import math
import os
from typing import Any
logger = logging.getLogger(__name__)
GAUSS_MYSQL_PRODUCT = "gauss_mysql"
GAUSS_MYSQL_SQLALCHEMY_VERSION = (5, 7, 20)
_DOLPHIN_INIT_COMMAND = "SET standard_conforming_strings TO off"
def is_gauss_mysql_product() -> bool:
"""Return whether this process was started for the Dolphin endpoint."""
return os.getenv("DEERFLOW_DATABASE_PRODUCT", "").strip().lower() == GAUSS_MYSQL_PRODUCT
def is_gauss_mysql_dialect(dialect: object) -> bool:
"""Return whether the selected SQLAlchemy dialect has the Dolphin hook."""
return bool(getattr(dialect, "_deerflow_gauss_mysql_compat_installed", False))
def _decode_scalar(value: Any) -> str:
if isinstance(value, bytes):
return value.decode(errors="replace")
return str(value)
def _gauss_mysql_default_schema_name(self, connection): # noqa: ANN001, ARG001
"""Return Dolphin's current schema rather than its physical database."""
schema = connection.exec_driver_sql("SELECT current_schema()").scalar()
if schema is None:
raise RuntimeError("GaussDB did not return current_schema()")
return _decode_scalar(schema)
def _gauss_mysql_server_version_info(self, connection): # noqa: ANN001
"""Expose a stable MySQL capability version for a Dolphin connection."""
dbapi_connection = connection.connection
cursor = dbapi_connection.cursor()
try:
cursor.execute("SELECT VERSION()")
row = cursor.fetchone()
finally:
cursor.close()
raw_version = "" if row is None or row[0] is None else _decode_scalar(row[0])
version = GAUSS_MYSQL_SQLALCHEMY_VERSION
self._deerflow_gauss_server_version = raw_version
self._set_mariadb(False, version)
self._mariadb_normalized_version_info = version
self.server_version_info = version
logger.info(
"GaussDB MySQL compatibility enabled: server=%r, SQLAlchemy capability=%s",
raw_version,
".".join(map(str, version)),
)
return version
def _gauss_mysql_isolation_level(self, dbapi_connection): # noqa: ANN001, ARG001
"""Read isolation without comparing an unparseable vendor version."""
cursor = dbapi_connection.cursor()
try:
cursor.execute("SELECT @@transaction_isolation")
row = cursor.fetchone()
finally:
cursor.close()
if row is None or row[0] is None:
raise NotImplementedError("GaussDB did not return a transaction isolation level")
return _decode_scalar(row[0]).upper().replace("-", " ")
def _gauss_mysql_has_table(self, connection, table_name, schema=None, **kw): # noqa: ANN001, ARG001
"""Check relation existence without a failing ``DESCRIBE`` transaction."""
self._ensure_has_table_connection(connection)
if schema is None:
schema = self.default_schema_name
if schema is None:
raise ValueError("GaussDB MySQL table check requires a database name")
statement = "SELECT 1 FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE n.nspname = %s AND c.relname = %s AND c.relkind IN ('r', 'p', 'v', 'm', 'f') LIMIT 1"
with connection.exec_driver_sql(
statement,
(schema, table_name),
execution_options={"skip_user_error_events": True},
) as result:
return result.fetchone() is not None
def _gauss_mysql_create_connect_args(self, url): # noqa: ANN001
"""Add the Dolphin session setting required by PyMySQL-compatible drivers."""
args, kwargs = self._deerflow_gauss_mysql_original_create_connect_args(url)
existing = kwargs.get("init_command")
if existing and existing.strip().rstrip(";").lower() != _DOLPHIN_INIT_COMMAND.lower():
raise ValueError(
"gauss_mysql requires init_command='SET standard_conforming_strings TO off'; "
"remove the conflicting init_command from the database URL"
)
kwargs["init_command"] = _DOLPHIN_INIT_COMMAND
return args, kwargs
def _unescape_gauss_mysql_json_container(value: str) -> Any:
"""Undo exactly one complete MySQL SQL-string escape layer."""
text = value.strip()
if not text.startswith(("[", "{")) or "\\" not in text:
raise ValueError("Not an escaped JSON container")
escapes = {
"0": "\x00",
"n": "\n",
"r": "\r",
"Z": "\x1a",
"\\": "\\",
"'": "'",
'"': '"',
}
output: list[str] = []
position = 0
while position < len(text):
character = text[position]
if character == "\\":
position += 1
if position == len(text) or text[position] not in escapes:
raise ValueError("Unknown or incomplete MySQL escape")
output.append(escapes[text[position]])
elif character in {'"', "'"}:
raise ValueError("Mixed escaped and unescaped quotes")
else:
output.append(character)
position += 1
decoded = json.loads("".join(output))
if not isinstance(decoded, (dict, list)):
raise ValueError("Escaped legacy JSON must be an object or array")
return decoded
def _is_json_domain(value: Any) -> bool:
"""Recursively reject Python-only values accepted by ``literal_eval``."""
if value is None or isinstance(value, (str, bool, int)):
return True
if isinstance(value, float):
return math.isfinite(value)
if isinstance(value, list):
return all(_is_json_domain(item) for item in value)
if isinstance(value, dict):
return all(isinstance(key, str) and _is_json_domain(item) for key, item in value.items())
return False
def decode_gauss_mysql_json(value: Any) -> Any:
"""Decode strict JSON plus the two legacy Dolphin encodings we shipped."""
if isinstance(value, memoryview):
value = value.tobytes()
if isinstance(value, bytes):
value = value.decode("utf-8")
if not isinstance(value, str):
return value
try:
return json.loads(value)
except json.JSONDecodeError:
pass
try:
decoded = ast.literal_eval(value)
except (SyntaxError, ValueError):
try:
return _unescape_gauss_mysql_json_container(value)
except (TypeError, ValueError):
preview = value[:160].replace("\n", "\\n")
raise ValueError(f"GaussDB MySQL JSON value is neither valid JSON, a supported legacy literal, nor a single SQL-escaped JSON container: {preview!r}") from None
if not _is_json_domain(decoded):
raise ValueError(f"GaussDB MySQL JSON legacy literal decoded to an unsupported type: {type(decoded).__name__}")
return decoded
def install_gauss_mysql_dialect_compatibility(driver: str = "aiomysql") -> None:
"""Patch only the selected async MySQL dialect class, once per process."""
if driver == "aiomysql":
from sqlalchemy.dialects.mysql.aiomysql import MySQLDialect_aiomysql as dialect_class
elif driver == "asyncmy":
from sqlalchemy.dialects.mysql.asyncmy import MySQLDialect_asyncmy as dialect_class
else:
raise ValueError(f"Unsupported GaussDB MySQL driver: {driver}")
if getattr(dialect_class, "_deerflow_gauss_mysql_compat_installed", False):
return
dialect_class._deerflow_gauss_mysql_original_create_connect_args = dialect_class.create_connect_args
dialect_class.create_connect_args = _gauss_mysql_create_connect_args
dialect_class._get_server_version_info = _gauss_mysql_server_version_info
dialect_class._get_default_schema_name = _gauss_mysql_default_schema_name
dialect_class.get_isolation_level = _gauss_mysql_isolation_level
dialect_class.has_table = _gauss_mysql_has_table
dialect_class._deerflow_gauss_mysql_compat_installed = True
@@ -0,0 +1,125 @@
"""Conservatively extend ORM tables through openGauss native catalogs.
This is used only through Dolphin's MySQL protocol. The stock MySQL inspector
cannot parse Dolphin's PostgreSQL-style ``SHOW CREATE TABLE`` result and can
therefore report existing columns as missing.
"""
from __future__ import annotations
import logging
from sqlalchemy import CheckConstraint, ForeignKeyConstraint, MetaData, UniqueConstraint
from sqlalchemy.engine import Connection
from sqlalchemy.schema import CreateColumn, CreateIndex
logger = logging.getLogger(__name__)
def _string(value: object) -> str:
return value.decode("utf-8") if isinstance(value, bytes) else str(value)
def ensure_gauss_schema_sync(connection: Connection, metadata: MetaData) -> None:
"""Add only safe missing columns and indexes involving those columns.
Existing columns and unrelated indexes are never altered. A missing
column without a nullable definition or database-side default cannot be
backfilled reliably and stops this repair before any DDL is issued.
"""
current_schema_value = connection.exec_driver_sql("SELECT current_schema()").scalar_one_or_none()
if not current_schema_value:
raise RuntimeError("openGauss schema integrity check: current_schema() is empty")
schema = _string(current_schema_value)
expected_schema = connection.engine.url.database
if expected_schema and expected_schema != schema:
raise RuntimeError(f"openGauss schema integrity check: URL schema {expected_schema!r} does not match current_schema() {schema!r}")
catalog_rows = connection.exec_driver_sql(
"SELECT c.relname, c.relkind, a.attname "
"FROM pg_catalog.pg_class AS c "
"JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace "
"LEFT JOIN pg_catalog.pg_attribute AS a ON a.attrelid = c.oid "
"AND a.attnum > 0 AND NOT a.attisdropped "
"WHERE n.nspname = %s AND c.relkind IN ('r', 'p', 'v', 'm', 'f')",
(schema,),
).fetchall()
columns_by_table: dict[str, set[str]] = {}
kinds_by_table: dict[str, str] = {}
for table_name, kind, column_name in catalog_rows:
table_name = _string(table_name)
kinds_by_table[table_name] = _string(kind)
columns_by_table.setdefault(table_name, set())
if column_name is not None:
columns_by_table[table_name].add(_string(column_name))
plan = []
failures: list[str] = []
for table in metadata.sorted_tables:
if table.schema is not None and table.schema != schema:
failures.append(f"{table.fullname}: explicit schema differs from {schema!r}")
continue
existing = columns_by_table.get(table.name)
if existing is None:
failures.append(f"{schema}.{table.name}: table is still missing after create_all")
continue
if kinds_by_table[table.name] not in {"r", "p"}:
failures.append(f"{schema}.{table.name}: existing relation is not an ordinary table")
continue
missing = [column for column in table.columns if column.name not in existing]
if not missing:
continue
missing_names = {column.name for column in missing}
for column in missing:
label = f"{schema}.{table.name}.{column.name}"
if not column.nullable and column.server_default is None:
failures.append(f"{label}: NOT NULL column has no server_default; explicit migration required")
elif column.primary_key or column.identity is not None or column.computed is not None:
failures.append(f"{label}: primary-key/identity/computed column requires an explicit migration")
for constraint in table.constraints:
if isinstance(constraint, (ForeignKeyConstraint, UniqueConstraint)):
constrained = {column.name for column in constraint.columns}
if constrained & missing_names:
failures.append(f"{schema}.{table.name}: missing columns participate in {type(constraint).__name__}; explicit constraint migration required")
elif isinstance(constraint, CheckConstraint):
constrained = {column.name for column in constraint.columns}
if not constrained or constrained & missing_names:
failures.append(f"{schema}.{table.name}: CHECK-constrained schema needs an explicit migration")
indexes = [index for index in table.indexes if {column.name for column in index.columns} & missing_names]
for index in indexes:
if not index.name:
failures.append(f"{schema}.{table.name}: index involving new columns has no stable name")
plan.append((table, missing, indexes))
if failures:
raise RuntimeError("openGauss schema is incomplete; startup repair stopped:\n- " + "\n- ".join(failures))
if not plan:
logger.info("openGauss schema integrity check passed: %s ORM tables", len(metadata.tables))
return
existing_indexes = {
(_string(table_name), _string(index_name))
for table_name, index_name in connection.exec_driver_sql(
"SELECT t.relname, i.relname FROM pg_catalog.pg_index AS x "
"JOIN pg_catalog.pg_class AS t ON t.oid = x.indrelid "
"JOIN pg_catalog.pg_class AS i ON i.oid = x.indexrelid "
"JOIN pg_catalog.pg_namespace AS n ON n.oid = t.relnamespace "
"WHERE n.nspname = %s",
(schema,),
).fetchall()
}
preparer = connection.dialect.identifier_preparer
added_columns = 0
for table, missing, indexes in plan:
qualified = f"{preparer.quote_schema(schema)}.{preparer.quote(table.name)}"
for column in missing:
definition = str(CreateColumn(column).compile(dialect=connection.dialect))
connection.exec_driver_sql(f"ALTER TABLE {qualified} ADD COLUMN {definition}")
added_columns += 1
logger.info("Added safe missing openGauss column: %s.%s", qualified, column.name)
for index in sorted(indexes, key=lambda item: str(item.name)):
if (table.name, str(index.name)) not in existing_indexes:
connection.execute(CreateIndex(index))
logger.info("Created openGauss index involving new columns: %s", index.name)
logger.info("openGauss schema integrity repair completed: %s columns", added_columns)
@@ -0,0 +1,22 @@
"""Durable, owner-scoped records for the public-document workbench."""
from __future__ import annotations
from deerflow.persistence.gongwen_documents.memory import MemoryGongwenDocumentStore
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore, SqlGongwenDocumentStore
__all__ = [
"GongwenDocumentRow",
"GongwenDocumentStore",
"GongwenDocumentVersionRow",
"MemoryGongwenDocumentStore",
"SqlGongwenDocumentStore",
"make_gongwen_document_store",
]
def make_gongwen_document_store(session_factory):
if session_factory is None:
return MemoryGongwenDocumentStore()
return SqlGongwenDocumentStore(session_factory)
@@ -0,0 +1,75 @@
"""Small in-memory fallback used when persistence is disabled in tests."""
from __future__ import annotations
from copy import deepcopy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore
class MemoryGongwenDocumentStore(GongwenDocumentStore):
def __init__(self) -> None:
self._documents: dict[str, dict[str, Any]] = {}
self._versions: dict[str, list[dict[str, Any]]] = {}
@staticmethod
def _owned(document: dict[str, Any] | None, user_id: str | None) -> bool:
return bool(document and document.get("user_id") == user_id)
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC).isoformat()
row = {
"id": str(data["id"]), "user_id": data.get("user_id"), "title": str(data.get("title") or ""),
"document_type": str(data.get("document_type") or "notice"), "source": str(data.get("source") or "upload"),
"status": str(data.get("status") or "checking"), "template_key": data.get("template_key"),
"research_session_id": data.get("research_session_id"), "original_version_id": None, "latest_version_id": None,
"issue_counts": deepcopy(data.get("issue_counts")), "error": data.get("error"), "created_at": now, "updated_at": now,
}
self._documents[row["id"]] = row
return deepcopy(row)
async def list_documents(self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
rows = [r for r in self._documents.values() if r.get("user_id") == user_id]
if source:
rows = [r for r in rows if r.get("source") == source]
if status:
rows = [r for r in rows if r.get("status") == status]
return deepcopy(sorted(rows, key=lambda item: item["updated_at"], reverse=True)[:limit])
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
row = self._documents.get(document_id)
return deepcopy(row) if self._owned(row, user_id) else None
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
if not self._owned(self._documents.get(document_id), user_id):
return []
return deepcopy(list(reversed(self._versions.get(document_id, []))))
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
if not self._owned(self._documents.get(document_id), user_id):
return None
return next((deepcopy(row) for row in self._versions.get(document_id, []) if row["id"] == version_id), None)
async def append_version(self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]] | None:
document = self._documents.get(document_id)
if not self._owned(document, user_id):
return None
rows = self._versions.setdefault(document_id, [])
row = {**deepcopy(version), "document_id": document_id, "version_no": len(rows) + 1, "created_at": datetime.now(UTC).isoformat()}
rows.append(row)
document.update(deepcopy(document_patch))
document["latest_version_id"] = row["id"]
if row["kind"] == "original" and not document.get("original_version_id"):
document["original_version_id"] = row["id"]
document["updated_at"] = datetime.now(UTC).isoformat()
return deepcopy(document), deepcopy(row)
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
document = self._documents.get(document_id)
if not self._owned(document, user_id):
return None
document.update(deepcopy(patch))
document["updated_at"] = datetime.now(UTC).isoformat()
return deepcopy(document)
@@ -0,0 +1,68 @@
"""ORM rows for the public-document library.
The binary files deliberately stay outside the database. A document row is
the durable library entry; every uploaded original, generated output and
format-repaired output is an immutable version row beneath it.
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class GongwenDocumentRow(Base):
__tablename__ = "gongwen_documents"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
document_type: Mapped[str] = mapped_column(String(48), nullable=False, default="notice", index=True)
# upload | generated. Keep source separate from status so the library can
# filter both axes independently.
source: Mapped[str] = mapped_column(String(24), nullable=False, default="upload", index=True)
# checking | review_required | verified | repaired | failed
status: Mapped[str] = mapped_column(String(32), nullable=False, default="checking", index=True)
template_key: Mapped[str | None] = mapped_column(String(96), nullable=True)
research_session_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
original_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
latest_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
issue_counts: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
class GongwenDocumentVersionRow(Base):
__tablename__ = "gongwen_document_versions"
__table_args__ = (
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
sa.Index("ix_gongwen_document_versions_document_created", "document_id", "created_at"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
document_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
version_no: Mapped[int] = mapped_column(Integer, nullable=False)
# original | generated | repaired | markdown
kind: Mapped[str] = mapped_column(String(32), nullable=False)
file_name: Mapped[str] = mapped_column(String(512), nullable=False)
storage_key: Mapped[str] = mapped_column(String(1024), nullable=False)
content_type: Mapped[str] = mapped_column(String(128), nullable=False, default="application/octet-stream")
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
sha256: Mapped[str] = mapped_column(String(64), nullable=False, default="")
# Exact review snapshot for this version. Later rule updates must not
# rewrite history or make past repair decisions impossible to explain.
review_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -0,0 +1,203 @@
"""SQL and in-memory stores for public-document library metadata."""
from __future__ import annotations
import json
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
def _json_load(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except json.JSONDecodeError:
return None
def _json_dump(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
def _as_dict(row: GongwenDocumentRow | GongwenDocumentVersionRow) -> dict[str, Any]:
result = row.to_dict()
for key, value in tuple(result.items()):
if isinstance(value, datetime):
result[key] = value.isoformat()
if "issue_counts" in result:
result["issue_counts"] = _json_load(result["issue_counts"])
if "review_snapshot" in result:
result["review_snapshot"] = _json_load(result["review_snapshot"])
return result
class GongwenDocumentStore(ABC):
@abstractmethod
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_documents(
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def append_version(
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
) -> tuple[dict[str, Any], dict[str, Any]] | None:
raise NotImplementedError
@abstractmethod
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
class SqlGongwenDocumentStore(GongwenDocumentStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _owned(stmt, user_id: str | None):
return stmt.where(GongwenDocumentRow.user_id == user_id)
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
row = GongwenDocumentRow(
id=str(data["id"]),
user_id=data.get("user_id"),
title=str(data.get("title") or ""),
document_type=str(data.get("document_type") or "notice"),
source=str(data.get("source") or "upload"),
status=str(data.get("status") or "checking"),
template_key=data.get("template_key"),
research_session_id=data.get("research_session_id"),
issue_counts=_json_dump(data.get("issue_counts")),
error=data.get("error"),
)
async with self._sf() as session:
session.add(row)
await session.commit()
await session.refresh(row)
return _as_dict(row)
async def list_documents(
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
) -> list[dict[str, Any]]:
stmt = self._owned(select(GongwenDocumentRow), user_id)
if source:
stmt = stmt.where(GongwenDocumentRow.source == source)
if status:
stmt = stmt.where(GongwenDocumentRow.status == status)
stmt = stmt.order_by(GongwenDocumentRow.updated_at.desc()).limit(max(1, min(limit, 200)))
async with self._sf() as session:
return [_as_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _as_dict(row) if row else None
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
async with self._sf() as session:
if (await session.execute(stmt)).scalar_one_or_none() is None:
return []
versions = await session.execute(
select(GongwenDocumentVersionRow)
.where(GongwenDocumentVersionRow.document_id == document_id)
.order_by(GongwenDocumentVersionRow.version_no.desc())
)
return [_as_dict(row) for row in versions.scalars()]
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
async with self._sf() as session:
if (await session.execute(stmt)).scalar_one_or_none() is None:
return None
row = (
await session.execute(
select(GongwenDocumentVersionRow).where(
GongwenDocumentVersionRow.document_id == document_id,
GongwenDocumentVersionRow.id == version_id,
)
)
).scalar_one_or_none()
return _as_dict(row) if row else None
async def append_version(
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
) -> tuple[dict[str, Any], dict[str, Any]] | None:
async with self._sf() as session:
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
document = (await session.execute(stmt)).scalar_one_or_none()
if document is None:
return None
version_no = int(
(await session.execute(
select(func.coalesce(func.max(GongwenDocumentVersionRow.version_no), 0)).where(
GongwenDocumentVersionRow.document_id == document_id
)
)).scalar_one()
) + 1
row = GongwenDocumentVersionRow(
id=str(version["id"]),
document_id=document_id,
version_no=version_no,
kind=str(version["kind"]),
file_name=str(version["file_name"]),
storage_key=str(version["storage_key"]),
content_type=str(version.get("content_type") or "application/octet-stream"),
size_bytes=int(version.get("size_bytes") or 0),
sha256=str(version.get("sha256") or ""),
review_snapshot=_json_dump(version.get("review_snapshot")),
)
for key, value in document_patch.items():
if key == "issue_counts":
value = _json_dump(value)
if hasattr(document, key):
setattr(document, key, value)
document.latest_version_id = row.id
if row.kind == "original" and not document.original_version_id:
document.original_version_id = row.id
session.add(row)
await session.commit()
await session.refresh(document)
await session.refresh(row)
return _as_dict(document), _as_dict(row)
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
return None
for key, value in patch.items():
if key == "issue_counts":
value = _json_dump(value)
if hasattr(row, key):
setattr(row, key, value)
await session.commit()
await session.refresh(row)
return _as_dict(row)
@@ -0,0 +1,30 @@
"""Database-backed global ordinary-knowledge retrieval settings."""
from __future__ import annotations
from deerflow.persistence.knowledge_retrieval_settings.base import (
KnowledgeRetrievalSettingsStore,
)
from deerflow.persistence.knowledge_retrieval_settings.memory import (
MemoryKnowledgeRetrievalSettingsStore,
)
from deerflow.persistence.knowledge_retrieval_settings.model import (
KnowledgeRetrievalSettingsRow,
)
from deerflow.persistence.knowledge_retrieval_settings.sql import (
KnowledgeRetrievalSettingsRepository,
)
__all__ = [
"KnowledgeRetrievalSettingsRepository",
"KnowledgeRetrievalSettingsRow",
"KnowledgeRetrievalSettingsStore",
"MemoryKnowledgeRetrievalSettingsStore",
"make_knowledge_retrieval_settings_store",
]
def make_knowledge_retrieval_settings_store(session_factory):
if session_factory is None:
return MemoryKnowledgeRetrievalSettingsStore()
return KnowledgeRetrievalSettingsRepository(session_factory)
@@ -0,0 +1,38 @@
"""Persistence contract for the global ordinary-knowledge retrieval switch."""
from __future__ import annotations
from abc import ABC, abstractmethod
class KnowledgeRetrievalSettingsStore(ABC):
"""Store the deployment-wide smart-reasoning availability flag."""
@abstractmethod
async def get(self) -> bool:
"""Return whether smart reasoning is enabled; missing state is disabled."""
raise NotImplementedError
@abstractmethod
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
"""Atomically replace the global flag and return the saved value."""
raise NotImplementedError
@abstractmethod
async def get_advanced_management_enabled(self) -> bool:
"""Return whether advanced knowledge-management entries are exposed."""
raise NotImplementedError
@abstractmethod
async def set_advanced_management_enabled(
self,
enabled: bool,
*,
updated_by: str | None = None,
) -> bool:
"""Atomically replace the advanced-management flag."""
raise NotImplementedError
@@ -0,0 +1,41 @@
"""In-memory ordinary-knowledge retrieval settings store."""
from __future__ import annotations
import asyncio
from deerflow.persistence.knowledge_retrieval_settings.base import (
KnowledgeRetrievalSettingsStore,
)
class MemoryKnowledgeRetrievalSettingsStore(KnowledgeRetrievalSettingsStore):
def __init__(self) -> None:
self._enabled = False
self._advanced_management_enabled = False
self._lock = asyncio.Lock()
async def get(self) -> bool:
async with self._lock:
return self._enabled
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
del updated_by
async with self._lock:
self._enabled = bool(enabled)
return self._enabled
async def get_advanced_management_enabled(self) -> bool:
async with self._lock:
return self._advanced_management_enabled
async def set_advanced_management_enabled(
self,
enabled: bool,
*,
updated_by: str | None = None,
) -> bool:
del updated_by
async with self._lock:
self._advanced_management_enabled = bool(enabled)
return self._advanced_management_enabled
@@ -0,0 +1,37 @@
"""ORM row for the database-backed ordinary-knowledge retrieval switch."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, false
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class KnowledgeRetrievalSettingsRow(Base):
__tablename__ = "knowledge_retrieval_settings"
scope: Mapped[str] = mapped_column(String(64), primary_key=True, default="global")
smart_reasoning_enabled: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
default=False,
server_default=false(),
)
advanced_management_enabled: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
default=False,
server_default=false(),
)
revision: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,117 @@
"""SQLAlchemy-backed ordinary-knowledge retrieval settings store."""
from __future__ import annotations
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.knowledge_retrieval_settings.base import (
KnowledgeRetrievalSettingsStore,
)
from deerflow.persistence.knowledge_retrieval_settings.model import (
KnowledgeRetrievalSettingsRow,
)
_GLOBAL_SCOPE = "global"
class KnowledgeRetrievalSettingsRepository(KnowledgeRetrievalSettingsStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def get(self) -> bool:
async with self._sf() as session:
row = await session.get(KnowledgeRetrievalSettingsRow, _GLOBAL_SCOPE)
return bool(row.smart_reasoning_enabled) if row is not None else False
async def get_advanced_management_enabled(self) -> bool:
async with self._sf() as session:
row = await session.get(KnowledgeRetrievalSettingsRow, _GLOBAL_SCOPE)
return bool(row.advanced_management_enabled) if row is not None else False
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
saved = bool(enabled)
async with self._sf() as session:
result = await session.execute(
select(KnowledgeRetrievalSettingsRow)
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
.with_for_update()
)
row = result.scalar_one_or_none()
if row is None:
row = KnowledgeRetrievalSettingsRow(
scope=_GLOBAL_SCOPE,
smart_reasoning_enabled=saved,
revision=1,
updated_by=updated_by,
)
session.add(row)
else:
row.smart_reasoning_enabled = saved
row.updated_by = updated_by
row.revision = int(row.revision or 0) + 1
try:
await session.commit()
except IntegrityError:
# Two workers may both observe the initially absent singleton.
# The loser retries as an update instead of surfacing a 500.
await session.rollback()
result = await session.execute(
select(KnowledgeRetrievalSettingsRow)
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
.with_for_update()
)
row = result.scalar_one_or_none()
if row is None:
raise
row.smart_reasoning_enabled = saved
row.updated_by = updated_by
row.revision = int(row.revision or 0) + 1
await session.commit()
return saved
async def set_advanced_management_enabled(
self,
enabled: bool,
*,
updated_by: str | None = None,
) -> bool:
saved = bool(enabled)
async with self._sf() as session:
result = await session.execute(
select(KnowledgeRetrievalSettingsRow)
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
.with_for_update()
)
row = result.scalar_one_or_none()
if row is None:
row = KnowledgeRetrievalSettingsRow(
scope=_GLOBAL_SCOPE,
smart_reasoning_enabled=False,
advanced_management_enabled=saved,
revision=1,
updated_by=updated_by,
)
session.add(row)
else:
row.advanced_management_enabled = saved
row.updated_by = updated_by
row.revision = int(row.revision or 0) + 1
try:
await session.commit()
except IntegrityError:
await session.rollback()
result = await session.execute(
select(KnowledgeRetrievalSettingsRow)
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
.with_for_update()
)
row = result.scalar_one_or_none()
if row is None:
raise
row.advanced_management_enabled = saved
row.updated_by = updated_by
row.revision = int(row.revision or 0) + 1
await session.commit()
return saved
@@ -6,11 +6,47 @@ from abc import ABC, abstractmethod
from typing import Any, Literal
LlmWikiScope = Literal["personal", "public", "all"]
UNSET = object()
def serialize_wiki_parent_directories(
value: list[str] | None,
) -> list[str] | None:
return None if value is None else [str(item) for item in value]
def serialize_wiki_directory_paths(
value: list[list[str]] | None,
) -> list[list[str]] | None:
return (
None
if value is None
else [[str(segment) for segment in path] for path in value]
)
class LlmWikiStore(ABC):
@abstractmethod
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]: ...
async def create_mapping(
self,
*,
weknora_id: str,
owner_user_id: str,
name: str,
description: str,
kb_type: str,
retrieval_settings: dict[str, Any] | None = None,
wiki_parent_directories: list[str] | None = None,
wiki_directory_paths: list[list[str]] | None = None,
wiki_focus_instructions: str | None = None,
wiki_content_instructions_raw: str | None = None,
wiki_extraction_instructions_raw: str | None = None,
wiki_generation_managed: bool | None = None,
wiki_config_schema_version: int | None = None,
wiki_config_revision: int | None = None,
wiki_config_apply_status: str | None = None,
wiki_config_apply_error: str | None = None,
) -> dict[str, Any]: ...
@abstractmethod
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]: ...
@@ -28,8 +64,21 @@ class LlmWikiStore(ABC):
*,
name: str | None = None,
description: str | None = None,
test_questions: list[str] | None = None,
test_questions_set: bool = False,
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
wiki_parent_directories: list[str] | None | object = UNSET,
wiki_directory_paths: list[list[str]] | None | object = UNSET,
wiki_focus_instructions: str | None | object = UNSET,
wiki_content_instructions_raw: str | None | object = UNSET,
wiki_extraction_instructions_raw: str | None | object = UNSET,
wiki_generation_managed: bool | None | object = UNSET,
wiki_config_schema_version: int | None | object = UNSET,
wiki_config_revision: int | None | object = UNSET,
wiki_config_apply_status: str | None | object = UNSET,
wiki_config_apply_error: str | None | object = UNSET,
) -> dict[str, Any] | None: ...
@abstractmethod
@@ -3,22 +3,58 @@
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from typing import Any, cast
from uuid import uuid4
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.base import (
UNSET,
LlmWikiScope,
LlmWikiStore,
serialize_wiki_directory_paths,
serialize_wiki_parent_directories,
)
def _now() -> str:
return datetime.now(UTC).isoformat()
def _mapping_to_dict(row: dict[str, Any]) -> dict[str, Any]:
data = dict(row)
data["wiki_parent_directories"] = serialize_wiki_parent_directories(
data.get("wiki_parent_directories")
)
data["wiki_directory_paths"] = serialize_wiki_directory_paths(
data.get("wiki_directory_paths")
)
return data
class MemoryLlmWikiStore(LlmWikiStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._conversation_deposits: dict[str, dict[str, Any]] = {}
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
async def create_mapping(
self,
*,
weknora_id: str,
owner_user_id: str,
name: str,
description: str,
kb_type: str,
retrieval_settings: dict[str, Any] | None = None,
wiki_parent_directories: list[str] | None = None,
wiki_directory_paths: list[list[str]] | None = None,
wiki_focus_instructions: str | None = None,
wiki_content_instructions_raw: str | None = None,
wiki_extraction_instructions_raw: str | None = None,
wiki_generation_managed: bool | None = None,
wiki_config_schema_version: int | None = None,
wiki_config_revision: int | None = None,
wiki_config_apply_status: str | None = None,
wiki_config_apply_error: str | None = None,
) -> dict[str, Any]:
if any(row["weknora_id"] == weknora_id for row in self._rows.values()):
raise ValueError("WeKnora knowledge base is already mapped")
now = _now()
@@ -28,6 +64,26 @@ class MemoryLlmWikiStore(LlmWikiStore):
"owner_user_id": owner_user_id,
"name": name,
"description": description,
"test_questions": None,
"retrieval_settings": (
dict(retrieval_settings)
if retrieval_settings is not None
else None
),
"wiki_parent_directories": serialize_wiki_parent_directories(
wiki_parent_directories
),
"wiki_directory_paths": serialize_wiki_directory_paths(
wiki_directory_paths
),
"wiki_focus_instructions": wiki_focus_instructions,
"wiki_content_instructions_raw": wiki_content_instructions_raw,
"wiki_extraction_instructions_raw": wiki_extraction_instructions_raw,
"wiki_generation_managed": wiki_generation_managed,
"wiki_config_schema_version": wiki_config_schema_version,
"wiki_config_revision": wiki_config_revision,
"wiki_config_apply_status": wiki_config_apply_status,
"wiki_config_apply_error": wiki_config_apply_error,
"kb_type": kb_type,
"publication_status": "private",
"published_by": None,
@@ -40,7 +96,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
"updated_at": now,
}
self._rows[row["id"]] = row
return dict(row)
return _mapping_to_dict(row)
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
rows = list(self._rows.values())
@@ -50,22 +106,22 @@ class MemoryLlmWikiStore(LlmWikiStore):
rows = [row for row in rows if row["publication_status"] == "published"]
else:
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id or row["publication_status"] == "published"]
return [dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
return [_mapping_to_dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
if is_admin or row["owner_user_id"] == user_id:
return dict(row)
return _mapping_to_dict(row)
if not write and row["publication_status"] == "published":
return dict(row)
return _mapping_to_dict(row)
return None
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
for row in self._rows.values():
if row["weknora_id"] == weknora_id:
return dict(row)
return _mapping_to_dict(row)
return None
async def update_mapping(
@@ -74,8 +130,21 @@ class MemoryLlmWikiStore(LlmWikiStore):
*,
name: str | None = None,
description: str | None = None,
test_questions: list[str] | None = None,
test_questions_set: bool = False,
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
wiki_parent_directories: list[str] | None | object = UNSET,
wiki_directory_paths: list[list[str]] | None | object = UNSET,
wiki_focus_instructions: str | None | object = UNSET,
wiki_content_instructions_raw: str | None | object = UNSET,
wiki_extraction_instructions_raw: str | None | object = UNSET,
wiki_generation_managed: bool | None | object = UNSET,
wiki_config_schema_version: int | None | object = UNSET,
wiki_config_revision: int | None | object = UNSET,
wiki_config_apply_status: str | None | object = UNSET,
wiki_config_apply_error: str | None | object = UNSET,
) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
@@ -84,13 +153,65 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["name"] = name
if description is not None:
row["description"] = description
if test_questions_set:
row["test_questions"] = list(test_questions or [])
if retrieval_settings is not None:
row["retrieval_settings"] = dict(retrieval_settings)
if wiki_index_enabled is not None:
row["wiki_index_enabled"] = wiki_index_enabled
if external_search_enabled is not None:
row["external_search_enabled"] = external_search_enabled
row["external_search_updated_at"] = _now()
if wiki_parent_directories is not UNSET:
row["wiki_parent_directories"] = serialize_wiki_parent_directories(
cast(list[str] | None, wiki_parent_directories)
)
if wiki_directory_paths is not UNSET:
row["wiki_directory_paths"] = serialize_wiki_directory_paths(
cast(list[list[str]] | None, wiki_directory_paths)
)
if wiki_focus_instructions is not UNSET:
row["wiki_focus_instructions"] = cast(
str | None,
wiki_focus_instructions,
)
if wiki_content_instructions_raw is not UNSET:
row["wiki_content_instructions_raw"] = cast(
str | None,
wiki_content_instructions_raw,
)
if wiki_extraction_instructions_raw is not UNSET:
row["wiki_extraction_instructions_raw"] = cast(
str | None,
wiki_extraction_instructions_raw,
)
if wiki_generation_managed is not UNSET:
row["wiki_generation_managed"] = cast(
bool | None,
wiki_generation_managed,
)
if wiki_config_schema_version is not UNSET:
row["wiki_config_schema_version"] = cast(
int | None,
wiki_config_schema_version,
)
if wiki_config_revision is not UNSET:
row["wiki_config_revision"] = cast(
int | None,
wiki_config_revision,
)
if wiki_config_apply_status is not UNSET:
row["wiki_config_apply_status"] = cast(
str | None,
wiki_config_apply_status,
)
if wiki_config_apply_error is not UNSET:
row["wiki_config_apply_error"] = cast(
str | None,
wiki_config_apply_error,
)
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def adopt_conversation_deposit_mapping(
self,
@@ -115,7 +236,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
"updated_at": now,
}
)
return dict(row)
return _mapping_to_dict(row)
async def delete_mapping(self, mapping_id: str) -> bool:
return self._rows.pop(mapping_id, None) is not None
@@ -129,7 +250,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["reviewed_by"] = None
row["published_at"] = _now()
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
@@ -139,7 +260,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["reviewed_by"] = reviewer_user_id
row["published_at"] = _now() if approved else None
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
@@ -149,18 +270,18 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["reviewed_by"] = actor_user_id if is_admin else row.get("reviewed_by")
row["published_at"] = None
row["updated_at"] = _now()
return dict(row)
return _mapping_to_dict(row)
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
rows = [row for row in self._rows.values() if row["publication_status"] != "private"]
if status:
rows = [row for row in rows if row["publication_status"] == status]
return [dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
return [_mapping_to_dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
for row in self._rows.values():
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
return dict(row)
return _mapping_to_dict(row)
return None
async def get_conversation_deposit(
@@ -4,11 +4,11 @@ from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class LlmWikiKnowledgeBaseRow(Base):
@@ -19,6 +19,21 @@ class LlmWikiKnowledgeBaseRow(Base):
owner_user_id: Mapped[str] = mapped_column(String(64), index=True)
name: Mapped[str] = mapped_column(String(255), default="")
description: Mapped[str] = mapped_column(String(2048), default="")
# ``None`` means "use the product defaults derived from the knowledge-base
# name". An explicit empty list is meaningful: the owner disabled starter
# questions for this library.
test_questions: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True)
retrieval_settings: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
wiki_parent_directories: Mapped[list[str] | None] = mapped_column(PortableJSON(), nullable=True)
wiki_directory_paths: Mapped[list[list[str]] | None] = mapped_column(PortableJSON(), nullable=True)
wiki_focus_instructions: Mapped[str | None] = mapped_column(Text, nullable=True)
wiki_content_instructions_raw: Mapped[str | None] = mapped_column(Text, nullable=True)
wiki_extraction_instructions_raw: Mapped[str | None] = mapped_column(Text, nullable=True)
wiki_generation_managed: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
wiki_config_schema_version: Mapped[int | None] = mapped_column(Integer, nullable=True)
wiki_config_revision: Mapped[int | None] = mapped_column(Integer, nullable=True)
wiki_config_apply_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
wiki_config_apply_error: Mapped[str | None] = mapped_column(Text, nullable=True)
kb_type: Mapped[str] = mapped_column(String(32), default="document")
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
@@ -3,13 +3,19 @@
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from typing import Any, cast
from uuid import uuid4
from sqlalchemy import delete, or_, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
from deerflow.persistence.llmwiki.base import (
UNSET,
LlmWikiScope,
LlmWikiStore,
serialize_wiki_directory_paths,
serialize_wiki_parent_directories,
)
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
@@ -19,6 +25,14 @@ def _now() -> datetime:
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
data = row.to_dict()
if "wiki_parent_directories" in data:
data["wiki_parent_directories"] = serialize_wiki_parent_directories(
data["wiki_parent_directories"]
)
if "wiki_directory_paths" in data:
data["wiki_directory_paths"] = serialize_wiki_directory_paths(
data["wiki_directory_paths"]
)
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
@@ -30,13 +44,51 @@ class LlmWikiRepository(LlmWikiStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
async def create_mapping(
self,
*,
weknora_id: str,
owner_user_id: str,
name: str,
description: str,
kb_type: str,
retrieval_settings: dict[str, Any] | None = None,
wiki_parent_directories: list[str] | None = None,
wiki_directory_paths: list[list[str]] | None = None,
wiki_focus_instructions: str | None = None,
wiki_content_instructions_raw: str | None = None,
wiki_extraction_instructions_raw: str | None = None,
wiki_generation_managed: bool | None = None,
wiki_config_schema_version: int | None = None,
wiki_config_revision: int | None = None,
wiki_config_apply_status: str | None = None,
wiki_config_apply_error: str | None = None,
) -> dict[str, Any]:
row = LlmWikiKnowledgeBaseRow(
id=str(uuid4()),
weknora_id=weknora_id,
owner_user_id=owner_user_id,
name=name,
description=description,
retrieval_settings=(
dict(retrieval_settings)
if retrieval_settings is not None
else None
),
wiki_parent_directories=serialize_wiki_parent_directories(
wiki_parent_directories
),
wiki_directory_paths=serialize_wiki_directory_paths(
wiki_directory_paths
),
wiki_focus_instructions=wiki_focus_instructions,
wiki_content_instructions_raw=wiki_content_instructions_raw,
wiki_extraction_instructions_raw=wiki_extraction_instructions_raw,
wiki_generation_managed=wiki_generation_managed,
wiki_config_schema_version=wiki_config_schema_version,
wiki_config_revision=wiki_config_revision,
wiki_config_apply_status=wiki_config_apply_status,
wiki_config_apply_error=wiki_config_apply_error,
kb_type=kb_type,
publication_status="private",
created_at=_now(),
@@ -88,8 +140,21 @@ class LlmWikiRepository(LlmWikiStore):
*,
name: str | None = None,
description: str | None = None,
test_questions: list[str] | None = None,
test_questions_set: bool = False,
retrieval_settings: dict[str, Any] | None = None,
wiki_index_enabled: bool | None = None,
external_search_enabled: bool | None = None,
wiki_parent_directories: list[str] | None | object = UNSET,
wiki_directory_paths: list[list[str]] | None | object = UNSET,
wiki_focus_instructions: str | None | object = UNSET,
wiki_content_instructions_raw: str | None | object = UNSET,
wiki_extraction_instructions_raw: str | None | object = UNSET,
wiki_generation_managed: bool | None | object = UNSET,
wiki_config_schema_version: int | None | object = UNSET,
wiki_config_revision: int | None | object = UNSET,
wiki_config_apply_status: str | None | object = UNSET,
wiki_config_apply_error: str | None | object = UNSET,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
@@ -99,11 +164,63 @@ class LlmWikiRepository(LlmWikiStore):
row.name = name
if description is not None:
row.description = description
if test_questions_set:
row.test_questions = list(test_questions or [])
if retrieval_settings is not None:
row.retrieval_settings = dict(retrieval_settings)
if wiki_index_enabled is not None:
row.wiki_index_enabled = wiki_index_enabled
if external_search_enabled is not None:
row.external_search_enabled = external_search_enabled
row.external_search_updated_at = _now()
if wiki_parent_directories is not UNSET:
row.wiki_parent_directories = serialize_wiki_parent_directories(
cast(list[str] | None, wiki_parent_directories)
)
if wiki_directory_paths is not UNSET:
row.wiki_directory_paths = serialize_wiki_directory_paths(
cast(list[list[str]] | None, wiki_directory_paths)
)
if wiki_focus_instructions is not UNSET:
row.wiki_focus_instructions = cast(
str | None,
wiki_focus_instructions,
)
if wiki_content_instructions_raw is not UNSET:
row.wiki_content_instructions_raw = cast(
str | None,
wiki_content_instructions_raw,
)
if wiki_extraction_instructions_raw is not UNSET:
row.wiki_extraction_instructions_raw = cast(
str | None,
wiki_extraction_instructions_raw,
)
if wiki_generation_managed is not UNSET:
row.wiki_generation_managed = cast(
bool | None,
wiki_generation_managed,
)
if wiki_config_schema_version is not UNSET:
row.wiki_config_schema_version = cast(
int | None,
wiki_config_schema_version,
)
if wiki_config_revision is not UNSET:
row.wiki_config_revision = cast(
int | None,
wiki_config_revision,
)
if wiki_config_apply_status is not UNSET:
row.wiki_config_apply_status = cast(
str | None,
wiki_config_apply_status,
)
if wiki_config_apply_error is not UNSET:
row.wiki_config_apply_error = cast(
str | None,
wiki_config_apply_error,
)
row.updated_at = _now()
await session.commit()
return _to_dict(row)
@@ -2,16 +2,17 @@
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.model import (
LlmWikiWikiEdgeRow,
LlmWikiWikiPageRow,
LlmWikiWikiSyncStateRow,
LlmWikiWikiVectorRow,
)
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
return (
MemoryLlmWikiIndexStore()
if session_factory is None
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
)
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
__all__ = [
@@ -19,6 +20,7 @@ __all__ = [
"MemoryLlmWikiIndexStore",
"SqlLlmWikiIndexStore",
"LlmWikiWikiPageRow",
"LlmWikiWikiEdgeRow",
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"make_llmwiki_index_store",
@@ -14,8 +14,102 @@ class LlmWikiIndexStore(ABC):
@abstractmethod
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
async def search_pages_by_keywords(
self,
mapping_ids: list[str],
tokens: list[str],
*,
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
"""Return local Wiki body pages containing any token.
The default keeps lightweight test doubles and third-party stores
backward compatible. Database and in-memory stores override it.
"""
return []
async def list_pages(
self,
mapping_ids: list[str],
*,
limit: int = 500,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
"""Return a bounded, newest-first Wiki page snapshot.
Smart-reasoning regex search applies its expression in Python so the
behavior stays identical on SQLite, MySQL and GaussDB. Third-party
stores may keep the empty default; the tool then uses the remote Wiki
API fallback.
"""
return []
async def replace_graph_edges(
self,
mapping_id: str,
edges: list[dict[str, Any]],
*,
owner: str,
sync_id: str,
) -> dict[str, Any] | None:
"""Replace one mapping's directed Wiki graph snapshot.
The default keeps lightweight third-party/test stores compatible. An
implementation must resolve both endpoints inside ``mapping_id`` and
discard dangling or cross-mapping edges. ``None`` means the caller no
longer owns the supplied sync lease and no mutation was made.
"""
return {"edge_count": 0, "changed": False}
async def get_adjacent_pages(
self,
mapping_ids: list[str],
seed_slugs: dict[str, list[str]] | list[str],
*,
direction: str = "both",
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
"""Return one-hop neighbors within the explicitly allowed mappings.
Passing a ``{mapping_id: [slug, ...]}`` mapping is preferred because it
retains the exact seed provenance. A flat slug list is applied only to
the supplied ``mapping_ids`` and never broadens authorization.
"""
return []
@abstractmethod
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
async def touch_page(
self,
page_id: str,
sync_id: str,
*,
mapping_id: str | None = None,
owner: str | None = None,
) -> bool: ...
@abstractmethod
async def refresh_page_metadata(
self,
*,
page: dict[str, Any],
sync_id: str,
owner: str,
) -> dict[str, Any] | None:
"""Refresh a mirrored page without replacing its vectors.
The write is fenced by the mapping sync lease. ``None`` means the
caller no longer owns that lease (or the expected page disappeared),
while the returned ``changed`` flag reports whether durable page
metadata changed. The page is always marked as seen by ``sync_id``.
"""
...
@abstractmethod
async def replace_page_vectors(
@@ -25,13 +119,34 @@ class LlmWikiIndexStore(ABC):
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
) -> dict[str, Any]: ...
owner: str | None = None,
) -> dict[str, Any] | None: ...
@abstractmethod
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
async def record_page_failure(
self,
*,
page: dict[str, Any],
sync_id: str,
error: str,
owner: str | None = None,
) -> bool: ...
@abstractmethod
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
async def mark_missing_pages_deleted(
self,
mapping_id: str,
sync_id: str,
*,
owner: str,
) -> int | None:
"""Retire pages absent from a fenced full scan.
``None`` means the supplied owner/sync pair no longer owns the mapping
lease and no mutation was made.
"""
...
@abstractmethod
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
@@ -73,7 +188,17 @@ class LlmWikiIndexStore(ABC):
) -> bool: ...
@abstractmethod
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
async def finish_sync(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
success: bool,
remote_page_count: int,
error: str | None = None,
result: dict[str, Any] | None = None,
) -> bool: ...
@abstractmethod
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
@@ -17,11 +17,28 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self) -> None:
self.pages: dict[str, dict[str, Any]] = {}
self.vectors: dict[str, list[dict[str, Any]]] = {}
self.edges: dict[str, list[dict[str, Any]]] = {}
self.states: dict[str, dict[str, Any]] = {}
def _state(self, mapping_id: str) -> dict[str, Any]:
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
def _has_current_sync_lease(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
) -> bool:
state = self.states.get(mapping_id)
return bool(
state is not None
and state.get("lease_owner") == owner
and state.get("active_sync_id") == sync_id
and isinstance(state.get("lease_expires_at"), datetime)
and state["lease_expires_at"] >= _now()
)
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
@@ -30,12 +47,241 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
return dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
if page_id in self.pages:
self.pages[page_id]["last_seen_sync_id"] = sync_id
self.pages[page_id]["last_error"] = None
async def search_pages_by_keywords(
self,
mapping_ids: list[str],
tokens: list[str],
*,
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
wanted = set(mapping_ids)
needles = [str(item).strip().lower() for item in tokens if str(item).strip()]
rows = []
for page in self.pages.values():
if page.get("knowledge_base_mapping_id") not in wanted or page.get("is_remote_deleted"):
continue
if page.get("status") == "draft" and not include_drafts:
continue
if page.get("status") not in {"published", "draft"}:
continue
haystack = " ".join(str(page.get(key) or "") for key in ("title", "summary", "content_md")).lower()
if needles and any(needle in haystack for needle in needles):
rows.append(dict(page))
if len(rows) >= max(1, limit):
break
return rows
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
async def list_pages(
self,
mapping_ids: list[str],
*,
limit: int = 500,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
wanted = {str(item).strip() for item in mapping_ids if str(item).strip()}
cap = max(1, min(int(limit), 2000))
rows = [
dict(page)
for page in self.pages.values()
if page.get("knowledge_base_mapping_id") in wanted
and not page.get("is_remote_deleted")
and page.get("status") in ({"published", "draft"} if include_drafts else {"published"})
]
rows.sort(
key=lambda page: str(page.get("remote_updated_at") or page.get("updated_at") or ""),
reverse=True,
)
return rows[:cap]
async def replace_graph_edges(
self,
mapping_id: str,
edges: list[dict[str, Any]],
*,
owner: str,
sync_id: str,
) -> dict[str, Any] | None:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return None
pages_by_slug = {str(page.get("slug") or ""): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") == mapping_id and not page.get("is_remote_deleted") and page.get("status") in {"published", "draft"}}
canonical = {
(
str(edge.get("source_slug") or "").strip(),
str(edge.get("target_slug") or "").strip(),
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
)
for edge in edges
if isinstance(edge, dict)
}
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
replacement = [
{
"knowledge_base_mapping_id": mapping_id,
"source_page_id": pages_by_slug[source]["id"],
"target_page_id": pages_by_slug[target]["id"],
"source_slug": source,
"target_slug": target,
"edge_type": edge_type,
}
for source, target, edge_type in sorted(canonical)
]
current = self.edges.get(mapping_id, [])
current_keys = {(row["source_slug"], row["target_slug"], row["edge_type"]) for row in current}
changed = current_keys != canonical
if changed:
self.edges[mapping_id] = replacement
self._state(mapping_id)["index_revision"] += 1
return {"edge_count": len(replacement), "changed": changed}
async def get_adjacent_pages(
self,
mapping_ids: list[str],
seed_slugs: dict[str, list[str]] | list[str],
*,
direction: str = "both",
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
allowed = set(str(item).strip() for item in mapping_ids if str(item).strip())
if direction not in {"incoming", "outgoing", "both"}:
raise ValueError("direction must be incoming, outgoing, or both")
if isinstance(seed_slugs, dict):
seeds = {mapping_id: {str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip()} for mapping_id in allowed}
else:
shared = {str(slug).strip() for slug in seed_slugs if str(slug).strip()}
seeds = {mapping_id: shared for mapping_id in allowed}
pages_by_id = {str(page["id"]): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") in allowed}
result: list[dict[str, Any]] = []
seen: set[tuple[str, str, str, str]] = set()
for mapping_id in sorted(allowed):
mapping_seeds = seeds.get(mapping_id) or set()
for edge in self.edges.get(mapping_id, []):
traversals: list[tuple[str, str, str, str]] = []
if direction in {"outgoing", "both"} and edge["source_slug"] in mapping_seeds:
traversals.append(
(
edge["source_slug"],
edge["target_slug"],
"outgoing",
edge["target_page_id"],
)
)
if direction in {"incoming", "both"} and edge["target_slug"] in mapping_seeds:
traversals.append(
(
edge["target_slug"],
edge["source_slug"],
"incoming",
edge["source_page_id"],
)
)
for seed_slug, neighbor_slug, traversal_direction, page_id in traversals:
page = pages_by_id.get(page_id)
if not page or page.get("is_remote_deleted"):
continue
if page.get("status") == "draft" and not include_drafts:
continue
if page.get("status") not in {"published", "draft"}:
continue
key = (mapping_id, seed_slug, neighbor_slug, traversal_direction)
if key in seen:
continue
seen.add(key)
result.append(
{
"knowledge_base_mapping_id": mapping_id,
"seed_slug": seed_slug,
"neighbor_slug": neighbor_slug,
"direction": traversal_direction,
"edge_type": edge["edge_type"],
"source_slug": edge["source_slug"],
"target_slug": edge["target_slug"],
"page": dict(page),
}
)
result.sort(
key=lambda row: (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
)
return result[: max(1, min(int(limit), 1000))]
async def touch_page(
self,
page_id: str,
sync_id: str,
*,
mapping_id: str | None = None,
owner: str | None = None,
) -> bool:
page = self.pages.get(page_id)
if page is None:
return False
actual_mapping_id = str(page.get("knowledge_base_mapping_id") or "")
if mapping_id is not None and mapping_id != actual_mapping_id:
return False
if owner is not None and not self._has_current_sync_lease(
actual_mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
page["last_seen_sync_id"] = sync_id
page["last_error"] = None
return True
async def refresh_page_metadata(
self,
*,
page: dict[str, Any],
sync_id: str,
owner: str,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
if not self._has_current_sync_lease(
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
existing = await self.get_page_by_slug_hash(mapping_id, str(page["slug_hash"]))
if existing is None:
return None
durable_changed = any(existing.get(key) != value for key, value in page.items())
row = self.pages[str(existing["id"])]
row.update(page)
row.update(
{
"last_seen_sync_id": sync_id,
"last_error": None,
"is_remote_deleted": False,
"updated_at": _now().isoformat(),
}
)
if durable_changed:
self._state(mapping_id)["index_revision"] += 1
return {"changed": durable_changed, "page": dict(row)}
async def replace_page_vectors(
self,
*,
page: dict[str, Any],
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
owner: str | None = None,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
if owner is not None and not self._has_current_sync_lease(
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
page_id = existing["id"] if existing else str(uuid4())
now = _now().isoformat()
@@ -59,7 +305,21 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
state["embedding_fingerprint"] = fingerprint
return dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
async def record_page_failure(
self,
*,
page: dict[str, Any],
sync_id: str,
error: str,
owner: str | None = None,
) -> bool:
mapping_id = str(page["knowledge_base_mapping_id"])
if owner is not None and not self._has_current_sync_lease(
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
if existing:
self.pages[existing["id"]].update(
@@ -70,11 +330,20 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
}
)
return
return True
page_id = str(uuid4())
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
return True
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
async def mark_missing_pages_deleted(
self,
mapping_id: str,
sync_id: str,
*,
owner: str,
) -> int | None:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return None
changed = 0
for page in self.pages.values():
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
@@ -82,6 +351,8 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
self.vectors.pop(page["id"], None)
changed += 1
if changed:
active_page_ids = {page["id"] for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")}
self.edges[mapping_id] = [edge for edge in self.edges.get(mapping_id, []) if edge["source_page_id"] in active_page_ids and edge["target_page_id"] in active_page_ids]
self._state(mapping_id)["index_revision"] += 1
return changed
@@ -131,17 +402,27 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
sync_id: str,
remote_page_count: int,
) -> bool:
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return False
state = self._state(mapping_id)
state["remote_page_count"] = max(0, remote_page_count)
state["updated_at"] = _now()
return True
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async def finish_sync(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
success: bool,
remote_page_count: int,
error: str | None = None,
result: dict[str, Any] | None = None,
) -> bool:
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
return False
state = self._state(mapping_id)
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
return
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
was_rebuilding = state.get("state") == "rebuilding"
state.update(
@@ -158,8 +439,10 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
"last_success_at": _now() if success else state.get("last_success_at"),
"last_sync_result": dict(result) if result is not None else state.get("last_sync_result"),
}
)
return True
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
wanted = set(mapping_ids or [])
@@ -168,11 +451,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
result = []
for page in sorted(
(
page
for page in self.pages.values()
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
),
(page for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")),
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
):
result.append(
@@ -72,6 +72,53 @@ class LlmWikiWikiVectorRow(Base):
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
class LlmWikiWikiEdgeRow(Base):
"""One directed, mapping-scoped link between two mirrored Wiki pages."""
__tablename__ = "llmwiki_wiki_edges"
__table_args__ = (
UniqueConstraint(
"knowledge_base_mapping_id",
"source_page_id",
"target_page_id",
"edge_type",
name="uq_llmwiki_wiki_edge_mapping_pages_type",
),
Index(
"ix_llmwiki_wiki_edge_mapping_source",
"knowledge_base_mapping_id",
"source_page_id",
),
Index(
"ix_llmwiki_wiki_edge_mapping_target",
"knowledge_base_mapping_id",
"target_page_id",
),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
knowledge_base_mapping_id: Mapped[str] = mapped_column(
String(36),
ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
source_page_id: Mapped[str] = mapped_column(
String(36),
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
target_page_id: Mapped[str] = mapped_column(
String(36),
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
edge_type: Mapped[str] = mapped_column(String(32), nullable=False, default="wiki_link")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
class LlmWikiWikiSyncStateRow(Base):
__tablename__ = "llmwiki_wiki_sync_states"
@@ -91,4 +138,5 @@ class LlmWikiWikiSyncStateRow(Base):
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_sync_result: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
@@ -7,12 +7,18 @@ from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import case, delete, func, insert, or_, select, update
from sqlalchemy import and_, case, delete, func, insert, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlalchemy.orm import aliased
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.model import (
LlmWikiWikiEdgeRow,
LlmWikiWikiPageRow,
LlmWikiWikiSyncStateRow,
LlmWikiWikiVectorRow,
)
def _now() -> datetime:
@@ -40,6 +46,32 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
await session.flush()
return row
async def _fence_current_sync_lease(
self,
session: AsyncSession,
mapping_id: str,
*,
owner: str,
sync_id: str,
) -> bool:
"""Fence destructive snapshot writes behind the current live lease."""
# Use a conditional UPDATE rather than SELECT ... FOR UPDATE. SQLite
# ignores FOR UPDATE, while this statement obtains its write lock on
# every supported backend before the page/edge mutation starts.
stmt = (
update(LlmWikiWikiSyncStateRow)
.where(
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiSyncStateRow.lease_owner == owner,
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
)
.values(updated_at=_now())
)
return bool((await session.execute(stmt)).rowcount)
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
@@ -53,18 +85,351 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
async def touch_page(self, page_id: str, sync_id: str) -> None:
async def search_pages_by_keywords(
self,
mapping_ids: list[str],
tokens: list[str],
*,
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
normalized_ids = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
normalized_tokens = list(dict.fromkeys(str(item).strip().lower() for item in tokens if str(item).strip()))
if not normalized_ids or not normalized_tokens:
return []
conditions = []
for token in normalized_tokens[:32]:
needle = f"%{token}%"
conditions.extend(
(
func.lower(LlmWikiWikiPageRow.title).like(needle),
func.lower(LlmWikiWikiPageRow.summary).like(needle),
func.lower(LlmWikiWikiPageRow.content_md).like(needle),
)
)
statuses = ["published", "draft"] if include_drafts else ["published"]
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(statuses),
func.length(func.trim(LlmWikiWikiPageRow.content_md)) > 0,
or_(*conditions),
)
.order_by(LlmWikiWikiPageRow.updated_at.desc())
.limit(max(1, min(int(limit), 2000)))
)
async with self._sf() as session:
await session.execute(
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def list_pages(
self,
mapping_ids: list[str],
*,
limit: int = 500,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
normalized_ids = list(
dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip())
)
if not normalized_ids:
return []
statuses = ["published", "draft"] if include_drafts else ["published"]
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(statuses),
)
.order_by(
LlmWikiWikiPageRow.remote_updated_at.desc(),
LlmWikiWikiPageRow.updated_at.desc(),
)
.limit(max(1, min(int(limit), 2000)))
)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def replace_graph_edges(
self,
mapping_id: str,
edges: list[dict[str, Any]],
*,
owner: str,
sync_id: str,
) -> dict[str, Any] | None:
"""Atomically replace an authorized mapping's resolved edge snapshot."""
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
page_stmt = select(LlmWikiWikiPageRow).where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
pages = list((await session.execute(page_stmt)).scalars())
pages_by_slug = {page.slug: page for page in pages}
canonical = {
(
str(edge.get("source_slug") or "").strip(),
str(edge.get("target_slug") or "").strip(),
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
)
for edge in edges
if isinstance(edge, dict)
}
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
existing_stmt = select(
LlmWikiWikiEdgeRow.source_page_id,
LlmWikiWikiEdgeRow.target_page_id,
LlmWikiWikiEdgeRow.edge_type,
).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id)
existing = {(str(source_id), str(target_id), str(edge_type)) for source_id, target_id, edge_type in (await session.execute(existing_stmt)).all()}
replacement_ids = {
(
pages_by_slug[source].id,
pages_by_slug[target].id,
edge_type,
)
for source, target, edge_type in canonical
}
changed = existing != replacement_ids
if changed:
await session.execute(delete(LlmWikiWikiEdgeRow).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id))
now = _now()
edge_rows = [
{
"id": str(uuid4()),
"knowledge_base_mapping_id": mapping_id,
"source_page_id": pages_by_slug[source].id,
"target_page_id": pages_by_slug[target].id,
"edge_type": edge_type,
"created_at": now,
}
for source, target, edge_type in sorted(canonical)
]
for start in range(0, len(edge_rows), self.write_batch_size):
await session.execute(
insert(LlmWikiWikiEdgeRow),
edge_rows[start : start + self.write_batch_size],
)
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return {"edge_count": len(canonical), "changed": changed}
async def get_adjacent_pages(
self,
mapping_ids: list[str],
seed_slugs: dict[str, list[str]] | list[str],
*,
direction: str = "both",
limit: int = 100,
include_drafts: bool = False,
) -> list[dict[str, Any]]:
allowed = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
if not allowed:
return []
if direction not in {"incoming", "outgoing", "both"}:
raise ValueError("direction must be incoming, outgoing, or both")
if isinstance(seed_slugs, dict):
seeds = {mapping_id: list(dict.fromkeys(str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip())) for mapping_id in allowed}
else:
shared = list(dict.fromkeys(str(slug).strip() for slug in seed_slugs if str(slug).strip()))
seeds = {mapping_id: shared for mapping_id in allowed}
seeds = {mapping_id: slugs for mapping_id, slugs in seeds.items() if slugs}
if not seeds:
return []
source_page = aliased(LlmWikiWikiPageRow)
target_page = aliased(LlmWikiWikiPageRow)
statuses = ["published", "draft"] if include_drafts else ["published"]
cap = max(1, min(int(limit), 1000))
async def _query(traversal_direction: str) -> list[dict[str, Any]]:
seed_page = source_page if traversal_direction == "outgoing" else target_page
neighbor_page = target_page if traversal_direction == "outgoing" else source_page
seed_conditions = or_(
*(
and_(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id,
seed_page.slug.in_(slugs),
)
for mapping_id, slugs in seeds.items()
)
)
stmt = (
select(LlmWikiWikiEdgeRow, source_page, target_page)
.join(source_page, source_page.id == LlmWikiWikiEdgeRow.source_page_id)
.join(target_page, target_page.id == LlmWikiWikiEdgeRow.target_page_id)
.where(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id.in_(allowed),
source_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
target_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
source_page.is_remote_deleted.is_(False),
target_page.is_remote_deleted.is_(False),
source_page.status.in_(statuses),
target_page.status.in_(statuses),
seed_conditions,
)
.order_by(
LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
seed_page.slug,
neighbor_page.slug,
)
.limit(cap)
)
async with self._sf() as session:
triples = list((await session.execute(stmt)).all())
rows: list[dict[str, Any]] = []
for edge, source, target in triples:
seed = source if traversal_direction == "outgoing" else target
neighbor = target if traversal_direction == "outgoing" else source
rows.append(
{
"knowledge_base_mapping_id": edge.knowledge_base_mapping_id,
"seed_slug": seed.slug,
"neighbor_slug": neighbor.slug,
"direction": traversal_direction,
"edge_type": edge.edge_type,
"source_slug": source.slug,
"target_slug": target.slug,
"page": _dict(neighbor),
}
)
return rows
result: list[dict[str, Any]] = []
if direction in {"outgoing", "both"}:
result.extend(await _query("outgoing"))
if direction in {"incoming", "both"}:
result.extend(await _query("incoming"))
result.sort(
key=lambda row: (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
)
deduplicated: list[dict[str, Any]] = []
seen: set[tuple[str, str, str, str]] = set()
for row in result:
key = (
row["knowledge_base_mapping_id"],
row["seed_slug"],
row["neighbor_slug"],
row["direction"],
)
if key in seen:
continue
seen.add(key)
deduplicated.append(row)
if len(deduplicated) >= cap:
break
return deduplicated
async def touch_page(
self,
page_id: str,
sync_id: str,
*,
mapping_id: str | None = None,
owner: str | None = None,
) -> bool:
async with self._sf() as session:
if owner is not None:
if not mapping_id or not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
conditions = [LlmWikiWikiPageRow.id == page_id]
if mapping_id is not None:
conditions.append(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id)
result = await session.execute(
update(LlmWikiWikiPageRow)
.where(LlmWikiWikiPageRow.id == page_id)
.where(*conditions)
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
)
await session.commit()
return bool(result.rowcount)
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
async def refresh_page_metadata(
self,
*,
page: dict[str, Any],
sync_id: str,
owner: str,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = (
select(LlmWikiWikiPageRow)
.where(
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiPageRow.slug_hash == page["slug_hash"],
)
.with_for_update()
)
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
return None
if row.slug != page["slug"]:
raise ValueError("Wiki slug hash collision detected")
durable_changed = bool(row.is_remote_deleted or row.last_error) or any(
getattr(row, key) != value for key, value in page.items()
)
for key, value in page.items():
setattr(row, key, value)
now = _now()
row.last_seen_sync_id = sync_id
row.last_error = None
row.is_remote_deleted = False
row.updated_at = now
if durable_changed:
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.updated_at = now
await session.commit()
return {"changed": durable_changed, "page": _dict(row)}
async def replace_page_vectors(
self,
*,
page: dict[str, Any],
vectors: list[dict[str, Any]],
fingerprint: str,
sync_id: str,
owner: str | None = None,
) -> dict[str, Any] | None:
mapping_id = str(page["knowledge_base_mapping_id"])
async with self._sf() as session:
if owner is not None and not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
row = (await session.execute(stmt)).scalar_one_or_none()
if row is not None and row.slug != page["slug"]:
@@ -115,10 +480,24 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
await session.commit()
return _dict(row)
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
async def record_page_failure(
self,
*,
page: dict[str, Any],
sync_id: str,
error: str,
owner: str | None = None,
) -> bool:
mapping_id = str(page["knowledge_base_mapping_id"])
safe_error = error[:4000]
async with self._sf() as session:
if owner is not None and not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
row = (await session.execute(stmt)).scalar_one_or_none()
now = _now()
@@ -136,9 +515,23 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row.updated_at = now
await self._ensure_state(session, mapping_id)
await session.commit()
return True
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
async def mark_missing_pages_deleted(
self,
mapping_id: str,
sync_id: str,
*,
owner: str,
) -> int | None:
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return None
stmt = (
select(LlmWikiWikiPageRow)
.where(
@@ -287,17 +680,36 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiSyncStateRow.lease_owner == owner,
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
)
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
)
await session.commit()
return bool(result.rowcount)
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
async def finish_sync(
self,
mapping_id: str,
*,
owner: str,
sync_id: str,
success: bool,
remote_page_count: int,
error: str | None = None,
result: dict[str, Any] | None = None,
) -> bool:
async with self._sf() as session:
if not await self._fence_current_sync_lease(
session,
mapping_id,
owner=owner,
sync_id=sync_id,
):
return False
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
return
return False
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
@@ -329,12 +741,15 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
state.ready_page_count = ready_count
state.failed_page_count = failed_count
state.vector_count = vector_count
if result is not None:
state.last_sync_result = dict(result)
if success and not error:
state.last_completed_at = now
if success:
state.last_success_at = now
state.updated_at = now
await session.commit()
return True
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
stmt = select(LlmWikiWikiSyncStateRow)
@@ -4,13 +4,14 @@ from __future__ import annotations
from deerflow.persistence.menu_overrides.base import MenuOverrideStore
from deerflow.persistence.menu_overrides.memory import MemoryMenuOverrideStore
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
from deerflow.persistence.menu_overrides.sql import MenuOverrideRepository
__all__ = [
"MemoryMenuOverrideStore",
"MenuOverrideRepository",
"MenuOverrideRow",
"UserMenuOverrideConfigRow",
"MenuOverrideStore",
"make_menu_override_store",
]
@@ -1,11 +1,11 @@
"""Abstract sidebar-menu override persistence API.
Global / shared store: no ``user_id`` scoping. Reads are open to any caller; the
router gates writes behind admin. ``updated_by`` is audit-only.
The whole override set is treated as one document: the editor computes the full
desired layout and replaces every row atomically via :meth:`replace_all`, so the
stored set never drifts out of sync with what the admin sees.
The original ``menu_overrides`` document remains the administrator's default
layout. User layouts are stored as a separate, complete snapshot so that a
user can customise the menu without changing the default for anybody else.
The router overlays a user's snapshot on top of the default and lets default
rows for node ids not present in that snapshot flow through (new administrator
menus therefore appear for existing users).
"""
from __future__ import annotations
@@ -28,3 +28,24 @@ class MenuOverrideStore(ABC):
Returns the persisted rows.
"""
raise NotImplementedError
@abstractmethod
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
"""Return one user's snapshot, or ``None`` when no snapshot exists."""
raise NotImplementedError
@abstractmethod
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
"""Replace one user's snapshot.
An empty list removes the snapshot and returns ``None``. This is the
reset-to-default operation and intentionally differs from the admin
``replace_all`` document, where an empty list means an empty default.
"""
raise NotImplementedError
@@ -23,6 +23,7 @@ def _normalize(data: dict[str, Any], updated_by: str | None) -> dict[str, Any]:
class MemoryMenuOverrideStore(MenuOverrideStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._user_rows: dict[str, list[dict[str, Any]]] = {}
async def list_overrides(self) -> list[dict[str, Any]]:
return [dict(r) for r in sorted(self._rows.values(), key=lambda r: r.get("sort_order", 0))]
@@ -35,3 +36,29 @@ class MemoryMenuOverrideStore(MenuOverrideStore):
continue
self._rows[str(node_id)] = _normalize(r, updated_by)
return [dict(r) for r in self._rows.values()]
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
rows = self._user_rows.get(str(user_id))
return [dict(r) for r in rows] if rows is not None else None
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
key = str(user_id)
if not rows:
self._user_rows.pop(key, None)
return None
by_id: dict[str, dict[str, Any]] = {}
for row in rows:
node_id = row.get("node_id")
if node_id:
by_id[str(node_id)] = _normalize(row, updated_by or key)
if not by_id:
self._user_rows.pop(key, None)
return None
self._user_rows[key] = [dict(row) for row in by_id.values()]
return [dict(row) for row in self._user_rows[key]]
@@ -1,9 +1,10 @@
"""ORM model for sidebar-menu overrides (菜单管理).
Global / shared — there is no ``user_id`` scoping; every account sees the same
menu layout. The static menu tree (icons / routes / adminOnly) lives in the
frontend code; this table only stores per-node **overrides** keyed by the static
node id (or an injected iframe light-app node id ``light-app-<appId>``):
``menu_overrides`` stores the administrator's shared default. Personal layouts
live separately in ``user_menu_override_configs`` as complete snapshots. The
static menu tree (icons / routes / adminOnly) lives in the frontend code; the
default table stores per-node **overrides** keyed by the static node id (or an
injected iframe light-app node id ``light-app-<appId>``):
- ``parent_id`` — the effective parent node id (NULL = top level). Re-parenting
a node moves its whole subtree.
@@ -23,7 +24,7 @@ from sqlalchemy import Boolean, Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
from deerflow.persistence.types import BeijingDateTime, PortableJSON
class MenuOverrideRow(Base):
@@ -40,3 +41,22 @@ class MenuOverrideRow(Base):
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
class UserMenuOverrideConfigRow(Base):
"""A complete personal sidebar-menu snapshot.
Keeping the snapshot in one row is deliberate: the presence of a row is
the ``has_user_override`` bit, while the node ids in ``overrides`` are the
user's known-node set. This lets us distinguish an administrator-added
node from an old node the user intentionally left at its default value.
"""
__tablename__ = "user_menu_override_configs"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
overrides: Mapped[list[dict]] = mapped_column(PortableJSON(), nullable=False, default=list)
updated_by: Mapped[str | None] = mapped_column(String(64))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
)
@@ -2,14 +2,14 @@
from __future__ import annotations
from datetime import datetime
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.menu_overrides.base import MenuOverrideStore
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
def _to_dict(row: MenuOverrideRow) -> dict[str, Any]:
@@ -31,6 +31,42 @@ def _row_from(data: dict[str, Any], updated_by: str | None) -> MenuOverrideRow:
)
def _normalize_user_row(data: dict[str, Any]) -> dict[str, Any]:
"""Keep only the public override fields in the personal JSON document."""
return {
"node_id": str(data["node_id"]),
"parent_id": (str(data["parent_id"]) if data.get("parent_id") not in (None, "") else None),
"sort_order": int(data.get("sort_order", 0)),
"label": (data.get("label") or None),
"disabled": bool(data.get("disabled", False)),
}
def _user_rows(row: UserMenuOverrideConfigRow) -> list[dict[str, Any]]:
value = row.overrides
if isinstance(value, str):
# PortableJSON handles MySQL automatically, but this keeps old rows
# readable if a driver returned the raw TEXT value.
import json
try:
value = json.loads(value)
except (TypeError, ValueError):
value = []
if not isinstance(value, list):
value = []
updated_at = row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at
return [
{
**_normalize_user_row(item),
"updated_by": row.updated_by,
"updated_at": updated_at,
}
for item in value
if isinstance(item, dict) and item.get("node_id")
]
class MenuOverrideRepository(MenuOverrideStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -56,3 +92,49 @@ class MenuOverrideRepository(MenuOverrideStore):
for row in new_rows:
await session.refresh(row)
return [_to_dict(row) for row in new_rows]
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
async with self._sf() as session:
row = await session.get(UserMenuOverrideConfigRow, str(user_id))
return _user_rows(row) if row is not None else None
async def replace_user_overrides(
self,
user_id: str,
rows: list[dict[str, Any]],
*,
updated_by: str | None = None,
) -> list[dict[str, Any]] | None:
key = str(user_id)
by_id: dict[str, dict[str, Any]] = {}
for data in rows:
node_id = data.get("node_id")
if node_id:
normalized = _normalize_user_row(data)
by_id[normalized["node_id"]] = normalized
async with self._sf() as session:
row = await session.get(UserMenuOverrideConfigRow, key)
if not by_id:
if row is not None:
await session.delete(row)
await session.commit()
return None
now = datetime.now(UTC)
payload = list(by_id.values())
if row is None:
row = UserMenuOverrideConfigRow(
user_id=key,
overrides=payload,
updated_by=updated_by or key,
)
session.add(row)
else:
row.overrides = payload
row.updated_by = updated_by or key
row.updated_at = now
await session.commit()
# The shared session factory uses ``expire_on_commit=False``. Build
# the response from the writer-side instance instead of issuing a
# post-commit refresh that a read/write-splitting endpoint could
# route to a lagging replica.
return _user_rows(row)
@@ -54,7 +54,9 @@ def do_run_migrations(connection):
# = 1 year; innodb_lock_wait_timeout = 50s). Without this, an idle
# transaction on another connection can freeze startup indefinitely.
# Tunable via DEER_FLOW_ALEMBIC_LOCK_WAIT (seconds, default 15).
if connection.dialect.name == "mysql":
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
if connection.dialect.name == "mysql" and not is_gauss_mysql_product():
try:
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
except ValueError:
@@ -0,0 +1,21 @@
"""Merge assistant-wiki metadata and report-collaboration heads.
Revision ID: 20260909_01
Revises: 20260903_01, 20260907_01
Create Date: 2026-09-09
"""
from __future__ import annotations
revision = "20260909_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -0,0 +1,74 @@
"""Create durable public-document library tables.
Revision ID: 20260914_01
Revises: 20260903_01, 20260907_01
Create Date: 2026-09-14
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260914_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("gongwen_documents"):
op.create_table(
"gongwen_documents",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("user_id", sa.String(length=64), nullable=True),
sa.Column("title", sa.String(length=256), nullable=False, server_default=""),
sa.Column("document_type", sa.String(length=48), nullable=False, server_default="notice"),
sa.Column("source", sa.String(length=24), nullable=False, server_default="upload"),
sa.Column("status", sa.String(length=32), nullable=False, server_default="checking"),
sa.Column("template_key", sa.String(length=96), nullable=True),
sa.Column("research_session_id", sa.String(length=64), nullable=True),
sa.Column("original_version_id", sa.String(length=64), nullable=True),
sa.Column("latest_version_id", sa.String(length=64), nullable=True),
sa.Column("issue_counts", sa.Text(), nullable=True),
sa.Column("error", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_gongwen_documents_user_id", "gongwen_documents", ["user_id"])
op.create_index("ix_gongwen_documents_document_type", "gongwen_documents", ["document_type"])
op.create_index("ix_gongwen_documents_source", "gongwen_documents", ["source"])
op.create_index("ix_gongwen_documents_status", "gongwen_documents", ["status"])
op.create_index("ix_gongwen_documents_research_session_id", "gongwen_documents", ["research_session_id"])
if not inspector.has_table("gongwen_document_versions"):
op.create_table(
"gongwen_document_versions",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("document_id", sa.String(length=64), nullable=False),
sa.Column("version_no", sa.Integer(), nullable=False),
sa.Column("kind", sa.String(length=32), nullable=False),
sa.Column("file_name", sa.String(length=512), nullable=False),
sa.Column("storage_key", sa.String(length=1024), nullable=False),
sa.Column("content_type", sa.String(length=128), nullable=False, server_default="application/octet-stream"),
sa.Column("size_bytes", sa.Integer(), nullable=False, server_default="0"),
sa.Column("sha256", sa.String(length=64), nullable=False, server_default=""),
sa.Column("review_snapshot", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
)
op.create_index("ix_gongwen_document_versions_document_id", "gongwen_document_versions", ["document_id"])
op.create_index(
"ix_gongwen_document_versions_document_created",
"gongwen_document_versions",
["document_id", "created_at"],
)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
if inspector.has_table("gongwen_document_versions"):
op.drop_table("gongwen_document_versions")
if inspector.has_table("gongwen_documents"):
op.drop_table("gongwen_documents")
@@ -0,0 +1,88 @@
"""Persist mapping-scoped links for local LLMWiki graph expansion.
Revision ID: 20260916_01
Revises: 20260903_01, 20260907_01
Create Date: 2026-09-16
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260916_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if inspector.has_table("llmwiki_wiki_edges"):
return
op.create_table(
"llmwiki_wiki_edges",
sa.Column("id", sa.String(length=36), primary_key=True),
sa.Column(
"knowledge_base_mapping_id",
sa.String(length=36),
sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"source_page_id",
sa.String(length=36),
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"target_page_id",
sa.String(length=36),
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
nullable=False,
),
sa.Column(
"edge_type",
sa.String(length=32),
nullable=False,
server_default="wiki_link",
),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint(
"knowledge_base_mapping_id",
"source_page_id",
"target_page_id",
"edge_type",
name="uq_llmwiki_wiki_edge_mapping_pages_type",
),
)
op.create_index(
"ix_llmwiki_wiki_edges_knowledge_base_mapping_id",
"llmwiki_wiki_edges",
["knowledge_base_mapping_id"],
)
op.create_index(
"ix_llmwiki_wiki_edges_source_page_id",
"llmwiki_wiki_edges",
["source_page_id"],
)
op.create_index(
"ix_llmwiki_wiki_edges_target_page_id",
"llmwiki_wiki_edges",
["target_page_id"],
)
op.create_index(
"ix_llmwiki_wiki_edge_mapping_source",
"llmwiki_wiki_edges",
["knowledge_base_mapping_id", "source_page_id"],
)
op.create_index(
"ix_llmwiki_wiki_edge_mapping_target",
"llmwiki_wiki_edges",
["knowledge_base_mapping_id", "target_page_id"],
)
def downgrade() -> None:
if sa.inspect(op.get_bind()).has_table("llmwiki_wiki_edges"):
op.drop_table("llmwiki_wiki_edges")
@@ -0,0 +1,240 @@
"""Add answer-level feedback reasons and agent score dimensions.
Revision ID: 20260916_02
Revises: 20260916_01
Create Date: 2026-09-16
"""
from __future__ import annotations
import hashlib
import sqlalchemy as sa
from alembic import op
revision = "20260916_02"
down_revision = "20260916_01"
branch_labels = None
depends_on = None
_TABLE = "feedback"
_OLD_INDEX = "uq_feedback_thread_run_user"
_TARGET_INDEX = "uq_feedback_target_key"
_AGENT_INDEX = "ix_feedback_agent_created"
def _columns() -> set[str]:
return {
str(column["name"])
for column in sa.inspect(op.get_bind()).get_columns(_TABLE)
}
def _indexes() -> set[str]:
return {
str(index["name"])
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE)
if index.get("name")
}
def _json_type() -> sa.types.TypeEngine:
return sa.Text() if op.get_bind().dialect.name == "mysql" else sa.JSON()
def _empty_reasons_value() -> str | list[str]:
# PortableJSON is TEXT on MySQL, where binding a Python list directly to
# a raw migration Text column is invalid. SQLite/PostgreSQL JSON columns
# should receive the native list so their type processors serialize it.
return "[]" if op.get_bind().dialect.name == "mysql" else []
def _target_key(
*,
user_id: str | None,
thread_id: str,
run_id: str,
message_id: str | None,
agent_id: str,
) -> str:
message_key = message_id or f"run:{run_id}"
raw = "\x1f".join(
(user_id or "", thread_id, message_key, agent_id or "lead_agent")
)
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
def _backfill() -> None:
bind = op.get_bind()
feedback = sa.table(
_TABLE,
sa.column("feedback_id", sa.String(64)),
sa.column("run_id", sa.String(64)),
sa.column("thread_id", sa.String(64)),
sa.column("user_id", sa.String(64)),
sa.column("message_id", sa.String(64)),
sa.column("agent_id", sa.String(128)),
sa.column("target_key", sa.String(64)),
sa.column("reasons", _json_type()),
sa.column("created_at", sa.DateTime(timezone=True)),
sa.column("updated_at", sa.DateTime(timezone=True)),
)
rows = list(
bind.execute(
sa.select(
feedback.c.feedback_id,
feedback.c.run_id,
feedback.c.thread_id,
feedback.c.user_id,
feedback.c.message_id,
feedback.c.agent_id,
feedback.c.created_at,
)
).mappings()
)
run_agents: dict[str, str] = {}
inspector = sa.inspect(bind)
if rows and inspector.has_table("runs"):
run_columns = {str(column["name"]) for column in inspector.get_columns("runs")}
if "assistant_id" in run_columns:
runs = sa.table(
"runs",
sa.column("run_id", sa.String(64)),
sa.column("assistant_id", sa.String(128)),
)
run_ids = sorted({str(row["run_id"]) for row in rows})
run_agents = {
str(row.run_id): str(row.assistant_id)
for row in bind.execute(
sa.select(runs.c.run_id, runs.c.assistant_id).where(
runs.c.run_id.in_(run_ids)
)
)
if row.assistant_id
}
seen_target_keys: set[str] = set()
for row in rows:
agent_id = str(
run_agents.get(str(row["run_id"]))
or row.get("agent_id")
or "lead_agent"
)[:128]
target_key = _target_key(
user_id=row.get("user_id"),
thread_id=str(row["thread_id"]),
run_id=str(row["run_id"]),
message_id=row.get("message_id"),
agent_id=agent_id,
)
# SQL unique indexes historically allowed several NULL-user rows.
# Keep those legacy records without blocking the new unique digest;
# one canonical row retains the deterministic future-upsert key.
if target_key in seen_target_keys:
legacy_raw = f"{target_key}\x1flegacy:{row['feedback_id']}"
target_key = hashlib.sha256(legacy_raw.encode("utf-8")).hexdigest()
else:
seen_target_keys.add(target_key)
bind.execute(
sa.update(feedback)
.where(feedback.c.feedback_id == row["feedback_id"])
.values(
agent_id=agent_id,
target_key=target_key,
reasons=_empty_reasons_value(),
updated_at=row.get("created_at"),
)
)
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TABLE):
return
columns = _columns()
if "agent_id" not in columns:
op.add_column(
_TABLE,
sa.Column(
"agent_id",
sa.String(length=128),
nullable=True,
server_default="lead_agent",
),
)
if "target_key" not in columns:
op.add_column(
_TABLE,
sa.Column("target_key", sa.String(length=64), nullable=True),
)
if "reasons" not in columns:
op.add_column(
_TABLE,
sa.Column("reasons", _json_type(), nullable=True),
)
if "updated_at" not in columns:
op.add_column(
_TABLE,
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
)
_backfill()
# SQLite needs a batch table rebuild for nullability changes; Alembic uses
# native ALTER statements on MySQL/PostgreSQL where possible.
with op.batch_alter_table(_TABLE) as batch:
batch.alter_column(
"agent_id",
existing_type=sa.String(length=128),
nullable=False,
server_default=None,
)
batch.alter_column(
"target_key",
existing_type=sa.String(length=64),
nullable=False,
)
batch.alter_column(
"reasons",
existing_type=_json_type(),
nullable=False,
)
batch.alter_column(
"updated_at",
existing_type=sa.DateTime(timezone=True),
nullable=False,
)
indexes = _indexes()
if _OLD_INDEX in indexes:
op.drop_index(_OLD_INDEX, table_name=_TABLE)
indexes = _indexes()
if _TARGET_INDEX not in indexes:
op.create_index(_TARGET_INDEX, _TABLE, ["target_key"], unique=True)
if _AGENT_INDEX not in indexes:
op.create_index(_AGENT_INDEX, _TABLE, ["agent_id", "created_at"])
def downgrade() -> None:
if not sa.inspect(op.get_bind()).has_table(_TABLE):
return
indexes = _indexes()
if _AGENT_INDEX in indexes:
op.drop_index(_AGENT_INDEX, table_name=_TABLE)
if _TARGET_INDEX in indexes:
op.drop_index(_TARGET_INDEX, table_name=_TABLE)
if _OLD_INDEX not in _indexes():
op.create_index(
_OLD_INDEX,
_TABLE,
["thread_id", "run_id", "user_id"],
unique=True,
mysql_length={"user_id": 63},
)
columns = _columns()
with op.batch_alter_table(_TABLE) as batch:
for name in ("updated_at", "reasons", "target_key", "agent_id"):
if name in columns:
batch.drop_column(name)
@@ -0,0 +1,25 @@
"""Merge feedback, public-document, and assistant-knowledge migration heads.
Revision ID: 20260916_03
Revises: 20260916_02, 20260909_01, 20260914_01
Create Date: 2026-09-16
This merge used to share revision id ``20260916_01`` with the LLMWiki graph
edge migration. Giving the no-op merge its own id preserves the graph-edge
revision for already stamped databases while restoring an unambiguous graph.
"""
from __future__ import annotations
revision = "20260916_03"
down_revision = ("20260916_02", "20260909_01", "20260914_01")
branch_labels = None
depends_on = None
def upgrade() -> None:
"""Record the merge; all parent revisions already own their DDL."""
def downgrade() -> None:
"""Splitting the migration history requires no schema change."""
@@ -0,0 +1,49 @@
"""Add editable starter questions and retrieval settings to LLMWiki mappings.
Revision ID: 20260920_01
Revises: 20260903_01, 20260907_01
Create Date: 2026-09-20
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20260920_01"
down_revision = ("20260903_01", "20260907_01")
branch_labels = None
depends_on = None
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table("llmwiki_knowledge_bases"):
return set()
return {str(column["name"]) for column in inspector.get_columns("llmwiki_knowledge_bases")}
def upgrade() -> None:
columns = _columns()
if not columns:
return
if "test_questions" not in columns:
op.add_column(
"llmwiki_knowledge_bases",
sa.Column("test_questions", PortableJSON(), nullable=True),
)
if "retrieval_settings" not in columns:
op.add_column(
"llmwiki_knowledge_bases",
sa.Column("retrieval_settings", PortableJSON(), nullable=True),
)
def downgrade() -> None:
columns = _columns()
if "retrieval_settings" in columns:
op.drop_column("llmwiki_knowledge_bases", "retrieval_settings")
if "test_questions" in columns:
op.drop_column("llmwiki_knowledge_bases", "test_questions")
@@ -0,0 +1,194 @@
"""Reconcile LLMWiki schema after historical duplicate revision ids.
Revision ID: 20260920_02
Revises: 20260920_01, 20260916_03
Create Date: 2026-09-20
Older source snapshots used ``20260916_01`` and ``20260920_01`` twice. A
database stamped with either value can therefore be missing the schema from
the other migration that carried the same id. This merge is deliberately
idempotent and restores every affected table, constraint, index, and JSON
column before recording the new unambiguous head.
"""
from __future__ import annotations
from collections.abc import Iterable
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20260920_02"
down_revision = ("20260920_01", "20260916_03")
branch_labels = None
depends_on = None
_EDGE_TABLE = "llmwiki_wiki_edges"
_EDGE_UNIQUE = "uq_llmwiki_wiki_edge_mapping_pages_type"
_EDGE_COLUMNS = (
"knowledge_base_mapping_id",
"source_page_id",
"target_page_id",
"edge_type",
)
_EDGE_FOREIGN_KEYS = (
(
"fk_llmwiki_wiki_edges_mapping",
"knowledge_base_mapping_id",
"llmwiki_knowledge_bases",
),
(
"fk_llmwiki_wiki_edges_source_page",
"source_page_id",
"llmwiki_wiki_pages",
),
(
"fk_llmwiki_wiki_edges_target_page",
"target_page_id",
"llmwiki_wiki_pages",
),
)
_EDGE_INDEXES = (
(
"ix_llmwiki_wiki_edges_knowledge_base_mapping_id",
("knowledge_base_mapping_id",),
),
("ix_llmwiki_wiki_edges_source_page_id", ("source_page_id",)),
("ix_llmwiki_wiki_edges_target_page_id", ("target_page_id",)),
(
"ix_llmwiki_wiki_edge_mapping_source",
("knowledge_base_mapping_id", "source_page_id"),
),
(
"ix_llmwiki_wiki_edge_mapping_target",
("knowledge_base_mapping_id", "target_page_id"),
),
)
def _create_edges_table() -> None:
op.create_table(
_EDGE_TABLE,
sa.Column("id", sa.String(length=36), primary_key=True),
sa.Column("knowledge_base_mapping_id", sa.String(length=36), nullable=False),
sa.Column("source_page_id", sa.String(length=36), nullable=False),
sa.Column("target_page_id", sa.String(length=36), nullable=False),
sa.Column(
"edge_type",
sa.String(length=32),
nullable=False,
server_default="wiki_link",
),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.ForeignKeyConstraint(
["knowledge_base_mapping_id"],
["llmwiki_knowledge_bases.id"],
name="fk_llmwiki_wiki_edges_mapping",
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["source_page_id"],
["llmwiki_wiki_pages.id"],
name="fk_llmwiki_wiki_edges_source_page",
ondelete="CASCADE",
),
sa.ForeignKeyConstraint(
["target_page_id"],
["llmwiki_wiki_pages.id"],
name="fk_llmwiki_wiki_edges_target_page",
ondelete="CASCADE",
),
sa.UniqueConstraint(*_EDGE_COLUMNS, name=_EDGE_UNIQUE),
)
def _foreign_key_shape(item: dict) -> tuple[tuple[str, ...], str | None, tuple[str, ...]]:
return (
tuple(str(name) for name in item.get("constrained_columns") or ()),
item.get("referred_table"),
tuple(str(name) for name in item.get("referred_columns") or ()),
)
def _ensure_edge_constraints(inspector: sa.Inspector) -> None:
unique_names = {
str(item["name"])
for item in inspector.get_unique_constraints(_EDGE_TABLE)
if item.get("name")
}
foreign_key_shapes = {
_foreign_key_shape(item) for item in inspector.get_foreign_keys(_EDGE_TABLE)
}
missing_foreign_keys = [
(name, column, referred_table)
for name, column, referred_table in _EDGE_FOREIGN_KEYS
if ((column,), referred_table, ("id",)) not in foreign_key_shapes
]
if _EDGE_UNIQUE in unique_names and not missing_foreign_keys:
return
with op.batch_alter_table(_EDGE_TABLE) as batch:
if _EDGE_UNIQUE not in unique_names:
batch.create_unique_constraint(_EDGE_UNIQUE, list(_EDGE_COLUMNS))
for name, column, referred_table in missing_foreign_keys:
batch.create_foreign_key(
name,
referred_table,
[column],
["id"],
ondelete="CASCADE",
)
def _ensure_edge_indexes(inspector: sa.Inspector) -> None:
index_names = {
str(item["name"])
for item in inspector.get_indexes(_EDGE_TABLE)
if item.get("name")
}
for name, columns in _EDGE_INDEXES:
if name not in index_names:
op.create_index(name, _EDGE_TABLE, list(columns))
def _ensure_edges() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table(_EDGE_TABLE):
_create_edges_table()
else:
_ensure_edge_constraints(inspector)
_ensure_edge_indexes(sa.inspect(bind))
def _columns(table: str) -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(table):
return set()
return {str(item["name"]) for item in inspector.get_columns(table)}
def _ensure_json_columns(table: str, columns: Iterable[str]) -> None:
existing = _columns(table)
if not existing:
return
for name in columns:
if name not in existing:
op.add_column(table, sa.Column(name, PortableJSON(), nullable=True))
def upgrade() -> None:
_ensure_edges()
_ensure_json_columns(
"llmwiki_knowledge_bases",
("test_questions", "retrieval_settings"),
)
_ensure_json_columns("llmwiki_wiki_sync_states", ("last_sync_result",))
def downgrade() -> None:
columns = _columns("llmwiki_wiki_sync_states")
if "last_sync_result" in columns:
op.drop_column("llmwiki_wiki_sync_states", "last_sync_result")
@@ -0,0 +1,39 @@
"""Persist DeerFlow-owned fixed parent directories for LLMWiki mappings.
Revision ID: 20261006_01
Revises: 20260920_02
Create Date: 2026-10-06
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20261006_01"
down_revision = "20260920_02"
branch_labels = None
depends_on = None
_TABLE = "llmwiki_knowledge_bases"
_COLUMN = "wiki_parent_directories"
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TABLE):
return set()
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
def upgrade() -> None:
columns = _columns()
if columns and _COLUMN not in columns:
op.add_column(_TABLE, sa.Column(_COLUMN, PortableJSON(), nullable=True))
def downgrade() -> None:
if _COLUMN in _columns():
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,40 @@
"""Persist administrator-managed per-model runtime capabilities.
Revision ID: 20261009_01
Revises: 20261006_01
Create Date: 2026-10-09
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20261009_01"
down_revision = "20261006_01"
branch_labels = None
depends_on = None
_TABLE = "model_capabilities"
def upgrade() -> None:
if sa.inspect(op.get_bind()).has_table(_TABLE):
return
op.create_table(
_TABLE,
sa.Column("model_name", sa.String(length=160), primary_key=True),
sa.Column(
"supports_reasoning_effort",
sa.Boolean(),
nullable=False,
server_default=sa.false(),
),
sa.Column("updated_by", sa.String(length=64), nullable=True),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
if sa.inspect(op.get_bind()).has_table(_TABLE):
op.drop_table(_TABLE)
@@ -0,0 +1,41 @@
"""Persist the global ordinary-knowledge smart-reasoning switch.
Revision ID: 20261014_01
Revises: 20261009_01
Create Date: 2026-10-14
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20261014_01"
down_revision = "20261009_01"
branch_labels = None
depends_on = None
_TABLE = "knowledge_retrieval_settings"
def upgrade() -> None:
if sa.inspect(op.get_bind()).has_table(_TABLE):
return
op.create_table(
_TABLE,
sa.Column("scope", sa.String(length=64), primary_key=True),
sa.Column(
"smart_reasoning_enabled",
sa.Boolean(),
nullable=False,
server_default=sa.false(),
),
sa.Column("revision", sa.Integer(), nullable=False, server_default="1"),
sa.Column("updated_by", sa.String(length=64), nullable=True),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
def downgrade() -> None:
if sa.inspect(op.get_bind()).has_table(_TABLE):
op.drop_table(_TABLE)
@@ -0,0 +1,54 @@
"""Persist hierarchical Wiki generation configuration.
Revision ID: 20261014_02
Revises: 20261014_01
Create Date: 2026-10-14
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20261014_02"
down_revision = "20261014_01"
branch_labels = None
depends_on = None
_TABLE = "llmwiki_knowledge_bases"
_COLUMNS: tuple[sa.Column, ...] = (
sa.Column("wiki_directory_paths", PortableJSON(), nullable=True),
sa.Column("wiki_focus_instructions", sa.Text(), nullable=True),
sa.Column("wiki_content_instructions_raw", sa.Text(), nullable=True),
sa.Column("wiki_extraction_instructions_raw", sa.Text(), nullable=True),
sa.Column("wiki_generation_managed", sa.Boolean(), nullable=True),
sa.Column("wiki_config_schema_version", sa.Integer(), nullable=True),
sa.Column("wiki_config_revision", sa.Integer(), nullable=True),
sa.Column("wiki_config_apply_status", sa.String(length=32), nullable=True),
sa.Column("wiki_config_apply_error", sa.Text(), nullable=True),
)
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TABLE):
return set()
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
def upgrade() -> None:
existing = _columns()
if not existing:
return
for column in _COLUMNS:
if column.name not in existing:
op.add_column(_TABLE, column)
def downgrade() -> None:
existing = _columns()
for column in reversed(_COLUMNS):
if column.name in existing:
op.drop_column(_TABLE, column.name)
@@ -0,0 +1,105 @@
"""Index task deep-link thread lookups outside the metadata JSON blob.
Revision ID: 20261015_01
Revises: 20260920_02
Create Date: 2026-10-15
``threads_meta.metadata_json`` remains the public source of truth. The new
columns are maintained projections for the high-frequency task workspace
lookup, whose task rows are shared across users and therefore cannot use the
ordinary ``user_id`` index.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect, text
revision = "20261015_01"
down_revision = "20260920_02"
branch_labels = None
depends_on = None
_TABLE = "threads_meta"
_COLUMNS = (
("task_id", 128),
("agent_id", 128),
("action_id", 128),
)
_INDEXES = (
("ix_threads_meta_task_agent_updated", ("task_id", "agent_id", "updated_at", "thread_id")),
("ix_threads_meta_task_action_agent_updated", ("task_id", "action_id", "agent_id", "updated_at", "thread_id")),
("ix_threads_meta_user_updated", ("user_id", "updated_at", "thread_id")),
)
def _has_table(table_name: str) -> bool:
return table_name in inspect(op.get_bind()).get_table_names()
def _has_column(column_name: str) -> bool:
return any(column["name"] == column_name for column in inspect(op.get_bind()).get_columns(_TABLE))
def _has_index(index_name: str) -> bool:
return index_name in {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
def _json_scalar_expression(key: str) -> str | None:
"""Return a portable-enough JSON scalar extractor for supported engines."""
dialect = op.get_bind().dialect.name
if dialect == "sqlite":
return f"json_extract(metadata_json, '$.{key}')"
if dialect == "mysql":
# PortableJSON is LONGTEXT on MySQL, but JSON_EXTRACT accepts valid
# JSON text and returns the quoted JSON scalar until unquoted here.
return f"JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.{key}'))"
if dialect in {"postgresql", "opengauss"}:
return f"metadata_json ->> '{key}'"
return None
def _backfill_projections() -> None:
expressions = {
"task_id": _json_scalar_expression("taskId"),
"agent_id": _json_scalar_expression("agent_id"),
"action_id": _json_scalar_expression("actionId"),
}
if any(expression is None for expression in expressions.values()):
return
assignments = ", ".join(
f"{column} = {expression}"
for column, expression in expressions.items()
)
# Only fresh projection columns are NULL. This makes a retry safe and
# never overwrites values maintained by a newer running application.
op.get_bind().execute(
text(
f"UPDATE {_TABLE} SET {assignments} "
"WHERE task_id IS NULL AND agent_id IS NULL AND action_id IS NULL"
)
)
def upgrade() -> None:
if not _has_table(_TABLE):
return
for column_name, length in _COLUMNS:
if not _has_column(column_name):
op.add_column(_TABLE, sa.Column(column_name, sa.String(length), nullable=True))
_backfill_projections()
for index_name, columns in _INDEXES:
if not _has_index(index_name):
op.create_index(index_name, _TABLE, list(columns))
def downgrade() -> None:
if not _has_table(_TABLE):
return
for index_name, _columns in reversed(_INDEXES):
if _has_index(index_name):
op.drop_index(index_name, table_name=_TABLE)
for column_name, _length in reversed(_COLUMNS):
if _has_column(column_name):
op.drop_column(_TABLE, column_name)
@@ -0,0 +1,58 @@
"""Add dedicated single-agent task deep-link conversation index.
Revision ID: 20261019_01
Revises: 20261015_01
Create Date: 2026-10-19
This is intentionally schema-only. Existing ``threads_meta`` task rows are not
backfilled; only conversations created/updated with the new ``taskGoPath``
metadata marker enter the dedicated history.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
from sqlalchemy import inspect
revision = "20261019_01"
down_revision = "20261015_01"
branch_labels = None
depends_on = None
_TABLE = "task_deeplink_threads"
_INDEX = "ix_task_deeplink_threads_scope"
def _has_table() -> bool:
return _TABLE in inspect(op.get_bind()).get_table_names()
def upgrade() -> None:
if _has_table():
return
op.create_table(
_TABLE,
sa.Column("thread_id", sa.String(64), primary_key=True),
sa.Column("task_id", sa.String(128), nullable=False),
sa.Column("source_task_id", sa.String(128), nullable=False),
sa.Column("go_path", sa.String(16), nullable=False),
sa.Column("agent_id", sa.String(128), nullable=True),
sa.Column("created_by", sa.String(64), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
)
op.create_index(
_INDEX,
_TABLE,
["task_id", "go_path", "updated_at", "thread_id"],
)
def downgrade() -> None:
if not _has_table():
return
indexes = {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
if _INDEX in indexes:
op.drop_index(_INDEX, table_name=_TABLE)
op.drop_table(_TABLE)
@@ -0,0 +1,90 @@
"""Persist the knowledge-management advanced-entry switch.
Revision ID: 20261020_01
Revises: 20261019_01, 20261014_02
Create Date: 2026-10-20
An earlier release accidentally reused revision ``20261019_01`` for this
column and for the task deep-link table. A database that ran the column
migration can therefore report the task migration as applied even though its
table is absent. The upgrade reconciles that table and its scope index before
ensuring the advanced-management column, so both fresh and affected databases
converge on the same schema. The task objects belong to the parent revision and
are intentionally left intact by this revision's downgrade.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20261020_01"
down_revision = ("20261019_01", "20261014_02")
branch_labels = None
depends_on = None
_TABLE = "knowledge_retrieval_settings"
_COLUMN = "advanced_management_enabled"
_TASK_TABLE = "task_deeplink_threads"
_TASK_INDEX = "ix_task_deeplink_threads_scope"
def _columns() -> set[str]:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TABLE):
return set()
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
def _ensure_task_deeplink_schema() -> None:
inspector = sa.inspect(op.get_bind())
if not inspector.has_table(_TASK_TABLE):
op.create_table(
_TASK_TABLE,
sa.Column("thread_id", sa.String(64), primary_key=True),
sa.Column("task_id", sa.String(128), nullable=False),
sa.Column("source_task_id", sa.String(128), nullable=False),
sa.Column("go_path", sa.String(16), nullable=False),
sa.Column("agent_id", sa.String(128), nullable=True),
sa.Column("created_by", sa.String(64), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
)
op.create_index(
_TASK_INDEX,
_TASK_TABLE,
["task_id", "go_path", "updated_at", "thread_id"],
)
return
indexes = {
str(index["name"])
for index in inspector.get_indexes(_TASK_TABLE)
if index.get("name")
}
if _TASK_INDEX not in indexes:
op.create_index(
_TASK_INDEX,
_TASK_TABLE,
["task_id", "go_path", "updated_at", "thread_id"],
)
def upgrade() -> None:
_ensure_task_deeplink_schema()
if _COLUMN in _columns():
return
op.add_column(
_TABLE,
sa.Column(
_COLUMN,
sa.Boolean(),
nullable=False,
server_default=sa.false(),
),
)
def downgrade() -> None:
if _COLUMN in _columns():
op.drop_column(_TABLE, _COLUMN)
@@ -0,0 +1,49 @@
"""Administrator-managed per-model runtime capabilities."""
from __future__ import annotations
from collections.abc import Iterable
from typing import Any
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
from deerflow.persistence.model_capabilities.memory import MemoryModelCapabilityStore
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
from deerflow.persistence.model_capabilities.sql import ModelCapabilityRepository
__all__ = [
"MemoryModelCapabilityStore",
"ModelCapabilityRepository",
"ModelCapabilityRow",
"ModelCapabilityStore",
"import_legacy_reasoning_effort_capabilities",
"make_model_capability_store",
]
async def import_legacy_reasoning_effort_capabilities(
store: ModelCapabilityStore,
models: Iterable[Any],
) -> list[str]:
"""One-time compatibility import for legacy config-backed ``true`` flags.
A configured ``false`` never creates a row. Once any database row exists,
its value is authoritative and cannot be overwritten by legacy YAML.
"""
imported: list[str] = []
for model in models:
if not bool(getattr(model, "supports_reasoning_effort", False)):
continue
model_name = str(getattr(model, "name", "") or "").strip()
if model_name and await store.seed_reasoning_effort_support(
model_name,
updated_by="legacy-config-import",
):
imported.append(model_name)
return imported
def make_model_capability_store(session_factory):
if session_factory is None:
return MemoryModelCapabilityStore()
return ModelCapabilityRepository(session_factory)
@@ -0,0 +1,48 @@
"""Persistence contract for administrator-managed model capabilities.
The provider connection details still live in ``config.yaml``. Runtime
capabilities that administrators change from the web UI live here instead so
they are shared by every Gateway worker and survive application restarts.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class ModelCapabilityStore(ABC):
"""Global capability settings keyed by the stable configured model name."""
@abstractmethod
async def list_capabilities(self) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def set_reasoning_effort_support(
self,
model_name: str,
*,
supports_reasoning_effort: bool,
updated_by: str | None = None,
) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def seed_reasoning_effort_support(
self,
model_name: str,
*,
updated_by: str | None = None,
) -> bool:
"""Insert legacy ``true`` only when no authoritative row exists."""
raise NotImplementedError
@abstractmethod
async def delete_capability(self, model_name: str) -> None:
raise NotImplementedError
@@ -0,0 +1,54 @@
"""In-memory model-capability store used by memory-mode deployments/tests."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
class MemoryModelCapabilityStore(ModelCapabilityStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
async def list_capabilities(self) -> list[dict[str, Any]]:
return [dict(self._rows[name]) for name in sorted(self._rows)]
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
row = self._rows.get(model_name)
return dict(row) if row is not None else None
async def set_reasoning_effort_support(
self,
model_name: str,
*,
supports_reasoning_effort: bool,
updated_by: str | None = None,
) -> dict[str, Any]:
row = {
"model_name": model_name,
"supports_reasoning_effort": bool(supports_reasoning_effort),
"updated_by": updated_by,
"updated_at": datetime.now(UTC).isoformat(),
}
self._rows[model_name] = row
return dict(row)
async def seed_reasoning_effort_support(
self,
model_name: str,
*,
updated_by: str | None = None,
) -> bool:
if model_name in self._rows:
return False
await self.set_reasoning_effort_support(
model_name,
supports_reasoning_effort=True,
updated_by=updated_by,
)
return True
async def delete_capability(self, model_name: str) -> None:
self._rows.pop(model_name, None)
@@ -0,0 +1,30 @@
"""ORM row for administrator-managed model capabilities."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, String, false
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ModelCapabilityRow(Base):
__tablename__ = "model_capabilities"
model_name: Mapped[str] = mapped_column(String(160), primary_key=True)
supports_reasoning_effort: Mapped[bool] = mapped_column(
Boolean,
nullable=False,
default=False,
server_default=false(),
)
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,92 @@
"""SQLAlchemy-backed model-capability store."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
def _to_dict(row: ModelCapabilityRow) -> dict[str, Any]:
return {
"model_name": row.model_name,
"supports_reasoning_effort": bool(row.supports_reasoning_effort),
"updated_by": row.updated_by,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
class ModelCapabilityRepository(ModelCapabilityStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_capabilities(self) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(select(ModelCapabilityRow).order_by(ModelCapabilityRow.model_name.asc()))
return [_to_dict(row) for row in result.scalars()]
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ModelCapabilityRow, model_name)
return _to_dict(row) if row is not None else None
async def set_reasoning_effort_support(
self,
model_name: str,
*,
supports_reasoning_effort: bool,
updated_by: str | None = None,
) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(ModelCapabilityRow, model_name)
if row is None:
row = ModelCapabilityRow(model_name=model_name)
session.add(row)
row.supports_reasoning_effort = bool(supports_reasoning_effort)
row.updated_by = updated_by
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def seed_reasoning_effort_support(
self,
model_name: str,
*,
updated_by: str | None = None,
) -> bool:
"""Insert a legacy ``true`` without ever overwriting an existing row.
The primary-key conflict handles concurrent Gateway startup safely: if
another worker or an administrator creates the row after our read, the
losing seed transaction rolls back and preserves that authoritative
value, including ``false``.
"""
async with self._sf() as session:
if await session.get(ModelCapabilityRow, model_name) is not None:
return False
session.add(
ModelCapabilityRow(
model_name=model_name,
supports_reasoning_effort=True,
updated_by=updated_by,
)
)
try:
await session.commit()
except IntegrityError:
await session.rollback()
return False
return True
async def delete_capability(self, model_name: str) -> None:
async with self._sf() as session:
await session.execute(delete(ModelCapabilityRow).where(ModelCapabilityRow.model_name == model_name))
await session.commit()
@@ -35,6 +35,18 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
from deerflow.persistence.analysis_project_memory.model import (
AnalysisProjectMemoryItemRow,
AnalysisProjectMemoryJobRow,
AnalysisProjectMemoryStateRow,
)
from deerflow.persistence.analysis_projects import AnalysisProjectConfigRow, AnalysisProjectRow, AnalysisProjectThreadRow
from deerflow.persistence.article_types.model import ArticleTypeRow
from deerflow.persistence.artifact_library.model import (
ArtifactLibraryItemRow,
ArtifactLibrarySyncStateRow,
)
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
@@ -48,7 +60,7 @@ from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow, UserCompactMenuOverrideConfigRow
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
@@ -60,13 +72,26 @@ from deerflow.persistence.document_rewrite_versions.model import DocumentRewrite
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
from deerflow.persistence.force_situation.model import (
ForceSituationDetailRow,
ForceSituationDetailVersionRow,
)
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.knowledge_retrieval_settings.model import KnowledgeRetrievalSettingsRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.llmwiki_index.model import (
LlmWikiWikiEdgeRow,
LlmWikiWikiPageRow,
LlmWikiWikiSyncStateRow,
LlmWikiWikiVectorRow,
)
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
from deerflow.persistence.models.run_event import RunEventRow
from deerflow.persistence.notifications.model import NotificationRow
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
@@ -125,13 +150,28 @@ from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.task_analysis.model import (
TaskAnalysisBindingRow,
TaskAnalysisContentVersionRow,
TaskAnalysisRunMessageRow,
TaskAnalysisRunRow,
TaskAnalysisSettingRow,
TaskAnalysisTaskRow,
TaskDataBlockDefRow,
TaskDataBlockRow,
TaskDataPeriodRow,
TaskDataSubmissionRow,
)
from deerflow.persistence.task_buttons.model import TaskButtonRow
from deerflow.persistence.task_deeplink_threads.model import TaskDeeplinkThreadRow
from deerflow.persistence.task_reports.model import TaskReportRow
from deerflow.persistence.taskcop_tasks.model import TaskCopTaskRow
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.persistence.thread_shares.model import ThreadShareRow
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
from deerflow.persistence.upload_policy.model import UploadPolicyRow
from deerflow.persistence.user.model import UserRow
from deerflow.persistence.user_preferences.model import UserPreferencesRow
from deerflow.persistence.user_prompts.model import UserCustomPromptRow
@@ -149,8 +189,21 @@ from deerflow.persistence.workflow_runs.model import (
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
__all__ = [
"AIWritingSessionRow",
"AnalysisProjectConfigRow",
"AnalysisProjectMemoryItemRow",
"AnalysisProjectMemoryJobRow",
"AnalysisProjectMemoryStateRow",
"ArtifactLibraryItemRow",
"ArtifactLibrarySyncStateRow",
"ArticleTypeRow",
"FeedbackRow",
"FixedQuestionRow",
"ForceSituationDetailRow",
"ForceSituationDetailVersionRow",
"GongwenDocumentRow",
"GongwenDocumentVersionRow",
"HtmlPageFavoriteRow",
"AgentExtrasRow",
"AgentFavoriteRow",
"AgentRow",
@@ -159,6 +212,7 @@ __all__ = [
"BusinessMappingRow",
"AssistantKnowledgeBaseRow",
"AssistantKnowledgeSourceRow",
"AssistantKnowledgeVectorRow",
"AssistantKnowledgeImportJobRow",
"AssistantKnowledgeWikiPageRow",
"AssistantKnowledgeWikiRevisionRow",
@@ -167,6 +221,7 @@ __all__ = [
"AssistantKnowledgeRelationRow",
"AssistantKnowledgeRelationContributionRow",
"CompactMenuOverrideRow",
"UserCompactMenuOverrideConfigRow",
"ConcurrencySampleRow",
"DashboardSessionRow",
"DocumentRewriteVersionRow",
@@ -186,6 +241,7 @@ __all__ = [
"KnowledgeRelationRow",
"KnowledgeSourceRow",
"KnowledgeTagRow",
"KnowledgeRetrievalSettingsRow",
"LightAppRow",
"WorkflowDataSourceRow",
"WorkflowDefinitionRow",
@@ -200,9 +256,12 @@ __all__ = [
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
"LlmWikiWikiPageRow",
"LlmWikiWikiEdgeRow",
"LlmWikiWikiVectorRow",
"LlmWikiWikiSyncStateRow",
"MenuOverrideRow",
"ModelCapabilityRow",
"UserMenuOverrideConfigRow",
"NotificationRow",
"ParallelAgentTaskRow",
"PositionRoundtableNodeRow",
@@ -236,6 +295,7 @@ __all__ = [
"RunEventRow",
"RunRow",
"ScheduledTaskDeliveryProfileRow",
"ScheduledHtmlPageRow",
"ScheduledTaskRow",
"ScheduledTaskRunRow",
"ScheduledTaskSubscriptionRow",
@@ -254,9 +314,23 @@ __all__ = [
"SkillKnowledgeSnapshotRow",
"SkillKnowledgeSyncItemRow",
"SkillKnowledgeSyncJobRow",
"StrategyAgentRoleRow",
"TagAssignmentRow",
"TagRow",
"TaskButtonRow",
"TaskAnalysisBindingRow",
"TaskAnalysisContentVersionRow",
"TaskAnalysisRunMessageRow",
"TaskAnalysisRunRow",
"TaskAnalysisSettingRow",
"TaskAnalysisTaskRow",
"TaskDataBlockDefRow",
"TaskDataBlockRow",
"TaskDataPeriodRow",
"TaskDataSubmissionRow",
"TaskDeeplinkThreadRow",
"AnalysisProjectRow",
"AnalysisProjectThreadRow",
"TaskCopTaskRow",
"TaskReportRow",
"EmbedSessionRow",
@@ -264,6 +338,7 @@ __all__ = [
"ThreadMetaRow",
"ThreadShareRow",
"ToolCallMetricRow",
"UploadPolicyRow",
"UserCustomPromptRow",
"UserMemoryConfigRow",
"UserPreferencesRow",
@@ -14,7 +14,7 @@ import json
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
@@ -57,17 +57,40 @@ class ParallelAgentTaskRepository:
data["messages"] = _loads(row.messages) or []
return data
async def list_tasks(self, user_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
"""列出当前用户的任务(轻量元信息,不含 messages,供历史列表)。"""
async def list_tasks(
self,
user_id: str,
*,
limit: int = 200,
offset: int = 0,
id_prefix: str | None = None,
include_messages: bool = False,
) -> list[dict[str, Any]]:
"""List the current user's tasks, optionally including saved snapshots.
History-list callers keep the lightweight default. Server-side
projections such as the artifact library can opt into ``messages`` in
one query instead of issuing an N+1 ``get_task`` sequence. ``id_prefix``
and ``offset`` allow those projections to page only their own task
namespace without loading unrelated, potentially large transcripts.
"""
stmt = select(ParallelAgentTaskRow).where(ParallelAgentTaskRow.user_id == user_id)
if id_prefix:
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix))
stmt = (
select(ParallelAgentTaskRow)
.where(ParallelAgentTaskRow.user_id == user_id)
.order_by(ParallelAgentTaskRow.updated_at.desc())
stmt.order_by(
ParallelAgentTaskRow.updated_at.desc(),
ParallelAgentTaskRow.id.asc(),
)
.offset(offset)
.limit(limit)
)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._to_dict(row, with_messages=False) for row in result.scalars()]
return [
self._to_dict(row, with_messages=include_messages)
for row in result.scalars()
]
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
@@ -103,22 +126,40 @@ class ParallelAgentTaskRepository:
brief=_UNSET,
agents=_UNSET,
messages=_UNSET,
expected_updated_at: datetime | str | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(ParallelAgentTaskRow, task_id)
if row is None or row.user_id != user_id:
expected_revision = expected_updated_at
if isinstance(expected_revision, str):
try:
expected_revision = datetime.fromisoformat(expected_revision.replace("Z", "+00:00"))
except ValueError:
return None
values: dict[str, Any] = {"updated_at": datetime.now(UTC)}
if title is not _UNSET and title is not None:
values["title"] = str(title).strip()
if brief is not _UNSET:
values["brief"] = brief or None
if agents is not _UNSET:
values["agents"] = _dumps(agents)
if messages is not _UNSET:
values["messages"] = _dumps(messages)
async with self._sf() as session:
statement = update(ParallelAgentTaskRow).where(
ParallelAgentTaskRow.id == task_id,
ParallelAgentTaskRow.user_id == user_id,
)
if expected_revision is not None:
statement = statement.where(ParallelAgentTaskRow.updated_at == expected_revision)
result = await session.execute(statement.values(**values))
if result.rowcount != 1:
await session.rollback()
return None
if title is not _UNSET and title is not None:
row.title = str(title).strip()
if brief is not _UNSET:
row.brief = brief or None
if agents is not _UNSET:
row.agents = _dumps(agents)
if messages is not _UNSET:
row.messages = _dumps(messages)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
row = await session.get(ParallelAgentTaskRow, task_id)
if row is None:
return None
return self._to_dict(row)
async def delete_task(self, task_id: str, user_id: str) -> bool:
@@ -0,0 +1,24 @@
"""Global strategy-selection agent role registry."""
from __future__ import annotations
from deerflow.persistence.strategy_agent_roles.base import StrategyAgentRoleStore
from deerflow.persistence.strategy_agent_roles.memory import (
MemoryStrategyAgentRoleStore,
)
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
from deerflow.persistence.strategy_agent_roles.sql import StrategyAgentRoleRepository
__all__ = [
"MemoryStrategyAgentRoleStore",
"StrategyAgentRoleRepository",
"StrategyAgentRoleRow",
"StrategyAgentRoleStore",
"make_strategy_agent_role_store",
]
def make_strategy_agent_role_store(session_factory):
if session_factory is None:
return MemoryStrategyAgentRoleStore()
return StrategyAgentRoleRepository(session_factory)
@@ -0,0 +1,39 @@
"""Abstract store for the global strategy-selection agent registry."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class StrategyAgentRoleStore(ABC):
@abstractmethod
async def ensure_defaults(self, rows: list[dict[str, Any]]) -> int:
"""Insert missing default rows without re-enabling disabled angles."""
@abstractmethod
async def list_roles(self, *, include_disabled: bool) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_role(self, agent_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_role(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update_role(
self,
agent_id: str,
*,
enabled: bool | None = None,
sort_order: int | None = None,
) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def delete_role(self, agent_id: str) -> bool:
"""Hard-delete a registry row for create rollback only."""
raise NotImplementedError
@@ -0,0 +1,114 @@
"""In-memory strategy-agent registry used by memory persistence and tests."""
from __future__ import annotations
import asyncio
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.strategy_agent_roles.base import StrategyAgentRoleStore
def _now() -> str:
return datetime.now(UTC).isoformat()
def _copy(row: dict[str, Any]) -> dict[str, Any]:
return dict(row)
class MemoryStrategyAgentRoleStore(StrategyAgentRoleStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._lock = asyncio.Lock()
async def ensure_defaults(self, rows: list[dict[str, Any]]) -> int:
inserted = 0
async with self._lock:
for source in rows:
agent_id = str(source["agent_id"])
existing = self._rows.get(agent_id)
if existing is not None:
# Canonical fixed roles are executable infrastructure, not
# administrator-managed lifecycle rows. Repair stale or
# manually-corrupted metadata on every bootstrap. Angle
# defaults deliberately retain their enabled/order state.
if bool(source.get("fixed", False)):
canonical = {
"category": str(source["category"]),
"fixed": True,
"enabled": bool(source.get("enabled", True)),
"sort_order": int(source.get("sort_order", 0)),
}
if any(existing.get(key) != value for key, value in canonical.items()):
existing.update(canonical)
existing["updated_at"] = _now()
continue
now = _now()
self._rows[agent_id] = {
"agent_id": agent_id,
"category": str(source["category"]),
"fixed": bool(source.get("fixed", False)),
"enabled": bool(source.get("enabled", True)),
"sort_order": int(source.get("sort_order", 0)),
"created_at": now,
"updated_at": now,
}
inserted += 1
return inserted
async def list_roles(self, *, include_disabled: bool) -> list[dict[str, Any]]:
async with self._lock:
rows = [_copy(row) for row in self._rows.values() if include_disabled or bool(row.get("enabled"))]
rows.sort(
key=lambda row: (
int(row.get("sort_order", 0)),
str(row.get("agent_id", "")),
)
)
return rows
async def get_role(self, agent_id: str) -> dict[str, Any] | None:
async with self._lock:
row = self._rows.get(agent_id)
return _copy(row) if row is not None else None
async def create_role(self, data: dict[str, Any]) -> dict[str, Any]:
agent_id = str(data["agent_id"])
async with self._lock:
if agent_id in self._rows:
raise ValueError(f"Strategy agent role '{agent_id}' already exists")
now = _now()
row = {
"agent_id": agent_id,
"category": str(data["category"]),
"fixed": bool(data.get("fixed", False)),
"enabled": bool(data.get("enabled", True)),
"sort_order": int(data.get("sort_order", 0)),
"created_at": now,
"updated_at": now,
}
self._rows[agent_id] = row
return _copy(row)
async def update_role(
self,
agent_id: str,
*,
enabled: bool | None = None,
sort_order: int | None = None,
) -> dict[str, Any] | None:
async with self._lock:
row = self._rows.get(agent_id)
if row is None:
return None
if enabled is not None:
row["enabled"] = bool(enabled)
if sort_order is not None:
row["sort_order"] = int(sort_order)
row["updated_at"] = _now()
return _copy(row)
async def delete_role(self, agent_id: str) -> bool:
async with self._lock:
return self._rows.pop(agent_id, None) is not None
@@ -0,0 +1,36 @@
"""Persistence model for the global strategy-selection agent registry."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class StrategyAgentRoleRow(Base):
"""Classify folder-backed agents used by the strategy-selection workflow.
The agent's model, skills and SOUL remain authoritative in its normal
``config.yaml`` / ``SOUL.md`` files. This small global table owns only the
workflow classification and lifecycle flags. Keeping a disabled row (and
the underlying agent) protects saved projects that still reference it.
"""
__tablename__ = "strategy_agent_roles"
agent_id: Mapped[str] = mapped_column(String(64), primary_key=True)
category: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
fixed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="1", index=True)
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,141 @@
"""SQLAlchemy-backed global strategy-selection agent registry."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.strategy_agent_roles.base import StrategyAgentRoleStore
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
def _to_dict(row: StrategyAgentRoleRow) -> dict[str, Any]:
return {
"agent_id": row.agent_id,
"category": row.category,
"fixed": row.fixed,
"enabled": row.enabled,
"sort_order": row.sort_order,
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
}
class StrategyAgentRoleRepository(StrategyAgentRoleStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def ensure_defaults(self, rows: list[dict[str, Any]]) -> int:
inserted = 0
for source in rows:
# Commit each of the small number of defaults independently. If
# two workers race on one primary key, rolling that save back must
# not also discard unrelated defaults inserted by this worker.
async with self._sf() as session:
agent_id = str(source["agent_id"])
existing = await session.get(StrategyAgentRoleRow, agent_id)
if existing is not None:
if bool(source.get("fixed", False)):
changed = False
canonical = {
"category": str(source["category"]),
"fixed": True,
"enabled": bool(source.get("enabled", True)),
"sort_order": int(source.get("sort_order", 0)),
}
for key, value in canonical.items():
if getattr(existing, key) != value:
setattr(existing, key, value)
changed = True
if changed:
existing.updated_at = datetime.now(UTC)
await session.commit()
continue
now = datetime.now(UTC)
session.add(
StrategyAgentRoleRow(
agent_id=agent_id,
category=str(source["category"]),
fixed=bool(source.get("fixed", False)),
enabled=bool(source.get("enabled", True)),
sort_order=int(source.get("sort_order", 0)),
created_at=now,
updated_at=now,
)
)
try:
await session.commit()
except IntegrityError:
# Another startup worker won this primary-key race.
await session.rollback()
continue
inserted += 1
return inserted
async def list_roles(self, *, include_disabled: bool) -> list[dict[str, Any]]:
stmt = select(StrategyAgentRoleRow)
if not include_disabled:
stmt = stmt.where(StrategyAgentRoleRow.enabled.is_(True))
stmt = stmt.order_by(
StrategyAgentRoleRow.sort_order.asc(),
StrategyAgentRoleRow.agent_id.asc(),
)
async with self._sf() as session:
result = await session.execute(stmt)
return [_to_dict(row) for row in result.scalars()]
async def get_role(self, agent_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(StrategyAgentRoleRow, agent_id)
return _to_dict(row) if row is not None else None
async def create_role(self, data: dict[str, Any]) -> dict[str, Any]:
now = datetime.now(UTC)
row = StrategyAgentRoleRow(
agent_id=str(data["agent_id"]),
category=str(data["category"]),
fixed=bool(data.get("fixed", False)),
enabled=bool(data.get("enabled", True)),
sort_order=int(data.get("sort_order", 0)),
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
try:
await session.commit()
except IntegrityError as exc:
await session.rollback()
raise ValueError(f"Strategy agent role '{row.agent_id}' already exists") from exc
await session.refresh(row)
return _to_dict(row)
async def update_role(
self,
agent_id: str,
*,
enabled: bool | None = None,
sort_order: int | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(StrategyAgentRoleRow, agent_id)
if row is None:
return None
if enabled is not None:
row.enabled = bool(enabled)
if sort_order is not None:
row.sort_order = int(sort_order)
row.updated_at = datetime.now(UTC)
await session.commit()
await session.refresh(row)
return _to_dict(row)
async def delete_role(self, agent_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(delete(StrategyAgentRoleRow).where(StrategyAgentRoleRow.agent_id == agent_id))
await session.commit()
return (result.rowcount or 0) > 0
@@ -23,7 +23,9 @@ from deerflow.persistence.types import BeijingDateTime
# Allowed tag/target types. Kept here so persistence + API share one source.
# ``recommended_question`` was added for the 岗位 (position) subscription system
# so admin-configured recommended questions can be tag-targeted like agents.
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task", "recommended_question")
# ``light_app`` labels 轻应用中心 apps (应用管理 / 应用列表); it is *not* a
# 岗位 resource type — light apps stay globally visible.
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task", "recommended_question", "light_app")
class TagRow(Base):
@@ -0,0 +1,48 @@
"""External-task multi-agent scheduled analysis persistence (任务分析定时)."""
from __future__ import annotations
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
from deerflow.persistence.task_analysis.data_blocks import (
DEFAULT_DATA_BLOCK_KEYS,
DEFAULT_DATA_BLOCKS,
merge_payload,
)
from deerflow.persistence.task_analysis.memory import MemoryTaskAnalysisStore
from deerflow.persistence.task_analysis.model import (
TaskAnalysisBindingRow,
TaskAnalysisContentVersionRow,
TaskAnalysisRunMessageRow,
TaskAnalysisRunRow,
TaskAnalysisTaskRow,
TaskDataBlockDefRow,
TaskDataBlockRow,
TaskDataPeriodRow,
TaskDataSubmissionRow,
)
from deerflow.persistence.task_analysis.sql import TaskAnalysisRepository
__all__ = [
"DEFAULT_DATA_BLOCK_KEYS",
"DEFAULT_DATA_BLOCKS",
"MemoryTaskAnalysisStore",
"TaskAnalysisBindingRow",
"TaskAnalysisContentVersionRow",
"TaskAnalysisRepository",
"TaskAnalysisRunMessageRow",
"TaskAnalysisRunRow",
"TaskAnalysisStore",
"TaskAnalysisTaskRow",
"TaskDataBlockDefRow",
"TaskDataBlockRow",
"TaskDataPeriodRow",
"TaskDataSubmissionRow",
"make_task_analysis_store",
"merge_payload",
]
def make_task_analysis_store(session_factory):
if session_factory is None:
return MemoryTaskAnalysisStore()
return TaskAnalysisRepository(session_factory)
@@ -0,0 +1,173 @@
"""Abstract store API for the external-task scheduled analysis feature."""
from __future__ import annotations
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any
class TaskAnalysisStore(ABC):
"""Tasks / bindings / runs / run-messages / content versions."""
# ── external tasks ──────────────────────────────────────────────────────
@abstractmethod
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
"""Create a task. Raises ValueError when source_task_id already exists."""
@abstractmethod
async def get_task(self, task_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def list_tasks(self) -> list[dict[str, Any]]:
"""All tasks newest first, each augmented with its binding (if any)."""
@abstractmethod
async def delete_task_cascade(self, task_id: str) -> bool:
"""Delete task + binding + runs + messages + versions. True if existed."""
# ── bindings ────────────────────────────────────────────────────────────
@abstractmethod
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
"""Create-or-update the 1:1 binding for a task (refresh chain snapshot)."""
@abstractmethod
async def delete_binding(self, external_task_id: str) -> bool: ...
@abstractmethod
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
"""Enabled bindings with next_run_at <= now (execution candidates)."""
@abstractmethod
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
"""Patch mutable fields (next_run_at / last_status / direction_notes...)."""
@abstractmethod
async def last_bound_chain_id(self) -> str | None:
"""Most recently updated binding's chain id (default option for create API)."""
# ── runs ────────────────────────────────────────────────────────────────
@abstractmethod
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
"""Idempotent create: same binding_id + scheduled_for returns existing row."""
@abstractmethod
async def get_run(self, run_id: str) -> dict[str, Any] | None: ...
@abstractmethod
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]: ...
@abstractmethod
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None: ...
@abstractmethod
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
"""Runs still pending/running past ``older_than`` (watchdog recovery)."""
# ── run messages ────────────────────────────────────────────────────────
@abstractmethod
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
"""Append batch of {seat_agent_id, stage_index, event_type, content}."""
@abstractmethod
async def list_run_messages(
self, run_id: str, *, seat: str | None = None, limit: int = 500
) -> list[dict[str, Any]]: ...
# ── content versions (timeline) ─────────────────────────────────────────
@abstractmethod
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
"""One version per run (idempotent on run_id: replaces existing)."""
@abstractmethod
async def list_versions(self, external_task_id: str, *, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:
"""Timeline versions newest first (对方详情接口数据源)."""
# ── data delivery: block defs / periods / submissions ────────────────────
@abstractmethod
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
"""Seed the 5 default block defs when absent (create/legacy tasks); return ordered."""
@abstractmethod
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
"""Block defs ordered by sort_order (empty list when never seeded)."""
@abstractmethod
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""Patch title/sort_order per block_key. ValueError on unknown block_key."""
@abstractmethod
async def open_period(
self,
task_id: str,
*,
source: str,
run_id: str | None = None,
opened_at: datetime | None = None,
) -> dict[str, Any]:
"""Append a new timeline period (period_index = max+1). Idempotent on run_id.
``opened_at`` 默认取当前时间;演示/回填场景可显式指定刻度时刻。
"""
@abstractmethod
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None: ...
@abstractmethod
async def current_period(self, task_id: str) -> dict[str, Any] | None:
"""Latest period (submissions always land here)."""
@abstractmethod
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
"""Periods ascending by period_index (timeline ticks)."""
@abstractmethod
async def append_submission(
self,
task_id: str,
period_index: int,
block_key: str,
payload: dict[str, Any],
*,
submitted_by: str | None = None,
) -> dict[str, Any]:
"""Append to the submission log (seq per task+period+block)."""
@abstractmethod
async def list_submissions(
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
) -> list[dict[str, Any]]:
"""Submission log for one block of one period ordered by seq (merge source)."""
@abstractmethod
async def upsert_period_block(
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
) -> dict[str, Any]:
"""Write the merged payload for (task, period, block); bumps updated_at."""
@abstractmethod
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
"""All merged block rows of a task ({block_key, period_index, payload, updated_at})."""
@abstractmethod
async def get_setting(self, key: str) -> str | None:
"""Read one module setting value (``None`` when absent)."""
@abstractmethod
async def set_setting(self, key: str, value: str) -> None:
"""Upsert one module setting value."""
@@ -0,0 +1,45 @@
"""Constants and pure helpers for the scheduled data-delivery blocks.
块 key 定死 ``block_1``~``block_6``(已定稿口径,不做席位↔块映射);payload 不做
schema 校验(schema 会变),只要求是 JSON 对象。合并规则:顶层 key 合并、同 key
后写覆盖 —— 多个智能体可各写一个块的不同部分,一个智能体也可写多个块。
"""
from __future__ import annotations
from typing import Any
# (block_key, 默认标题);建任务时自动种 6 条,标题/排序后续可经管理端修改。
DEFAULT_DATA_BLOCKS: tuple[tuple[str, str], ...] = (
("block_1", "数据块一"),
("block_2", "数据块二"),
("block_3", "数据块三"),
("block_4", "数据块四"),
("block_5", "数据块五"),
("block_6", "数据块六"),
)
DEFAULT_DATA_BLOCK_KEYS: tuple[str, ...] = tuple(key for key, _title in DEFAULT_DATA_BLOCKS)
# 期(时间轴刻度)触发来源。
PERIOD_TRIGGER_CRON = "cron"
PERIOD_TRIGGER_MANUAL = "manual"
PERIOD_TRIGGER_EXTERNAL = "external"
def default_block_defs() -> list[dict[str, Any]]:
return [
{"block_key": key, "title": title, "sort_order": index}
for index, (key, title) in enumerate(DEFAULT_DATA_BLOCKS, start=1)
]
def is_default_block_key(block_key: str) -> bool:
return block_key in DEFAULT_DATA_BLOCK_KEYS
def merge_payload(base: dict[str, Any], incoming: dict[str, Any]) -> dict[str, Any]:
"""顶层 key 合并、同 key 后写覆盖(纯函数,返回新 dict)。"""
merged = dict(base)
merged.update(incoming)
return merged
@@ -0,0 +1,467 @@
"""In-memory task-analysis store (no DB configured / tests).
Also serves as the behavioural reference for the SQL repository: field
normalisation (JSON encode/decode, ISO datetime) lives in ``_normalize_*``
helpers that the SQL layer mirrors.
"""
from __future__ import annotations
import json
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
def _iso(value: datetime | str | None) -> str | None:
if value is None:
return None
if isinstance(value, datetime):
return value.isoformat()
return str(value)
def _parse_dt(value: Any) -> datetime | None:
if value is None or isinstance(value, datetime):
return value
try:
return datetime.fromisoformat(str(value))
except ValueError:
return None
def _dump(value: Any) -> str | None:
return None if value is None else json.dumps(value, ensure_ascii=False)
def _load(value: str | None, default: Any) -> Any:
if value is None or value == "":
return default
try:
return json.loads(value)
except (TypeError, ValueError):
return default
class MemoryTaskAnalysisStore(TaskAnalysisStore):
def __init__(self) -> None:
self._tasks: dict[str, dict[str, Any]] = {}
self._settings: dict[str, str] = {}
self._bindings: dict[str, dict[str, Any]] = {} # keyed by external_task_id
self._runs: dict[str, dict[str, Any]] = {}
self._messages: list[dict[str, Any]] = []
self._versions: dict[str, dict[str, Any]] = {} # keyed by run_id
# 数据交付:块定义 / 期刻度 / 合并块 / 提交流水。
self._block_defs: dict[str, dict[str, dict[str, Any]]] = {} # task_id -> block_key -> def
self._periods: dict[str, list[dict[str, Any]]] = {} # task_id -> ordered by period_index
self._task_blocks: dict[tuple[str, int, str], dict[str, Any]] = {} # (task, period, block)
self._submissions: dict[tuple[str, int, str], list[dict[str, Any]]] = {}
self._seq = 0
# 绑定最近写入序号(时钟精度不足时仍能稳定判定“最近一次绑定”)。
self._binding_rev = 0
def _next_id(self, prefix: str) -> str:
self._seq += 1
return f"{prefix}-{self._seq:06d}"
# ── data delivery helpers ───────────────────────────────────────────────
def _seed_block_defs(self, task_id: str) -> None:
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
existing = self._block_defs.setdefault(task_id, {})
now = datetime.now(UTC).isoformat()
for definition in default_block_defs():
if definition["block_key"] in existing:
continue
existing[definition["block_key"]] = {
"id": self._next_id("tdbd"),
"task_id": task_id,
**definition,
"created_at": now,
"updated_at": now,
}
def _ordered_defs(self, task_id: str) -> list[dict[str, Any]]:
rows = list(self._block_defs.get(task_id, {}).values())
rows.sort(key=lambda d: (d.get("sort_order", 0), d.get("block_key", "")))
return [dict(row) for row in rows]
def _period_rows(self, task_id: str) -> list[dict[str, Any]]:
rows = list(self._periods.get(task_id, []))
rows.sort(key=lambda p: p["period_index"])
return rows
# ── tasks ───────────────────────────────────────────────────────────────
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
source = str(data["source_task_id"])
if await self.get_task_by_source(source) is not None:
raise ValueError(f"source_task_id already exists: {source}")
now = datetime.now(UTC).isoformat()
task = {
"id": self._next_id("ta"),
"source_task_id": source,
"name": str(data.get("name") or ""),
"description": data.get("description"),
"created_at": now,
"updated_at": now,
}
self._tasks[task["id"]] = dict(task)
# 建任务即种 6 条数据块定义(块 key 定死,标题后续可改)。
self._seed_block_defs(task["id"])
return dict(task)
async def get_task(self, task_id: str) -> dict[str, Any] | None:
task = self._tasks.get(task_id)
return dict(task) if task else None
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None:
for task in self._tasks.values():
if task["source_task_id"] == source_task_id:
return dict(task)
return None
async def list_tasks(self) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for task in sorted(self._tasks.values(), key=lambda t: t["created_at"], reverse=True):
row = dict(task)
row["binding"] = dict(self._bindings[task["id"]]) if task["id"] in self._bindings else None
out.append(row)
return out
async def delete_task_cascade(self, task_id: str) -> bool:
if task_id not in self._tasks:
return False
del self._tasks[task_id]
self._bindings.pop(task_id, None)
run_ids = [run_id for run_id, run in self._runs.items() if run["external_task_id"] == task_id]
for run_id in run_ids:
del self._runs[run_id]
self._versions.pop(run_id, None)
self._messages = [m for m in self._messages if m["run_id"] not in set(run_ids)]
self._block_defs.pop(task_id, None)
self._periods.pop(task_id, None)
self._task_blocks = {
key: row for key, row in self._task_blocks.items() if key[0] != task_id
}
self._submissions = {
key: rows for key, rows in self._submissions.items() if key[0] != task_id
}
return True
# ── bindings ────────────────────────────────────────────────────────────
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None:
binding = self._bindings.get(external_task_id)
return dict(binding) if binding else None
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None:
for binding in self._bindings.values():
if binding["id"] == binding_id:
return dict(binding)
return None
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
existing = self._bindings.get(external_task_id)
now = datetime.now(UTC).isoformat()
self._binding_rev += 1
if existing is None:
row = {
"id": self._next_id("tab"),
"external_task_id": external_task_id,
"chain_id": str(data["chain_id"]),
"chain_snapshot": data.get("chain_snapshot") or {},
"cron_expr": str(data.get("cron_expr") or ""),
"timezone": str(data.get("timezone") or "Asia/Shanghai"),
"schedule_type": str(data.get("schedule_type") or "cron"),
"enabled": bool(data.get("enabled", True)),
"next_run_at": _iso(data.get("next_run_at")),
"coordinator_prompt": data.get("coordinator_prompt"),
"direction_notes": list(data.get("direction_notes") or []),
"last_run_at": None,
"last_status": None,
"model": data.get("model"),
"created_by": data.get("created_by"),
"created_at": now,
"updated_at": now,
"_rev": self._binding_rev,
}
else:
row = dict(existing)
row["_rev"] = self._binding_rev
row.update({k: v for k, v in data.items() if k in {
"chain_id", "chain_snapshot", "cron_expr", "timezone", "schedule_type", "enabled",
"next_run_at", "coordinator_prompt", "model",
}})
if "direction_notes" in data:
row["direction_notes"] = list(data["direction_notes"])
row["updated_at"] = now
self._bindings[external_task_id] = row
return dict(row)
async def delete_binding(self, external_task_id: str) -> bool:
return self._bindings.pop(external_task_id, None) is not None
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
due = []
for binding in self._bindings.values():
if not binding.get("enabled"):
continue
next_at = _parse_dt(binding.get("next_run_at"))
if next_at is not None and next_at <= now:
due.append(binding)
due.sort(key=lambda b: b.get("next_run_at") or "")
return [dict(b) for b in due[:limit]]
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
for key, binding in self._bindings.items():
if binding["id"] == binding_id:
binding.update({k: v for k, v in patch.items() if k != "id"})
binding["updated_at"] = datetime.now(UTC).isoformat()
return dict(binding)
return None
async def last_bound_chain_id(self) -> str | None:
if not self._bindings:
return None
latest = max(self._bindings.values(), key=lambda b: b.get("_rev", 0))
return latest.get("chain_id")
# ── runs ────────────────────────────────────────────────────────────────
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
binding_id = data["binding_id"]
scheduled_for = _parse_dt(data["scheduled_for"]) or datetime.now(UTC)
for run in self._runs.values():
if run["binding_id"] == binding_id and run["scheduled_for"] == scheduled_for.isoformat():
return dict(run)
now = datetime.now(UTC).isoformat()
row = {
"id": self._next_id("tar"),
"binding_id": binding_id,
"external_task_id": str(data["external_task_id"]),
"trigger": str(data.get("trigger") or "schedule"),
"scheduled_for": scheduled_for.isoformat(),
"started_at": now,
"finished_at": None,
"status": str(data.get("status") or "pending"),
"stages_snapshot": data.get("stages_snapshot") or [],
"stage_goals_snapshot": data.get("stage_goals_snapshot") or [],
"seat_names_snapshot": data.get("seat_names_snapshot") or {},
"seat_results": data.get("seat_results") or {},
"error": data.get("error"),
"retry_count": 0,
"created_at": now,
"updated_at": now,
}
self._runs[row["id"]] = row
return dict(row)
async def get_run(self, run_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
return dict(run) if run else None
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
rows = [r for r in self._runs.values() if r["external_task_id"] == external_task_id]
rows.sort(key=lambda r: r["started_at"], reverse=True)
return [dict(r) for r in rows[:limit]]
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None:
return None
run.update({k: v for k, v in patch.items() if k not in {"id", "binding_id"}})
run["updated_at"] = datetime.now(UTC).isoformat()
return dict(run)
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
out = []
for run in self._runs.values():
if run["status"] in {"pending", "running"}:
started = _parse_dt(run["started_at"])
if started is not None and started < older_than:
out.append(run)
out.sort(key=lambda r: r["started_at"])
return [dict(r) for r in out[:limit]]
# ── messages ────────────────────────────────────────────────────────────
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
base_seq = sum(1 for m in self._messages if m["run_id"] == run_id)
added = 0
for index, message in enumerate(messages):
self._messages.append({
"id": self._next_id("tam"),
"run_id": run_id,
"seq": base_seq + index,
"seat_agent_id": str(message.get("seat_agent_id") or ""),
"stage_index": int(message.get("stage_index") or 0),
"event_type": str(message.get("event_type") or "llm.ai"),
"content": message.get("content"),
"created_at": datetime.now(UTC).isoformat(),
})
added += 1
return added
async def list_run_messages(
self, run_id: str, *, seat: str | None = None, limit: int = 500
) -> list[dict[str, Any]]:
rows = [m for m in self._messages if m["run_id"] == run_id]
if seat is not None:
rows = [m for m in rows if m["seat_agent_id"] == seat]
rows.sort(key=lambda m: m["seq"])
return [dict(m) for m in rows[:limit]]
# ── versions ────────────────────────────────────────────────────────────
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
run_id = str(data["run_id"])
row = {
"id": self._next_id("tav"),
"external_task_id": str(data["external_task_id"]),
"run_id": run_id,
"blocks": data.get("blocks") or {},
"created_at": datetime.now(UTC).isoformat(),
}
self._versions[run_id] = row
return dict(row)
async def list_versions(
self, external_task_id: str, *, limit: int = 50, offset: int = 0
) -> list[dict[str, Any]]:
rows = [v for v in self._versions.values() if v["external_task_id"] == external_task_id]
rows.sort(key=lambda v: v["created_at"], reverse=True)
return [dict(v) for v in rows[offset : offset + limit]]
# ── data delivery: block defs / periods / submissions ────────────────────
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
self._seed_block_defs(task_id)
return self._ordered_defs(task_id)
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
return self._ordered_defs(task_id)
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
definitions = self._block_defs.setdefault(task_id, {})
now = datetime.now(UTC).isoformat()
for update in updates:
block_key = str(update.get("block_key") or "")
row = definitions.get(block_key)
if row is None:
raise ValueError(f"未知数据块:{block_key}")
if update.get("title") is not None:
row["title"] = str(update["title"])
if update.get("sort_order") is not None:
row["sort_order"] = int(update["sort_order"])
row["updated_at"] = now
return self._ordered_defs(task_id)
async def open_period(
self,
task_id: str,
*,
source: str,
run_id: str | None = None,
opened_at: datetime | None = None,
) -> dict[str, Any]:
if run_id:
for period in self._periods.get(task_id, []):
if period.get("run_id") == run_id:
return dict(period)
rows = self._periods.setdefault(task_id, [])
index = (rows[-1]["period_index"] if rows else 0) + 1
now = datetime.now(UTC).isoformat()
row = {
"id": self._next_id("tdp"),
"task_id": task_id,
"period_index": index,
"opened_at": _iso(opened_at) or now,
"trigger": str(source or "cron"),
"run_id": run_id,
"created_at": now,
}
rows.append(row)
return dict(row)
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None:
for period in self._period_rows(task_id):
if period["period_index"] == period_index:
return dict(period)
return None
async def current_period(self, task_id: str) -> dict[str, Any] | None:
rows = self._period_rows(task_id)
return dict(rows[-1]) if rows else None
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
rows = self._period_rows(task_id)
return [dict(p) for p in rows[-limit:]]
async def append_submission(
self,
task_id: str,
period_index: int,
block_key: str,
payload: dict[str, Any],
*,
submitted_by: str | None = None,
) -> dict[str, Any]:
key = (task_id, period_index, block_key)
rows = self._submissions.setdefault(key, [])
row = {
"id": self._next_id("tds"),
"task_id": task_id,
"period_index": period_index,
"block_key": block_key,
"payload": dict(payload),
"submitted_by": submitted_by,
"seq": len(rows),
"created_at": datetime.now(UTC).isoformat(),
}
rows.append(row)
return dict(row)
async def list_submissions(
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
) -> list[dict[str, Any]]:
rows = list(self._submissions.get((task_id, period_index, block_key), []))
rows.sort(key=lambda s: s["seq"])
return [dict(s) for s in rows[:limit]]
async def upsert_period_block(
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
) -> dict[str, Any]:
key = (task_id, period_index, block_key)
now = datetime.now(UTC).isoformat()
row = self._task_blocks.get(key)
if row is None:
row = {
"id": self._next_id("tdb"),
"task_id": task_id,
"period_index": period_index,
"block_key": block_key,
"payload": dict(payload),
"updated_at": now,
"created_at": now,
}
self._task_blocks[key] = row
else:
row["payload"] = dict(payload)
row["updated_at"] = now
return dict(row)
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
rows = [dict(row) for key, row in self._task_blocks.items() if key[0] == task_id]
rows.sort(key=lambda r: (r["period_index"], r["block_key"]))
return rows[:limit]
# ── settings ────────────────────────────────────────────────────────────
async def get_setting(self, key: str) -> str | None:
return self._settings.get(key)
async def set_setting(self, key: str, value: str) -> None:
self._settings[key] = value
@@ -0,0 +1,217 @@
"""ORM models for the external-task multi-agent scheduled analysis (任务分析定时).
Feature summary (design doc in the frontend ``official-task-analysis`` module):
an external system pushes tasks via our create API; each task binds 1:1 to a
business-chain orchestration plus a cron schedule; the scheduler runs the chain
stage-by-stage (seats within a stage in parallel) and persists per-seat process
messages plus a content "timeline" version per run. Manual runs wrap the
roundtable Step-2 coordinator Q&A and are stored with the same message table
(``seat_agent_id`` distinguishes seats; the coordinator uses ``__coordinator__``).
Tables:
- ``task_analysis_tasks`` — external tasks (source_task_id unique)
- ``task_analysis_bindings`` — 1:1 binding (chain snapshot + cron + notes)
- ``task_analysis_runs`` — executions (trigger: schedule | manual)
- ``task_analysis_run_messages`` — per-seat process messages for a run
- ``task_analysis_content_versions`` — one timeline version per run
- ``task_data_block_defs`` — the 5 fixed data blocks per task (title/sort editable)
- ``task_data_periods`` — data-delivery timeline ticks (period_index per task)
- ``task_data_blocks`` — merged payload per (task, period, block)
- ``task_data_submissions`` — append-only submission log feeding the merge
JSON payloads are stored as LONGTEXT-backed JSON strings (PortableLongText),
mirroring the sibling roundtable persistence modules for headroom.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
def _utcnow() -> datetime:
return datetime.now(UTC)
class TaskAnalysisTaskRow(Base):
__tablename__ = "task_analysis_tasks"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# 对方系统任务号(创建接口传入;唯一)。
source_task_id: Mapped[str] = mapped_column(String(128), nullable=False, unique=True, index=True)
name: Mapped[str] = mapped_column(String(512), nullable=False, default="")
# 任务简介:创建时落库的存量即执行时用的版本(不回拉对方接口)。
description: Mapped[str | None] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
class TaskAnalysisBindingRow(Base):
__tablename__ = "task_analysis_bindings"
__table_args__ = (UniqueConstraint("external_task_id", name="uq_task_analysis_binding_task"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# 引用 roundtable_chains.id;执行前校验,链缺失/停用 → 跳过并标记 chain_missing。
chain_id: Mapped[str] = mapped_column(String(64), nullable=False)
# 执行编排快照 JSON:{seats, stages, stage_goals, coordinator_prompt}。
# 每次执行前刷新(跟随链最新版),历史 run 用自己的 stages_snapshot 回放。
chain_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
cron_expr: Mapped[str] = mapped_column(String(64), nullable=False, default="")
timezone: Mapped[str] = mapped_column(String(64), nullable=False, default="Asia/Shanghai")
# 执行方式:cron = 按频率定时;once = 只执行一次(创建即执行,next_run_at 恒空)。
schedule_type: Mapped[str] = mapped_column(String(16), nullable=False, default="cron", server_default="cron")
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="1")
next_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
# 绑定级总控提示(与链内 coordinator_prompt 拼接使用)。
coordinator_prompt: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# 问答沉淀的「执行方向备注」JSON 数组(逐条追加,注入后续每次定时执行)。
direction_notes: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
last_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
# 执行模型(席位 run 的 model_name;NULL = 系统/席位默认)。
model: Mapped[str | None] = mapped_column(String(128), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
class TaskAnalysisRunRow(Base):
__tablename__ = "task_analysis_runs"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
binding_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# schedule = cron 定时执行;manual = 总控问答(对方详情页发起)。
trigger: Mapped[str] = mapped_column(String(16), nullable=False, default="schedule")
# 幂等键:同 binding + scheduled_for 只建一条(对齐 scheduled_tasks.create_run_once)。
scheduled_for: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False)
started_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
# pending | running | succeeded | failed | partial
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
# 本次编排快照 JSON:[[seat_id,...],...](回放不受链后续修改影响)。
stages_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
stage_goals_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
seat_names_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# 各席位结果 JSON:{agent_id: {state, duration_ms, thread_id, error}}。
seat_results: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
error: Mapped[str | None] = mapped_column(Text, nullable=True)
retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
class TaskAnalysisRunMessageRow(Base):
__tablename__ = "task_analysis_run_messages"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# 消息归属席位(agent_id);总控消息用固定标识 __coordinator__。
seat_agent_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
stage_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
# input | llm.ai | tool.result | artifact(对齐现有 run messages 事件分类)。
event_type: Mapped[str] = mapped_column(String(32), nullable=False, default="llm.ai")
content: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
class TaskAnalysisContentVersionRow(Base):
__tablename__ = "task_analysis_content_versions"
__table_args__ = (UniqueConstraint("run_id", name="uq_task_analysis_version_run"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# 一次执行 = 一期时间线版本(1:1)。
run_id: Mapped[str] = mapped_column(String(64), nullable=False)
# JSON:{key: {name, seat_agent_id, content, format}} —— 对方详情的各块内容。
blocks: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
class TaskDataBlockDefRow(Base):
"""数据块定义:块 key 定死 block_1~block_6,标题默认「数据块一~六」可改。"""
__tablename__ = "task_data_block_defs"
__table_args__ = (UniqueConstraint("task_id", "block_key", name="uq_task_data_block_def_key"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
# 关联 task_analysis_tasks.id(内部任务 id)。
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
title: Mapped[str] = mapped_column(String(128), nullable=False, default="")
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
class TaskDataPeriodRow(Base):
"""数据交付时间轴刻度:每个任务 period_index 从 1 递增;run_id 作幂等锚点。"""
__tablename__ = "task_data_periods"
__table_args__ = (
UniqueConstraint("task_id", "period_index", name="uq_task_data_period_index"),
UniqueConstraint("run_id", name="uq_task_data_period_run"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
opened_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
# cron = 定时执行开期;manual = 手动触发/总控问答开期;external = 首次提交兜底开期。
trigger: Mapped[str] = mapped_column(String(16), nullable=False, default="cron")
# 触发本期的执行记录(问答/定时执行共用 run 表);提交兜底开的期没有 run。
run_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
class TaskDataBlockRow(Base):
"""每期每块的合并结果(由 task_data_submissions 流水按序合并得出)。"""
__tablename__ = "task_data_blocks"
__table_args__ = (
UniqueConstraint("task_id", "period_index", "block_key", name="uq_task_data_block_tpb"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
# JSON 对象(顶层 key 合并、同 key 后写覆盖)。
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
class TaskDataSubmissionRow(Base):
"""入库流水(追加只读):块 payload 的一致性来源。"""
__tablename__ = "task_data_submissions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
# 本次提交的 JSON 对象片段(合并语义:顶层 key 合并、同 key 后写覆盖)。
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# 提交方(席位 agent id / 用户 id,仅审计)。
submitted_by: Mapped[str | None] = mapped_column(String(128), nullable=True)
# 同 (task, period, block) 内的提交序号,从 0 递增 —— 合并顺序依据。
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
class TaskAnalysisSettingRow(Base):
"""模块级 KV 设置(当前仅默认业务链)。"""
__tablename__ = "task_analysis_settings"
key: Mapped[str] = mapped_column(String(64), primary_key=True)
value: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
@@ -0,0 +1,802 @@
"""SQLAlchemy-backed task-analysis store.
Mirrors the memory implementation's normalisation: JSON payloads
(chain_snapshot / stages / seat_results / blocks / direction_notes) are stored
as LONGTEXT JSON strings; datetimes are ISO strings in dict form.
"""
from __future__ import annotations
import json
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
from deerflow.persistence.task_analysis.model import (
TaskAnalysisBindingRow,
TaskAnalysisContentVersionRow,
TaskAnalysisRunMessageRow,
TaskAnalysisRunRow,
TaskAnalysisSettingRow,
TaskAnalysisTaskRow,
TaskDataBlockDefRow,
TaskDataBlockRow,
TaskDataPeriodRow,
TaskDataSubmissionRow,
)
def _iso(value: datetime | str | None) -> str | None:
if value is None:
return None
if isinstance(value, datetime):
return value.isoformat()
return str(value)
def _utcnow() -> datetime:
return datetime.now(UTC)
def _parse_dt(value: Any) -> datetime | None:
if value is None or isinstance(value, datetime):
return value
try:
return datetime.fromisoformat(str(value))
except ValueError:
return None
def _dump(value: Any) -> str | None:
return None if value is None else json.dumps(value, ensure_ascii=False)
def _load(value: str | None, default: Any) -> Any:
if value is None or value == "":
return default
try:
return json.loads(value)
except (TypeError, ValueError):
return default
def _task_dict(row: TaskAnalysisTaskRow) -> dict[str, Any]:
data = row.to_dict()
data["created_at"] = _iso(row.created_at)
data["updated_at"] = _iso(row.updated_at)
return data
def _binding_dict(row: TaskAnalysisBindingRow) -> dict[str, Any]:
data = row.to_dict()
data["chain_snapshot"] = _load(row.chain_snapshot, {})
data["direction_notes"] = _load(row.direction_notes, [])
for key in ("next_run_at", "last_run_at", "created_at", "updated_at"):
data[key] = _iso(getattr(row, key))
return data
def _run_dict(row: TaskAnalysisRunRow) -> dict[str, Any]:
data = row.to_dict()
data["scheduled_for"] = _iso(row.scheduled_for)
data["started_at"] = _iso(row.started_at)
data["finished_at"] = _iso(row.finished_at)
data["created_at"] = _iso(row.created_at)
data["updated_at"] = _iso(row.updated_at)
for key, default in (
("stages_snapshot", []),
("stage_goals_snapshot", []),
("seat_names_snapshot", {}),
("seat_results", {}),
):
data[key] = _load(getattr(row, key), default)
return data
def _message_dict(row: TaskAnalysisRunMessageRow) -> dict[str, Any]:
data = row.to_dict()
data["created_at"] = _iso(row.created_at)
return data
def _version_dict(row: TaskAnalysisContentVersionRow) -> dict[str, Any]:
data = row.to_dict()
data["blocks"] = _load(row.blocks, {})
data["created_at"] = _iso(row.created_at)
return data
def _block_def_dict(row: TaskDataBlockDefRow) -> dict[str, Any]:
data = row.to_dict()
data["created_at"] = _iso(row.created_at)
data["updated_at"] = _iso(row.updated_at)
return data
def _period_dict(row: TaskDataPeriodRow) -> dict[str, Any]:
data = row.to_dict()
data["opened_at"] = _iso(row.opened_at)
data["created_at"] = _iso(row.created_at)
return data
def _task_block_dict(row: TaskDataBlockRow) -> dict[str, Any]:
data = row.to_dict()
data["payload"] = _load(row.payload, {})
data["updated_at"] = _iso(row.updated_at)
data["created_at"] = _iso(row.created_at)
return data
def _submission_dict(row: TaskDataSubmissionRow) -> dict[str, Any]:
data = row.to_dict()
data["payload"] = _load(row.payload, {})
data["created_at"] = _iso(row.created_at)
return data
class TaskAnalysisRepository(TaskAnalysisStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# ── tasks ───────────────────────────────────────────────────────────────
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
source = str(data["source_task_id"])
async with self._sf() as session:
existing = await session.execute(
select(TaskAnalysisTaskRow).where(TaskAnalysisTaskRow.source_task_id == source)
)
if existing.scalar_one_or_none() is not None:
raise ValueError(f"source_task_id already exists: {source}")
row = TaskAnalysisTaskRow(
id=str(data.get("id") or f"ta-{source}"),
source_task_id=source,
name=str(data.get("name") or ""),
description=data.get("description"),
)
session.add(row)
# 建任务即种 6 条数据块定义(块 key 定死,标题后续可改)。
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
for definition in default_block_defs():
session.add(
TaskDataBlockDefRow(
id=f"tdbd-{row.id}-{definition['block_key']}",
task_id=row.id,
block_key=definition["block_key"],
title=definition["title"],
sort_order=definition["sort_order"],
)
)
await session.commit()
await session.refresh(row)
return _task_dict(row)
async def get_task(self, task_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(TaskAnalysisTaskRow, task_id)
return _task_dict(row) if row else None
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisTaskRow).where(TaskAnalysisTaskRow.source_task_id == source_task_id)
)
row = result.scalar_one_or_none()
return _task_dict(row) if row else None
async def list_tasks(self) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisTaskRow).order_by(TaskAnalysisTaskRow.created_at.desc())
)
tasks = [_task_dict(row) for row in result.scalars()]
bindings = await session.execute(select(TaskAnalysisBindingRow))
by_task: dict[str, dict[str, Any]] = {}
for row in bindings.scalars():
by_task[row.external_task_id] = _binding_dict(row)
for task in tasks:
task["binding"] = by_task.get(task["id"])
return tasks
async def delete_task_cascade(self, task_id: str) -> bool:
async with self._sf() as session:
row = await session.get(TaskAnalysisTaskRow, task_id)
if row is None:
return False
run_ids = (
(
await session.execute(
select(TaskAnalysisRunRow.id).where(TaskAnalysisRunRow.external_task_id == task_id)
)
)
.scalars()
.all()
)
await session.execute(
delete(TaskAnalysisBindingRow).where(TaskAnalysisBindingRow.external_task_id == task_id)
)
await session.execute(
delete(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
)
await session.execute(
delete(TaskDataPeriodRow).where(TaskDataPeriodRow.task_id == task_id)
)
await session.execute(
delete(TaskDataBlockRow).where(TaskDataBlockRow.task_id == task_id)
)
await session.execute(
delete(TaskDataSubmissionRow).where(TaskDataSubmissionRow.task_id == task_id)
)
if run_ids:
await session.execute(
delete(TaskAnalysisRunMessageRow).where(TaskAnalysisRunMessageRow.run_id.in_(run_ids))
)
await session.execute(
delete(TaskAnalysisContentVersionRow).where(
TaskAnalysisContentVersionRow.run_id.in_(run_ids)
)
)
await session.execute(
delete(TaskAnalysisRunRow).where(TaskAnalysisRunRow.external_task_id == task_id)
)
await session.delete(row)
await session.commit()
return True
# ── bindings ────────────────────────────────────────────────────────────
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisBindingRow).where(
TaskAnalysisBindingRow.external_task_id == external_task_id
)
)
row = result.scalar_one_or_none()
return _binding_dict(row) if row else None
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(TaskAnalysisBindingRow, binding_id)
return _binding_dict(row) if row else None
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisBindingRow).where(
TaskAnalysisBindingRow.external_task_id == external_task_id
)
)
row = result.scalar_one_or_none()
if row is None:
row = TaskAnalysisBindingRow(
id=str(data.get("id") or f"tab-{external_task_id}"),
external_task_id=external_task_id,
chain_id=str(data["chain_id"]),
created_by=data.get("created_by"),
)
session.add(row)
row.chain_id = str(data.get("chain_id") or row.chain_id)
row.chain_snapshot = _dump(data.get("chain_snapshot") or {})
row.cron_expr = str(data.get("cron_expr") or row.cron_expr or "")
row.timezone = str(data.get("timezone") or row.timezone or "Asia/Shanghai")
if "schedule_type" in data:
row.schedule_type = str(data.get("schedule_type") or "cron")
row.model = data.get("model")
if "enabled" in data:
row.enabled = bool(data["enabled"])
if "next_run_at" in data:
row.next_run_at = _parse_dt(data.get("next_run_at"))
row.coordinator_prompt = data.get("coordinator_prompt")
if "direction_notes" in data:
row.direction_notes = _dump(list(data["direction_notes"] or []))
await session.commit()
await session.refresh(row)
return _binding_dict(row)
async def delete_binding(self, external_task_id: str) -> bool:
async with self._sf() as session:
result = await session.execute(
delete(TaskAnalysisBindingRow).where(
TaskAnalysisBindingRow.external_task_id == external_task_id
)
)
await session.commit()
return bool(result.rowcount)
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisBindingRow)
.where(
TaskAnalysisBindingRow.enabled.is_(True),
TaskAnalysisBindingRow.next_run_at.is_not(None),
TaskAnalysisBindingRow.next_run_at <= now,
)
.order_by(TaskAnalysisBindingRow.next_run_at.asc())
.limit(limit)
)
return [_binding_dict(row) for row in result.scalars()]
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
allowed = {
"chain_id", "cron_expr", "timezone", "schedule_type", "enabled", "next_run_at",
"last_run_at", "last_status", "coordinator_prompt", "model",
}
values: dict[str, Any] = {k: v for k, v in patch.items() if k in allowed}
if "next_run_at" in values:
values["next_run_at"] = _parse_dt(values["next_run_at"])
if "last_run_at" in values:
values["last_run_at"] = _parse_dt(values["last_run_at"])
if "direction_notes" in patch:
values["direction_notes"] = _dump(list(patch["direction_notes"] or []))
if not values:
row = await self._get_binding_row(binding_id)
return _binding_dict(row) if row else None
async with self._sf() as session:
await session.execute(
update(TaskAnalysisBindingRow).where(TaskAnalysisBindingRow.id == binding_id).values(**values)
)
await session.commit()
row = await session.get(TaskAnalysisBindingRow, binding_id)
return _binding_dict(row) if row else None
async def _get_binding_row(self, binding_id: str) -> TaskAnalysisBindingRow | None:
async with self._sf() as session:
return await session.get(TaskAnalysisBindingRow, binding_id)
async def last_bound_chain_id(self) -> str | None:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisBindingRow.chain_id)
.order_by(TaskAnalysisBindingRow.updated_at.desc())
.limit(1)
)
return result.scalar_one_or_none()
# ── runs ────────────────────────────────────────────────────────────────
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
binding_id = str(data["binding_id"])
scheduled_for = _parse_dt(data["scheduled_for"]) or datetime.now(UTC)
import uuid as _uuid
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisRunRow).where(
TaskAnalysisRunRow.binding_id == binding_id,
TaskAnalysisRunRow.scheduled_for == scheduled_for,
)
)
existing = result.scalar_one_or_none()
if existing is not None:
return _run_dict(existing)
row = TaskAnalysisRunRow(
id=str(data.get("id") or f"tar-{_uuid.uuid4()}"),
binding_id=binding_id,
external_task_id=str(data["external_task_id"]),
trigger=str(data.get("trigger") or "schedule"),
scheduled_for=scheduled_for,
status=str(data.get("status") or "pending"),
stages_snapshot=_dump(data.get("stages_snapshot") or []),
stage_goals_snapshot=_dump(data.get("stage_goals_snapshot") or []),
seat_names_snapshot=_dump(data.get("seat_names_snapshot") or {}),
seat_results=_dump(data.get("seat_results") or {}),
)
session.add(row)
await session.commit()
await session.refresh(row)
return _run_dict(row)
async def get_run(self, run_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(TaskAnalysisRunRow, run_id)
return _run_dict(row) if row else None
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisRunRow)
.where(TaskAnalysisRunRow.external_task_id == external_task_id)
.order_by(TaskAnalysisRunRow.started_at.desc())
.limit(limit)
)
return [_run_dict(row) for row in result.scalars()]
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
allowed = {
"trigger", "status", "finished_at", "error", "retry_count",
"stages_snapshot", "stage_goals_snapshot", "seat_names_snapshot", "seat_results",
}
values: dict[str, Any] = {}
for key, value in patch.items():
if key not in allowed:
continue
if key == "finished_at":
values[key] = _parse_dt(value)
elif key in {"stages_snapshot", "stage_goals_snapshot", "seat_names_snapshot", "seat_results"}:
values[key] = _dump(value)
else:
values[key] = value
if not values:
row = await self._get_run_row(run_id)
return _run_dict(row) if row else None
async with self._sf() as session:
await session.execute(
update(TaskAnalysisRunRow).where(TaskAnalysisRunRow.id == run_id).values(**values)
)
await session.commit()
row = await session.get(TaskAnalysisRunRow, run_id)
return _run_dict(row) if row else None
async def _get_run_row(self, run_id: str) -> TaskAnalysisRunRow | None:
async with self._sf() as session:
return await session.get(TaskAnalysisRunRow, run_id)
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisRunRow)
.where(
TaskAnalysisRunRow.status.in_(("pending", "running")),
TaskAnalysisRunRow.started_at < older_than,
)
.order_by(TaskAnalysisRunRow.started_at.asc())
.limit(limit)
)
return [_run_dict(row) for row in result.scalars()]
# ── messages ────────────────────────────────────────────────────────────
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
import uuid as _uuid
async with self._sf() as session:
existing = (
(
await session.execute(
select(TaskAnalysisRunMessageRow.id).where(
TaskAnalysisRunMessageRow.run_id == run_id
)
)
)
.scalars()
.all()
)
base_seq = len(existing)
rows = [
TaskAnalysisRunMessageRow(
id=str(_uuid.uuid4()),
run_id=run_id,
seq=base_seq + index,
seat_agent_id=str(message.get("seat_agent_id") or ""),
stage_index=int(message.get("stage_index") or 0),
event_type=str(message.get("event_type") or "llm.ai"),
content=message.get("content"),
)
for index, message in enumerate(messages)
]
session.add_all(rows)
await session.commit()
return len(rows)
async def list_run_messages(
self, run_id: str, *, seat: str | None = None, limit: int = 500
) -> list[dict[str, Any]]:
async with self._sf() as session:
stmt = (
select(TaskAnalysisRunMessageRow)
.where(TaskAnalysisRunMessageRow.run_id == run_id)
.order_by(TaskAnalysisRunMessageRow.seq.asc())
.limit(limit)
)
if seat is not None:
stmt = stmt.where(TaskAnalysisRunMessageRow.seat_agent_id == seat)
result = await session.execute(stmt)
return [_message_dict(row) for row in result.scalars()]
# ── versions ────────────────────────────────────────────────────────────
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
import uuid as _uuid
run_id = str(data["run_id"])
async with self._sf() as session:
await session.execute(
delete(TaskAnalysisContentVersionRow).where(
TaskAnalysisContentVersionRow.run_id == run_id
)
)
row = TaskAnalysisContentVersionRow(
id=str(_uuid.uuid4()),
external_task_id=str(data["external_task_id"]),
run_id=run_id,
blocks=_dump(data.get("blocks") or {}),
)
session.add(row)
await session.commit()
await session.refresh(row)
return _version_dict(row)
async def list_versions(
self, external_task_id: str, *, limit: int = 50, offset: int = 0
) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskAnalysisContentVersionRow)
.where(TaskAnalysisContentVersionRow.external_task_id == external_task_id)
.order_by(TaskAnalysisContentVersionRow.created_at.desc())
.offset(offset)
.limit(limit)
)
return [_version_dict(row) for row in result.scalars()]
# ── data delivery: block defs / periods / submissions ────────────────────
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
async with self._sf() as session:
result = await session.execute(
select(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
)
existing_keys = {row.block_key for row in result.scalars()}
missing = [
definition
for definition in default_block_defs()
if definition["block_key"] not in existing_keys
]
if not missing:
return await self._ordered_defs(session, task_id)
for definition in missing:
session.add(
TaskDataBlockDefRow(
id=f"tdbd-{task_id}-{definition['block_key']}",
task_id=task_id,
block_key=definition["block_key"],
title=definition["title"],
sort_order=definition["sort_order"],
)
)
try:
await session.commit()
except IntegrityError:
# 并发补种:另一个请求先种好了,沿用其结果。
await session.rollback()
return await self._ordered_defs(session, task_id)
@staticmethod
async def _ordered_defs(session: AsyncSession, task_id: str) -> list[dict[str, Any]]:
result = await session.execute(
select(TaskDataBlockDefRow)
.where(TaskDataBlockDefRow.task_id == task_id)
.order_by(TaskDataBlockDefRow.sort_order.asc(), TaskDataBlockDefRow.block_key.asc())
)
return [_block_def_dict(row) for row in result.scalars()]
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
async with self._sf() as session:
return await self._ordered_defs(session, task_id)
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
)
rows = {row.block_key: row for row in result.scalars()}
touched = False
for item in updates:
block_key = str(item.get("block_key") or "")
row = rows.get(block_key)
if row is None:
raise ValueError(f"未知数据块:{block_key}")
patch: dict[str, Any] = {}
if item.get("title") is not None:
patch["title"] = str(item["title"])
if item.get("sort_order") is not None:
patch["sort_order"] = int(item["sort_order"])
if patch:
await session.execute(
update(TaskDataBlockDefRow)
.where(TaskDataBlockDefRow.id == row.id)
.values(**patch)
)
touched = True
if touched:
await session.commit()
return await self._ordered_defs(session, task_id)
async def open_period(
self,
task_id: str,
*,
source: str,
run_id: str | None = None,
opened_at: datetime | None = None,
) -> dict[str, Any]:
import uuid as _uuid
for _attempt in range(3):
async with self._sf() as session:
if run_id:
existing = await session.execute(
select(TaskDataPeriodRow).where(TaskDataPeriodRow.run_id == run_id)
)
row = existing.scalar_one_or_none()
if row is not None:
return _period_dict(row)
max_index = (
await session.execute(
select(func.max(TaskDataPeriodRow.period_index)).where(
TaskDataPeriodRow.task_id == task_id
)
)
).scalar()
row = TaskDataPeriodRow(
id=f"tdp-{_uuid.uuid4()}",
task_id=task_id,
period_index=(max_index or 0) + 1,
opened_at=opened_at or _utcnow(),
trigger=str(source or "cron"),
run_id=run_id,
)
session.add(row)
try:
await session.commit()
except IntegrityError:
# 并发开期撞了期号唯一键(或 run_id 幂等锚点)——重查/重试。
await session.rollback()
continue
await session.refresh(row)
return _period_dict(row)
raise RuntimeError(f"open_period 连续冲突,task={task_id} source={source}")
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(TaskDataPeriodRow).where(
TaskDataPeriodRow.task_id == task_id,
TaskDataPeriodRow.period_index == period_index,
)
)
row = result.scalar_one_or_none()
return _period_dict(row) if row else None
async def current_period(self, task_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
result = await session.execute(
select(TaskDataPeriodRow)
.where(TaskDataPeriodRow.task_id == task_id)
.order_by(TaskDataPeriodRow.period_index.desc())
.limit(1)
)
row = result.scalar_one_or_none()
return _period_dict(row) if row else None
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskDataPeriodRow)
.where(TaskDataPeriodRow.task_id == task_id)
.order_by(TaskDataPeriodRow.period_index.desc())
.limit(limit)
)
rows = [_period_dict(row) for row in result.scalars()]
rows.reverse() # 时间轴语义:按期号升序返回
return rows
async def append_submission(
self,
task_id: str,
period_index: int,
block_key: str,
payload: dict[str, Any],
*,
submitted_by: str | None = None,
) -> dict[str, Any]:
import uuid as _uuid
async with self._sf() as session:
base_seq = (
(
await session.execute(
select(func.count(TaskDataSubmissionRow.id)).where(
TaskDataSubmissionRow.task_id == task_id,
TaskDataSubmissionRow.period_index == period_index,
TaskDataSubmissionRow.block_key == block_key,
)
)
)
.scalar_one()
)
row = TaskDataSubmissionRow(
id=f"tds-{_uuid.uuid4()}",
task_id=task_id,
period_index=period_index,
block_key=block_key,
payload=_dump(payload),
submitted_by=submitted_by,
seq=int(base_seq),
)
session.add(row)
await session.commit()
await session.refresh(row)
return _submission_dict(row)
async def list_submissions(
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskDataSubmissionRow)
.where(
TaskDataSubmissionRow.task_id == task_id,
TaskDataSubmissionRow.period_index == period_index,
TaskDataSubmissionRow.block_key == block_key,
)
.order_by(TaskDataSubmissionRow.seq.asc())
.limit(limit)
)
return [_submission_dict(row) for row in result.scalars()]
async def upsert_period_block(
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
) -> dict[str, Any]:
import uuid as _uuid
async with self._sf() as session:
result = await session.execute(
select(TaskDataBlockRow).where(
TaskDataBlockRow.task_id == task_id,
TaskDataBlockRow.period_index == period_index,
TaskDataBlockRow.block_key == block_key,
)
)
row = result.scalar_one_or_none()
if row is None:
row = TaskDataBlockRow(
id=f"tdb-{_uuid.uuid4()}",
task_id=task_id,
period_index=period_index,
block_key=block_key,
payload=_dump(payload),
)
session.add(row)
else:
row.payload = _dump(payload)
row.updated_at = _utcnow()
await session.commit()
await session.refresh(row)
return _task_block_dict(row)
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
async with self._sf() as session:
result = await session.execute(
select(TaskDataBlockRow)
.where(TaskDataBlockRow.task_id == task_id)
.order_by(TaskDataBlockRow.period_index.asc(), TaskDataBlockRow.block_key.asc())
.limit(limit)
)
return [_task_block_dict(row) for row in result.scalars()]
# ── settings ────────────────────────────────────────────────────────────
async def get_setting(self, key: str) -> str | None:
async with self._sf() as session:
row = await session.get(TaskAnalysisSettingRow, key)
return row.value if row else None
async def set_setting(self, key: str, value: str) -> None:
async with self._sf() as session:
row = await session.get(TaskAnalysisSettingRow, key)
if row is None:
session.add(TaskAnalysisSettingRow(key=key, value=value))
else:
row.value = value
await session.commit()
@@ -0,0 +1,34 @@
"""Dedicated taskId + goPath conversation-history index."""
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from langgraph.store.base import BaseStore
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.thread_meta.base import ThreadMetaStore
__all__ = ["make_task_deeplink_thread_store"]
def make_task_deeplink_thread_store(
session_factory: async_sessionmaker[AsyncSession] | None,
langgraph_store: BaseStore,
thread_store: ThreadMetaStore,
):
# Keep imports local: ThreadMetaStore's in-memory implementation imports
# the sync helpers from this package, so eager imports here would form a
# package-initialization cycle.
if session_factory is not None:
from deerflow.persistence.task_deeplink_threads.sql import (
TaskDeeplinkThreadRepository,
)
return TaskDeeplinkThreadRepository(session_factory)
from deerflow.persistence.task_deeplink_threads.memory import (
MemoryTaskDeeplinkThreadStore,
)
return MemoryTaskDeeplinkThreadStore(langgraph_store, thread_store)
@@ -0,0 +1,46 @@
"""Shared helpers for task deep-link conversation indexing.
The LangGraph thread/checkpoint remains the source of truth for messages and
execution state. This package only owns a small, dedicated lookup index so the
task workspaces never have to search/filter the global ``threads_meta`` table.
"""
from __future__ import annotations
from typing import Any, TypedDict
TASK_DEEPLINK_GO_PATHS = frozenset({"3qfx", "rwfx", "xdfx"})
class TaskDeeplinkBinding(TypedDict):
task_id: str
source_task_id: str
go_path: str
agent_id: str | None
def task_deeplink_binding(metadata: dict[str, Any] | None) -> TaskDeeplinkBinding | None:
"""Extract a valid dedicated-history binding from thread metadata.
``taskRecordId`` is the original deep-link ``?taskId=``. It equals the
actual task id for 3qfx/rwfx and the action id for xdfx. Older callers may
omit it, in which case ``taskId`` is a safe fallback. A ``taskGoPath`` is
mandatory, deliberately preventing legacy task rows from being migrated or
accidentally mixed into the new store.
"""
md = metadata or {}
go_path = str(md.get("taskGoPath") or "").strip().lower()
if go_path not in TASK_DEEPLINK_GO_PATHS:
return None
source_task_id = str(md.get("taskId") or "").strip()
task_id = str(md.get("taskRecordId") or source_task_id).strip()
if not task_id or not source_task_id:
return None
agent_id = str(md.get("agent_id") or "").strip() or None
return {
"task_id": task_id,
"source_task_id": source_task_id,
"go_path": go_path,
"agent_id": agent_id,
}
@@ -0,0 +1,86 @@
"""In-memory task deep-link index backed by LangGraph BaseStore."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from langgraph.store.base import BaseStore
from deerflow.persistence.task_deeplink_threads.base import task_deeplink_binding
from deerflow.utils.time import now_iso
if TYPE_CHECKING:
from deerflow.persistence.thread_meta.base import ThreadMetaStore
TASK_DEEPLINK_THREADS_NS: tuple[str, ...] = ("task_deeplink_threads",)
async def sync_task_deeplink_index(store: BaseStore, thread: dict[str, Any]) -> None:
metadata = thread.get("metadata") or {}
binding = task_deeplink_binding(metadata)
thread_id = str(thread.get("thread_id") or "")
if not thread_id:
return
if binding is None:
if "taskGoPath" in metadata or "taskRecordId" in metadata:
await store.adelete(TASK_DEEPLINK_THREADS_NS, thread_id)
return
await store.aput(
TASK_DEEPLINK_THREADS_NS,
thread_id,
{
"thread_id": thread_id,
**binding,
"created_by": thread.get("user_id"),
"created_at": thread.get("created_at") or now_iso(),
"updated_at": thread.get("updated_at") or now_iso(),
},
)
async def delete_task_deeplink_index(store: BaseStore, thread_id: str) -> None:
await store.adelete(TASK_DEEPLINK_THREADS_NS, thread_id)
class MemoryTaskDeeplinkThreadStore:
def __init__(self, store: BaseStore, thread_store: ThreadMetaStore) -> None:
self._store = store
self._thread_store = thread_store
async def list_threads(
self,
task_id: str,
go_path: str,
*,
limit: int = 100,
) -> list[dict[str, Any]]:
items = await self._store.asearch(
TASK_DEEPLINK_THREADS_NS,
filter={"task_id": task_id, "go_path": go_path},
limit=limit,
)
rows: list[dict[str, Any]] = []
for item in items:
thread_id = str(item.value.get("thread_id") or "")
if not thread_id:
continue
thread = await self._thread_store.get(thread_id, user_id=None)
if thread is None:
continue
rows.append(
{
"thread_id": thread_id,
"status": thread.get("status", "idle"),
"created_at": thread.get("created_at", ""),
"updated_at": thread.get("updated_at", ""),
"metadata": thread.get("metadata") or {},
"values": ({"title": thread.get("display_name")} if thread.get("display_name") else {}),
"interrupts": {},
}
)
rows.sort(
key=lambda row: (str(row.get("updated_at") or ""), row["thread_id"]),
reverse=True,
)
return rows[:limit]
@@ -0,0 +1,48 @@
"""Dedicated lookup rows for single-agent task deep-link conversations."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Index, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class TaskDeeplinkThreadRow(Base):
"""One task-workspace history entry pointing at one LangGraph thread.
The row intentionally stores no messages or title. Those remain in the
canonical checkpoint / ``threads_meta`` record and are joined by primary
key when listing a task's history.
"""
__tablename__ = "task_deeplink_threads"
__table_args__ = (
Index(
"ix_task_deeplink_threads_scope",
"task_id",
"go_path",
"updated_at",
"thread_id",
),
)
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
# Original URL ?taskId=. For xdfx this is the action id by product contract.
task_id: Mapped[str] = mapped_column(String(128), nullable=False)
# Resolved real task id used by model/run context (equals task_id except xdfx).
source_task_id: Mapped[str] = mapped_column(String(128), nullable=False)
go_path: Mapped[str] = mapped_column(String(16), nullable=False)
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
# Audit only. Task deep-link histories remain shared across authenticated users.
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,105 @@
"""SQL implementation of the dedicated task deep-link conversation index."""
from __future__ import annotations
from datetime import UTC, datetime
from typing import TYPE_CHECKING, Any
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.task_deeplink_threads.base import task_deeplink_binding
from deerflow.persistence.task_deeplink_threads.model import TaskDeeplinkThreadRow
if TYPE_CHECKING:
from deerflow.persistence.thread_meta.model import ThreadMetaRow
async def sync_task_deeplink_index(session: AsyncSession, thread: ThreadMetaRow) -> None:
"""Synchronize one index row inside the caller's thread-meta transaction."""
metadata = thread.metadata_json or {}
binding = task_deeplink_binding(metadata)
if binding is None:
# Ordinary chats take no extra query. Only an explicitly cleared/invalid
# deep-link marker can require removal of a previously registered row.
if "taskGoPath" in metadata or "taskRecordId" in metadata:
await session.execute(delete(TaskDeeplinkThreadRow).where(TaskDeeplinkThreadRow.thread_id == thread.thread_id))
return
existing = await session.get(TaskDeeplinkThreadRow, thread.thread_id)
now = thread.updated_at if isinstance(thread.updated_at, datetime) else datetime.now(UTC)
if existing is None:
session.add(
TaskDeeplinkThreadRow(
thread_id=thread.thread_id,
task_id=binding["task_id"],
source_task_id=binding["source_task_id"],
go_path=binding["go_path"],
agent_id=binding["agent_id"],
created_by=thread.user_id,
created_at=(thread.created_at if isinstance(thread.created_at, datetime) else now),
updated_at=now,
)
)
return
existing.task_id = binding["task_id"]
existing.source_task_id = binding["source_task_id"]
existing.go_path = binding["go_path"]
existing.agent_id = binding["agent_id"]
existing.updated_at = now
async def delete_task_deeplink_index(session: AsyncSession, thread_id: str) -> None:
await session.execute(delete(TaskDeeplinkThreadRow).where(TaskDeeplinkThreadRow.thread_id == thread_id))
class TaskDeeplinkThreadRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_threads(
self,
task_id: str,
go_path: str,
*,
limit: int = 100,
) -> list[dict[str, Any]]:
"""List one task/goPath partition without scanning thread metadata JSON."""
# Local import avoids a package-initialization cycle: thread_meta.sql
# imports the sync helpers above for atomic dual writes.
from deerflow.persistence.thread_meta.model import ThreadMetaRow
stmt = (
select(TaskDeeplinkThreadRow, ThreadMetaRow)
.join(
ThreadMetaRow,
ThreadMetaRow.thread_id == TaskDeeplinkThreadRow.thread_id,
)
.where(
TaskDeeplinkThreadRow.task_id == task_id,
TaskDeeplinkThreadRow.go_path == go_path,
)
.order_by(
ThreadMetaRow.updated_at.desc(),
ThreadMetaRow.thread_id.desc(),
)
.limit(limit)
)
async with self._sf() as session:
rows = (await session.execute(stmt)).all()
return [
{
"thread_id": thread.thread_id,
"status": thread.status,
"created_at": thread.created_at,
"updated_at": thread.updated_at,
"metadata": thread.metadata_json or {},
"values": ({"title": thread.display_name} if thread.display_name else {}),
"interrupts": {},
}
for _index, thread in rows
]
@@ -30,19 +30,49 @@ EXCLUDED_SYSTEM_THREAD_TYPES = frozenset(
"deep_research",
"writing_setup",
"report_structure_setup",
"task-analysis",
"task-analysis-qa",
"task_deeplink",
}
)
# 这些系统线程 id 只存在于编排内部,不进「最近的对话」;loopback 派活必须能跨
# 「点击触发的 HTTP 用户」和「绑定 created_by 签发的 JWT」读写,否则会 404 not accessible。
SHARED_ACCESS_THREAD_TYPES = frozenset(
{
"roundtable",
"task-analysis",
"task-analysis-qa",
}
)
def is_excluded_system_thread(metadata: dict[str, Any] | None) -> bool:
"""Return True when a thread should be hidden from user conversation lists."""
"""Return True when a thread should be hidden from user conversation lists.
Task-analysis seat runs created before ``thread_type`` was stamped still
carry ``task_analysis_seat``. Those rows stay in the execution record and
out of recent chats.
"""
md = metadata or {}
if md.get("system"):
return True
seat = md.get("task_analysis_seat")
if isinstance(seat, str) and seat.strip():
return True
thread_type = md.get("thread_type")
return isinstance(thread_type, str) and thread_type in EXCLUDED_SYSTEM_THREAD_TYPES
def is_shared_access_thread(metadata: dict[str, Any] | None) -> bool:
"""Return True when any authenticated caller may read/write this thread."""
md = metadata or {}
if md.get("taskId"):
return True
thread_type = md.get("thread_type")
return isinstance(thread_type, str) and thread_type in SHARED_ACCESS_THREAD_TYPES
class ThreadMetaStore(abc.ABC):
@abc.abstractmethod
async def create(
@@ -11,7 +11,15 @@ from typing import Any
from langgraph.store.base import BaseStore
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread
from deerflow.persistence.task_deeplink_threads.memory import (
delete_task_deeplink_index,
sync_task_deeplink_index,
)
from deerflow.persistence.thread_meta.base import (
ThreadMetaStore,
is_excluded_system_thread,
is_shared_access_thread,
)
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
from deerflow.utils.time import coerce_iso, now_iso
@@ -64,6 +72,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
"updated_at": now,
}
await self._store.aput(THREADS_NS, thread_id, record)
await sync_task_deeplink_index(self._store, record)
return record
async def get(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> dict | None:
@@ -183,8 +192,8 @@ class MemoryThreadMetaStore(ThreadMetaStore):
item = await self._store.aget(THREADS_NS, thread_id)
if item is None:
return not require_existing
# 任务深链对话(metadata.taskId)全局共享、不区分用户:任何登录用户都可读写/续聊。
if (item.value.get("metadata") or {}).get("taskId"):
# 任务深链 / 圆桌 / 任务分析系统线程:全局共享(与 SQL store 同口径)。
if is_shared_access_thread(item.value.get("metadata") or {}):
return True
record_user_id = item.value.get("user_id")
if record_user_id is None:
@@ -224,11 +233,13 @@ class MemoryThreadMetaStore(ThreadMetaStore):
record["metadata"] = merged
record["updated_at"] = now_iso()
await self._store.aput(THREADS_NS, thread_id, record)
await sync_task_deeplink_index(self._store, record)
async def delete(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.delete")
if record is None:
return
await delete_task_deeplink_index(self._store, thread_id)
await self._store.adelete(THREADS_NS, thread_id)
@staticmethod
@@ -16,12 +16,27 @@ class ThreadMetaRow(Base):
__table_args__ = (
Index("ix_threads_meta_created_user", "created_at", "user_id"),
Index("ix_threads_meta_user_created", "user_id", "created_at"),
# Task deep-link history is globally shared and therefore cannot use
# the normal ``user_id`` scope. Keep its high-cardinality lookup
# fields out of ``metadata_json`` so task entry never has to scan and
# Python-filter every thread in the installation.
Index("ix_threads_meta_task_agent_updated", "task_id", "agent_id", "updated_at", "thread_id"),
Index("ix_threads_meta_task_action_agent_updated", "task_id", "action_id", "agent_id", "updated_at", "thread_id"),
# The ordinary recent-chat path has an owner filter and the same
# updated-at ordering. This prevents a full table sort there too.
Index("ix_threads_meta_user_updated", "user_id", "updated_at", "thread_id"),
)
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
assistant_id: Mapped[str | None] = mapped_column(String(128), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
display_name: Mapped[str | None] = mapped_column(String(256))
assistant_id: Mapped[str | None] = mapped_column(String(128), index=True)
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
# Indexed projections of the corresponding metadata values. The JSON
# blob remains authoritative and supports arbitrary metadata filters;
# these columns exist solely for the task-workspace hot path.
task_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
action_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
display_name: Mapped[str | None] = mapped_column(String(256))
status: Mapped[str] = mapped_column(String(20), default="idle")
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -8,12 +8,69 @@ from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread
from deerflow.persistence.task_deeplink_threads.sql import (
delete_task_deeplink_index,
sync_task_deeplink_index,
)
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
from deerflow.persistence.thread_meta.model import ThreadMetaRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
class ThreadMetaRepository(ThreadMetaStore):
_INDEXED_METADATA_COLUMNS = {
"taskId": ThreadMetaRow.task_id,
"agent_id": ThreadMetaRow.agent_id,
"actionId": ThreadMetaRow.action_id,
}
_INDEXED_METADATA_ATTRIBUTES = {
"taskId": "task_id",
"agent_id": "agent_id",
"actionId": "action_id",
}
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
"""Return metadata predicates that have a maintained SQL projection.
Task deep-link callers use string ids. Restrict the SQL shortcut to
strings so legacy callers that intentionally store another JSON type keep
the exact Python-side equality semantics of the generic metadata filter.
"""
return {
key: value
for key, value in (metadata or {}).items()
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
}
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
"""Build ORM column values from authoritative thread metadata.
Always include every projection so a metadata update that changes a value
from a string to another JSON type cannot leave a stale indexed value.
"""
values = _indexed_metadata_values(metadata)
return {
attribute: values.get(metadata_key)
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
}
def _apply_indexed_metadata_filters(stmt: Any, metadata: dict[str, Any] | None) -> tuple[Any, set[str]]:
"""Apply known metadata predicates in SQL and return their keys.
The caller still verifies the full metadata object in Python whenever a
generic filter or system-thread exclusion is involved. This keeps the
public API's exact-match contract while reducing that verification set to
rows for the requested task/agent/action instead of the entire table.
"""
values = _indexed_metadata_values(metadata)
for key, value in values.items():
stmt = stmt.where(_INDEXED_METADATA_COLUMNS[key] == value)
return stmt, set(values)
class ThreadMetaRepository(ThreadMetaStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@@ -40,18 +97,20 @@ class ThreadMetaRepository(ThreadMetaStore):
# creates an orphan row (used by migration scripts).
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.create")
now = datetime.now(UTC)
row = ThreadMetaRow(
thread_id=thread_id,
assistant_id=assistant_id,
user_id=resolved_user_id,
display_name=display_name,
status="idle",
metadata_json=metadata or {},
row = ThreadMetaRow(
thread_id=thread_id,
assistant_id=assistant_id,
user_id=resolved_user_id,
**_indexed_thread_metadata(metadata),
display_name=display_name,
status="idle",
metadata_json=metadata or {},
created_at=now,
updated_at=now,
)
async with self._sf() as session:
session.add(row)
await sync_task_deeplink_index(session, row)
await session.commit()
# Deliberately no session.refresh(): every column above is assigned
# explicitly and the session factory uses expire_on_commit=False, so
@@ -108,10 +167,10 @@ class ThreadMetaRepository(ThreadMetaStore):
md = row.metadata_json or {}
# 全局共享、不区分用户(任何登录用户都可读写/续聊):
# ① 任务深链对话(metadata.taskId);
# ② 圆桌会商系统线程(metadata.thread_type=="roundtable")——其 Step2/Step3
# 沙箱产物(md/流程图/报告)需让历史草稿或其他账号打开同一会商时也能查看,
# 与任务深链对话一致(线程 id 仅存在于草稿中,不会被无关账号发现)。
if md.get("taskId") or md.get("thread_type") == "roundtable":
# ② 圆桌 / 任务分析等系统编排线程(thread_type 见 SHARED_ACCESS_THREAD_TYPES)
# —— loopback 派活签发的 JWT 可能与建 thread 时 HTTP 上下文用户不是同一人,
# 若按属主拦截会把刚建好的总控 thread 打成 404 not accessible。
if is_shared_access_thread(md):
return True
if row.user_id is None:
return True
@@ -144,16 +203,24 @@ class ThreadMetaRepository(ThreadMetaStore):
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
if status:
stmt = stmt.where(ThreadMetaRow.status == status)
if query and query.strip():
if query and query.strip():
# Case-insensitive title search. display_name is a real column, so
# this filter runs in SQL even when the JSON filters below fall
# back to Python-side scanning.
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
if not metadata and not exclude_system:
stmt = stmt.limit(limit).offset(offset)
async with self._sf() as session:
result = await session.execute(stmt)
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
# ``metadata_json`` is deliberately portable, so arbitrary filters
# still receive Python-side equality checks below. These three task
# identifiers are maintained projections and can safely narrow the
# SQL result first. A deep-link query that used to scan every thread
# now reads only that task's rows.
stmt, indexed_metadata_keys = _apply_indexed_metadata_filters(stmt, metadata)
remaining_metadata_keys = set(metadata or {}) - indexed_metadata_keys
if not remaining_metadata_keys and not exclude_system:
stmt = stmt.limit(limit).offset(offset)
async with self._sf() as session:
result = await session.execute(stmt)
return [self._row_to_dict(r) for r in result.scalars()]
# Metadata filters live inside the JSON blob, so rows are scanned in
@@ -205,12 +272,17 @@ class ThreadMetaRepository(ThreadMetaStore):
conditions.append(ThreadMetaRow.user_id == resolved_user_id)
if status:
conditions.append(ThreadMetaRow.status == status)
if query and query.strip():
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
if not metadata and not exclude_system:
stmt = select(func.count()).select_from(ThreadMetaRow)
for condition in conditions:
if query and query.strip():
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
indexed_metadata = _indexed_metadata_values(metadata)
for key, value in indexed_metadata.items():
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
remaining_metadata_keys = set(metadata or {}) - set(indexed_metadata)
if not remaining_metadata_keys and not exclude_system:
stmt = select(func.count()).select_from(ThreadMetaRow)
for condition in conditions:
stmt = stmt.where(condition)
async with self._sf() as session:
return int((await session.execute(stmt)).scalar_one())
@@ -318,11 +390,14 @@ class ThreadMetaRepository(ThreadMetaStore):
task_shared = bool((row.metadata_json or {}).get("taskId"))
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
return
merged = dict(row.metadata_json or {})
merged.update(metadata)
row.metadata_json = merged
row.updated_at = datetime.now(UTC)
await session.commit()
merged = dict(row.metadata_json or {})
merged.update(metadata)
row.metadata_json = merged
for key, value in _indexed_thread_metadata(merged).items():
setattr(row, key, value)
row.updated_at = datetime.now(UTC)
await sync_task_deeplink_index(session, row)
await session.commit()
async def delete(
self,
@@ -339,5 +414,6 @@ class ThreadMetaRepository(ThreadMetaStore):
task_shared = bool((row.metadata_json or {}).get("taskId"))
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
return
await delete_task_deeplink_index(session, thread_id)
await session.delete(row)
await session.commit()
@@ -43,7 +43,17 @@ class PortableJSON(TypeDecorator):
return value
if not value:
return None
return json.loads(value)
try:
return json.loads(value)
except json.JSONDecodeError:
from deerflow.persistence.gauss_mysql_compat import (
decode_gauss_mysql_json,
is_gauss_mysql_dialect,
)
if not is_gauss_mysql_dialect(dialect):
raise
return decode_gauss_mysql_json(value)
class PortableLongText(TypeDecorator):
@@ -0,0 +1,22 @@
"""Global database-backed upload policy persistence."""
from __future__ import annotations
from deerflow.persistence.upload_policy.base import UploadPolicyStore
from deerflow.persistence.upload_policy.memory import MemoryUploadPolicyStore
from deerflow.persistence.upload_policy.model import UploadPolicyRow
from deerflow.persistence.upload_policy.sql import UploadPolicyRepository
__all__ = [
"MemoryUploadPolicyStore",
"UploadPolicyRepository",
"UploadPolicyRow",
"UploadPolicyStore",
"make_upload_policy_store",
]
def make_upload_policy_store(session_factory):
if session_factory is None:
return MemoryUploadPolicyStore()
return UploadPolicyRepository(session_factory)
@@ -0,0 +1,26 @@
"""Persistence contract for the global upload policy document."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class UploadPolicyStore(ABC):
@abstractmethod
async def ensure_seed(self, policy: dict[str, Any]) -> dict[str, Any]:
"""Create the default row if missing without overwriting saved edits."""
raise NotImplementedError
@abstractmethod
async def get_policy(self) -> dict[str, Any] | None:
"""Return the stored policy, or ``None`` before the first save."""
raise NotImplementedError
@abstractmethod
async def set_policy(self, policy: dict[str, Any], *, updated_by: str | None = None) -> dict[str, Any]:
"""Atomically replace the global policy document."""
raise NotImplementedError
@@ -0,0 +1,39 @@
"""In-memory upload-policy store used without a configured SQL database."""
from __future__ import annotations
from copy import deepcopy
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.upload_policy.base import UploadPolicyStore
class MemoryUploadPolicyStore(UploadPolicyStore):
def __init__(self) -> None:
self._policy: dict[str, Any] | None = None
self._updated_by: str | None = None
self._updated_at: str | None = None
self._revision = 0
async def ensure_seed(self, policy: dict[str, Any]) -> dict[str, Any]:
if self._policy is None:
return await self.set_policy(policy, updated_by=None)
return (await self.get_policy()) or {}
async def get_policy(self) -> dict[str, Any] | None:
if self._policy is None:
return None
return {
"policy": deepcopy(self._policy),
"updated_by": self._updated_by,
"updated_at": self._updated_at,
"revision": self._revision,
}
async def set_policy(self, policy: dict[str, Any], *, updated_by: str | None = None) -> dict[str, Any]:
self._policy = deepcopy(policy)
self._updated_by = updated_by
self._updated_at = datetime.now(UTC).isoformat()
self._revision += 1
return (await self.get_policy()) or {}
Loaded 100 of 1651 files, more files were not shown because too many files have changed in this diff. Show more