sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+34
@@ -0,0 +1,34 @@
|
||||
"""Project-scoped cross-session memory persistence."""
|
||||
|
||||
from deerflow.persistence.analysis_project_memory.model import (
|
||||
AnalysisProjectMemoryItemRow,
|
||||
AnalysisProjectMemoryJobRow,
|
||||
AnalysisProjectMemoryStateRow,
|
||||
)
|
||||
from deerflow.persistence.analysis_project_memory.store import (
|
||||
ACTIVE_ITEM_STATUS,
|
||||
ACTIVE_JOB_STATUSES,
|
||||
ITEM_KINDS,
|
||||
ITEM_STATUSES,
|
||||
TERMINAL_JOB_STATUSES,
|
||||
AnalysisProjectMemoryStore,
|
||||
)
|
||||
|
||||
|
||||
def make_analysis_project_memory_store(session_factory=None) -> AnalysisProjectMemoryStore:
|
||||
"""Create the SQL store, or its compatible memory fallback when factory is None."""
|
||||
return AnalysisProjectMemoryStore(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ACTIVE_ITEM_STATUS",
|
||||
"ACTIVE_JOB_STATUSES",
|
||||
"AnalysisProjectMemoryItemRow",
|
||||
"AnalysisProjectMemoryJobRow",
|
||||
"AnalysisProjectMemoryStateRow",
|
||||
"AnalysisProjectMemoryStore",
|
||||
"ITEM_KINDS",
|
||||
"ITEM_STATUSES",
|
||||
"TERMINAL_JOB_STATUSES",
|
||||
"make_analysis_project_memory_store",
|
||||
]
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
"""ORM rows for durable, project-scoped cross-session memory.
|
||||
|
||||
The SQL rows are deliberately independent from the generic user/agent memory
|
||||
tables. Their authoritative scope is ``(project_id, owner_user_id)`` and every
|
||||
child row carries the owner as a defence-in-depth filter for repository reads.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Boolean, Float, ForeignKey, Index, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class AnalysisProjectMemoryStateRow(Base):
|
||||
"""One materialized memory snapshot per analysis project."""
|
||||
|
||||
__tablename__ = "analysis_project_memory_states"
|
||||
|
||||
project_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("analysis_projects.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
owner_user_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True, server_default=sa.true()
|
||||
)
|
||||
settings_json: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
|
||||
snapshot_json: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
|
||||
revision: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default=sa.text("0")
|
||||
)
|
||||
item_count: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default=sa.text("0")
|
||||
)
|
||||
# ready / building / rebuilding / recovering / error
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(24), nullable=False, default="ready", server_default="ready", index=True
|
||||
)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
rebuild_requested_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
rebuilt_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=_now, onupdate=_now
|
||||
)
|
||||
|
||||
|
||||
class AnalysisProjectMemoryItemRow(Base):
|
||||
"""An atomic fact/decision/constraint/episode with auditable provenance."""
|
||||
|
||||
__tablename__ = "analysis_project_memory_items"
|
||||
__table_args__ = (
|
||||
# ``active_key`` is populated only for the active version of a logical
|
||||
# item. NULL values do not conflict, so superseded/deleted versions
|
||||
# remain as an audit trail while one active version is enforced.
|
||||
UniqueConstraint("active_key", name="uq_analysis_project_memory_item_active_key"),
|
||||
Index(
|
||||
"ix_analysis_project_memory_item_scope_status",
|
||||
"project_id",
|
||||
"owner_user_id",
|
||||
"status",
|
||||
),
|
||||
Index(
|
||||
"ix_analysis_project_memory_item_source_thread",
|
||||
"project_id",
|
||||
"source_thread_id",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
project_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("analysis_projects.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
owner_user_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
logical_key: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
key_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
active_key: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False, default="fact", index=True)
|
||||
content: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
|
||||
# active / superseded / deleted
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="active", server_default="active", index=True
|
||||
)
|
||||
confidence: Mapped[float] = mapped_column(Float, nullable=False, default=1.0)
|
||||
importance: Mapped[float] = mapped_column(Float, nullable=False, default=0.5)
|
||||
pinned: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default=sa.false()
|
||||
)
|
||||
source_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
source_message_ids: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
source_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
source_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), nullable=False, default=dict)
|
||||
extractor_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
superseded_by: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("analysis_project_memory_items.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=1, server_default=sa.text("1")
|
||||
)
|
||||
# Phase-two foundation: the encoder remains optional. JSON keeps this
|
||||
# portable across SQLite/MySQL/Postgres/Gauss; retrieval validates dimensions.
|
||||
embedding_json: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
embedding_model: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
embedding_fingerprint: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=_now, onupdate=_now
|
||||
)
|
||||
|
||||
|
||||
class AnalysisProjectMemoryJobRow(Base):
|
||||
"""Durable, idempotent extraction/rebuild work with an expiring lease."""
|
||||
|
||||
__tablename__ = "analysis_project_memory_jobs"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("idempotency_key", name="uq_analysis_project_memory_job_idempotency"),
|
||||
Index(
|
||||
"ix_analysis_project_memory_job_claim",
|
||||
"status",
|
||||
"available_at",
|
||||
"lease_until",
|
||||
),
|
||||
Index(
|
||||
"ix_analysis_project_memory_job_scope",
|
||||
"project_id",
|
||||
"owner_user_id",
|
||||
"status",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
project_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("analysis_projects.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
owner_user_id: Mapped[str] = mapped_column(String(255), nullable=False, index=True)
|
||||
source_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
source_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
source_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
extractor_version: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
idempotency_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
payload_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
|
||||
result_json: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
# queued / running / completed / failed / cancelled
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(24), nullable=False, default="queued", server_default="queued", index=True
|
||||
)
|
||||
attempt: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default=sa.text("0")
|
||||
)
|
||||
max_attempts: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=3, server_default=sa.text("3")
|
||||
)
|
||||
available_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=_now, onupdate=_now
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AnalysisProjectMemoryItemRow",
|
||||
"AnalysisProjectMemoryJobRow",
|
||||
"AnalysisProjectMemoryStateRow",
|
||||
]
|
||||
+2733
File diff suppressed because it is too large.
Load diff
+202
@@ -0,0 +1,202 @@
|
||||
"""User-owned analysis projects and immutable project/thread bindings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import ForeignKey, String, Text, delete, select
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
MODES = frozenset({"qa", "strategy", "solution"})
|
||||
CONFIG_FIELDS = frozenset({"agent_id", "report_config", "strategy_config"})
|
||||
EDITABLE_FIELDS = frozenset({"name", "background", "knowledge_base_ids", "assistant_knowledge_base_ids"}) | CONFIG_FIELDS
|
||||
|
||||
|
||||
class AnalysisProjectConfigRow(Base):
|
||||
"""Additive table: upgrades do not alter existing project/thread tables."""
|
||||
|
||||
__tablename__ = "analysis_project_configs"
|
||||
key: Mapped[str] = mapped_column(String(80), primary_key=True)
|
||||
config: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
|
||||
|
||||
class AnalysisProjectRow(Base):
|
||||
__tablename__ = "analysis_projects"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(255), index=True)
|
||||
name: Mapped[str] = mapped_column(String(160))
|
||||
mode: Mapped[str] = mapped_column(String(16))
|
||||
background: Mapped[str] = mapped_column(Text, default="")
|
||||
knowledge_base_ids: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
assistant_knowledge_base_ids: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime())
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime())
|
||||
|
||||
|
||||
class AnalysisProjectThreadRow(Base):
|
||||
__tablename__ = "analysis_project_threads"
|
||||
|
||||
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
project_id: Mapped[str] = mapped_column(ForeignKey("analysis_projects.id"), index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime())
|
||||
|
||||
|
||||
def _dict(row):
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
if isinstance(data.get(key), datetime):
|
||||
data[key] = data[key].isoformat()
|
||||
return deepcopy(data)
|
||||
|
||||
|
||||
class AnalysisProjectStore:
|
||||
"""SQL in persistent deployments; isolated in-memory fallback for memory mode.
|
||||
|
||||
Every public project operation requires an explicit owner. The unscoped
|
||||
thread lookup is only for the Gateway's ownership and run-policy checks.
|
||||
Mode has no update path, and bindings are separate from client-editable metadata.
|
||||
"""
|
||||
|
||||
def __init__(self, session_factory=None):
|
||||
self._sf = session_factory
|
||||
self._projects: dict[str, dict] = {}
|
||||
self._threads: dict[str, dict] = {}
|
||||
self._configs: dict[str, dict] = {}
|
||||
|
||||
async def config_for(self, key: str) -> dict:
|
||||
if not self._sf:
|
||||
return deepcopy(self._configs.get(key, {}))
|
||||
async with self._sf() as session:
|
||||
row = await session.get(AnalysisProjectConfigRow, key)
|
||||
return deepcopy(row.config) if row else {}
|
||||
|
||||
async def _enrich(self, project: dict | None) -> dict | None:
|
||||
if project is None:
|
||||
return None
|
||||
return {
|
||||
**project,
|
||||
"agent_id": None,
|
||||
"report_config": {},
|
||||
"strategy_config": {},
|
||||
**await self.config_for(f"project:{project['id']}"),
|
||||
}
|
||||
|
||||
async def create(self, user_id: str, *, mode: str, **fields) -> dict:
|
||||
if mode not in MODES or set(fields) - EDITABLE_FIELDS:
|
||||
raise ValueError("Invalid project fields")
|
||||
now = datetime.now(UTC)
|
||||
row = AnalysisProjectRow(
|
||||
id=str(uuid4()),
|
||||
user_id=user_id,
|
||||
mode=mode,
|
||||
name=fields["name"],
|
||||
background=fields.get("background", ""),
|
||||
knowledge_base_ids=list(fields.get("knowledge_base_ids", [])),
|
||||
assistant_knowledge_base_ids=list(fields.get("assistant_knowledge_base_ids", [])),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
if self._sf:
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
session.add(AnalysisProjectConfigRow(key=f"project:{row.id}", config={key: deepcopy(fields[key]) for key in CONFIG_FIELDS if key in fields}))
|
||||
await session.commit()
|
||||
else:
|
||||
self._projects[row.id] = _dict(row)
|
||||
self._configs[f"project:{row.id}"] = {key: deepcopy(fields[key]) for key in CONFIG_FIELDS if key in fields}
|
||||
return await self._enrich(_dict(row))
|
||||
|
||||
async def list(self, user_id: str) -> list[dict]:
|
||||
if not self._sf:
|
||||
return [await self._enrich(deepcopy(p)) for p in sorted(self._projects.values(), key=lambda p: p["updated_at"], reverse=True) if p["user_id"] == user_id]
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.user_id == user_id).order_by(AnalysisProjectRow.updated_at.desc()))
|
||||
rows = [_dict(row) for row in result.scalars()]
|
||||
return [await self._enrich(row) for row in rows]
|
||||
|
||||
async def get(self, project_id: str, user_id: str) -> dict | None:
|
||||
if not self._sf:
|
||||
row = self._projects.get(project_id)
|
||||
return await self._enrich(deepcopy(row)) if row and row["user_id"] == user_id else None
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.id == project_id, AnalysisProjectRow.user_id == user_id))).scalar_one_or_none()
|
||||
data = _dict(row) if row else None
|
||||
return await self._enrich(data)
|
||||
|
||||
async def update(self, project_id: str, user_id: str, **fields) -> dict | None:
|
||||
if set(fields) - EDITABLE_FIELDS:
|
||||
raise ValueError("项目模式创建后不可更改")
|
||||
if not self._sf:
|
||||
row = self._projects.get(project_id)
|
||||
if not row or row["user_id"] != user_id:
|
||||
return None
|
||||
row.update({key: deepcopy(value) for key, value in fields.items() if key not in CONFIG_FIELDS}, updated_at=datetime.now(UTC).isoformat())
|
||||
self._configs.setdefault(f"project:{project_id}", {}).update({key: deepcopy(value) for key, value in fields.items() if key in CONFIG_FIELDS})
|
||||
return await self._enrich(deepcopy(row))
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(AnalysisProjectRow).where(AnalysisProjectRow.id == project_id, AnalysisProjectRow.user_id == user_id).with_for_update())).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
for key, value in fields.items():
|
||||
if key not in CONFIG_FIELDS:
|
||||
setattr(row, key, deepcopy(value))
|
||||
options = await session.get(AnalysisProjectConfigRow, f"project:{project_id}")
|
||||
config = {**(options.config if options else {}), **{key: deepcopy(value) for key, value in fields.items() if key in CONFIG_FIELDS}}
|
||||
await session.merge(AnalysisProjectConfigRow(key=f"project:{project_id}", config=config))
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
data = _dict(row)
|
||||
return await self._enrich(data)
|
||||
|
||||
async def bind_thread(self, project_id: str, user_id: str, thread_id: str, *, agent_id: str | None = None) -> None:
|
||||
if await self.get(project_id, user_id) is None:
|
||||
raise ValueError("Project not found")
|
||||
row = AnalysisProjectThreadRow(thread_id=thread_id, project_id=project_id, created_at=datetime.now(UTC))
|
||||
if not self._sf:
|
||||
if thread_id in self._threads:
|
||||
raise ValueError("Thread already bound")
|
||||
self._threads[thread_id] = _dict(row)
|
||||
self._configs[f"thread:{thread_id}"] = {"agent_id": agent_id}
|
||||
return
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
session.add(AnalysisProjectConfigRow(key=f"thread:{thread_id}", config={"agent_id": agent_id}))
|
||||
await session.commit()
|
||||
|
||||
async def project_for_thread(self, thread_id: str) -> dict | None:
|
||||
if not self._sf:
|
||||
binding = self._threads.get(thread_id)
|
||||
return await self._enrich(deepcopy(self._projects.get(binding["project_id"]))) if binding else None
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(AnalysisProjectRow).join(AnalysisProjectThreadRow).where(AnalysisProjectThreadRow.thread_id == thread_id))).scalar_one_or_none()
|
||||
data = _dict(row) if row else None
|
||||
return await self._enrich(data)
|
||||
|
||||
async def thread_ids(self, project_id: str, user_id: str) -> list[str]:
|
||||
if await self.get(project_id, user_id) is None:
|
||||
return []
|
||||
if not self._sf:
|
||||
return [key for key, value in reversed(list(self._threads.items())) if value["project_id"] == project_id]
|
||||
async with self._sf() as session:
|
||||
return list((await session.execute(select(AnalysisProjectThreadRow.thread_id).where(AnalysisProjectThreadRow.project_id == project_id).order_by(AnalysisProjectThreadRow.created_at.desc()))).scalars())
|
||||
|
||||
async def unbind_failed_creation(self, thread_id: str) -> None:
|
||||
if not self._sf:
|
||||
self._threads.pop(thread_id, None)
|
||||
self._configs.pop(f"thread:{thread_id}", None)
|
||||
return
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(AnalysisProjectThreadRow).where(AnalysisProjectThreadRow.thread_id == thread_id))
|
||||
await session.execute(delete(AnalysisProjectConfigRow).where(AnalysisProjectConfigRow.key == f"thread:{thread_id}"))
|
||||
await session.commit()
|
||||
|
||||
async def unbind_thread(self, thread_id: str) -> None:
|
||||
"""Remove a durable project binding when its ordinary thread is deleted."""
|
||||
|
||||
await self.unbind_failed_creation(thread_id)
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
"""Persisted read index for the generated-file library."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
|
||||
from deerflow.persistence.artifact_library.memory import MemoryArtifactLibraryStore
|
||||
from deerflow.persistence.artifact_library.model import (
|
||||
ArtifactLibraryItemRow,
|
||||
ArtifactLibrarySyncStateRow,
|
||||
)
|
||||
from deerflow.persistence.artifact_library.sql import SqlArtifactLibraryStore
|
||||
|
||||
__all__ = [
|
||||
"ArtifactLibraryItemRow",
|
||||
"ArtifactLibraryStore",
|
||||
"ArtifactLibrarySyncStateRow",
|
||||
"MemoryArtifactLibraryStore",
|
||||
"SqlArtifactLibraryStore",
|
||||
"make_artifact_library_store",
|
||||
]
|
||||
|
||||
|
||||
def make_artifact_library_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryArtifactLibraryStore()
|
||||
return SqlArtifactLibraryStore(session_factory)
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
"""Storage contract for the generated-file library index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ArtifactLibraryStore(ABC):
|
||||
"""A user-scoped, replace-on-sync artifact metadata index."""
|
||||
|
||||
@abstractmethod
|
||||
async def claim_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
force: bool = False,
|
||||
stale_after_seconds: float = 15.0,
|
||||
lease_seconds: float = 300.0,
|
||||
) -> str | None:
|
||||
"""Claim one background refresh and return its generation token.
|
||||
|
||||
``None`` means another worker owns a live lease or the persisted index
|
||||
is still fresh enough. Implementations must make this operation safe
|
||||
when multiple Gateway workers receive the same request concurrently.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def finish_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
generation: str,
|
||||
items: list[dict[str, Any]],
|
||||
) -> bool:
|
||||
"""Atomically publish ``items`` when ``generation`` still owns the lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
|
||||
"""Record a failed refresh when ``generation`` still owns the lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
|
||||
"""Return the persisted refresh state for one user."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_items(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
query: str | None = None,
|
||||
file_kind: str = "all",
|
||||
source_type: str = "all",
|
||||
offset: int = 0,
|
||||
limit: int = 50,
|
||||
) -> tuple[int, list[dict[str, Any]]]:
|
||||
"""Return an indexed, newest-first page and its filtered total."""
|
||||
raise NotImplementedError
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
"""In-memory artifact-library index used by memory-mode deployments and tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _artifact_id(user_id: str, item: dict[str, Any]) -> str:
|
||||
identity = "\0".join((user_id, str(item.get("thread_id") or ""), str(item.get("path") or "")))
|
||||
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class MemoryArtifactLibraryStore(ArtifactLibraryStore):
|
||||
def __init__(self) -> None:
|
||||
self._items: dict[str, dict[str, Any]] = {}
|
||||
self._states: dict[str, dict[str, Any]] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def claim_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
force: bool = False,
|
||||
stale_after_seconds: float = 15.0,
|
||||
lease_seconds: float = 300.0,
|
||||
) -> str | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._lock:
|
||||
state = self._states.get(user_id)
|
||||
if state is not None:
|
||||
lease = state.get("lease_expires_at")
|
||||
if state.get("status") == "syncing" and isinstance(lease, datetime) and lease > now:
|
||||
return None
|
||||
freshness_anchor = state.get("completed_at")
|
||||
if state.get("status") == "error":
|
||||
freshness_anchor = state.get("started_at")
|
||||
if not force and isinstance(freshness_anchor, datetime) and freshness_anchor + timedelta(seconds=stale_after_seconds) > now:
|
||||
return None
|
||||
generation = str(uuid4())
|
||||
self._states[user_id] = {
|
||||
"status": "syncing",
|
||||
"generation": generation,
|
||||
"started_at": now,
|
||||
"lease_expires_at": now + timedelta(seconds=lease_seconds),
|
||||
"completed_at": state.get("completed_at") if state else None,
|
||||
"last_error": None,
|
||||
}
|
||||
return generation
|
||||
|
||||
async def finish_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
generation: str,
|
||||
items: list[dict[str, Any]],
|
||||
) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
async with self._lock:
|
||||
state = self._states.get(user_id)
|
||||
if state is None or state.get("generation") != generation or state.get("status") != "syncing":
|
||||
return False
|
||||
self._items = {key: value for key, value in self._items.items() if value.get("user_id") != user_id}
|
||||
for item in items:
|
||||
row = dict(item)
|
||||
row["artifact_id"] = _artifact_id(user_id, item)
|
||||
row["user_id"] = user_id
|
||||
row["sync_generation"] = generation
|
||||
row["indexed_at"] = now.isoformat()
|
||||
self._items[row["artifact_id"]] = row
|
||||
state.update(
|
||||
status="ready",
|
||||
completed_at=now,
|
||||
lease_expires_at=None,
|
||||
last_error=None,
|
||||
)
|
||||
return True
|
||||
|
||||
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
|
||||
async with self._lock:
|
||||
state = self._states.get(user_id)
|
||||
if state is None or state.get("generation") != generation or state.get("status") != "syncing":
|
||||
return False
|
||||
state.update(
|
||||
status="error",
|
||||
lease_expires_at=None,
|
||||
last_error=(error or "Artifact library refresh failed")[:1024],
|
||||
)
|
||||
return True
|
||||
|
||||
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
|
||||
async with self._lock:
|
||||
state = self._states.get(user_id)
|
||||
if state is None:
|
||||
return {
|
||||
"status": "idle",
|
||||
"generation": None,
|
||||
"started_at": None,
|
||||
"completed_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
return {
|
||||
"status": str(state.get("status") or "idle"),
|
||||
"generation": state.get("generation"),
|
||||
"started_at": _iso(state.get("started_at")),
|
||||
"completed_at": _iso(state.get("completed_at")),
|
||||
"last_error": state.get("last_error"),
|
||||
}
|
||||
|
||||
async def list_items(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
query: str | None = None,
|
||||
file_kind: str = "all",
|
||||
source_type: str = "all",
|
||||
offset: int = 0,
|
||||
limit: int = 50,
|
||||
) -> tuple[int, list[dict[str, Any]]]:
|
||||
normalized = (query or "").strip().casefold()
|
||||
async with self._lock:
|
||||
rows = [dict(row) for row in self._items.values() if row.get("user_id") == user_id]
|
||||
|
||||
def matches(row: dict[str, Any]) -> bool:
|
||||
if file_kind != "all" and row.get("file_kind") != file_kind:
|
||||
return False
|
||||
if source_type != "all" and row.get("source_type") != source_type:
|
||||
return False
|
||||
if not normalized:
|
||||
return True
|
||||
return any(
|
||||
normalized in str(row.get(field) or "").casefold()
|
||||
for field in (
|
||||
"name",
|
||||
"thread_title",
|
||||
"agent_name",
|
||||
"strategy_name",
|
||||
"strategy_project_id",
|
||||
)
|
||||
)
|
||||
|
||||
filtered = [row for row in rows if matches(row)]
|
||||
filtered.sort(
|
||||
key=lambda row: (
|
||||
str(row.get("modified_at") or ""),
|
||||
str(row.get("artifact_id") or ""),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return len(filtered), filtered[offset : offset + limit]
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
"""ORM rows for the generated-file library read index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ArtifactLibraryItemRow(Base):
|
||||
"""One generated file projected into the fast library listing."""
|
||||
|
||||
__tablename__ = "artifact_library_items"
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_artifact_library_user_modified",
|
||||
"user_id",
|
||||
"modified_at",
|
||||
"artifact_id",
|
||||
),
|
||||
Index(
|
||||
"ix_artifact_library_user_kind",
|
||||
"user_id",
|
||||
"file_kind",
|
||||
"modified_at",
|
||||
"artifact_id",
|
||||
),
|
||||
Index(
|
||||
"ix_artifact_library_user_source",
|
||||
"user_id",
|
||||
"source_type",
|
||||
"modified_at",
|
||||
"artifact_id",
|
||||
),
|
||||
Index("ix_artifact_library_user_generation", "user_id", "sync_generation"),
|
||||
)
|
||||
|
||||
# A deterministic SHA-256 of (user_id, thread_id, virtual path) avoids a
|
||||
# long composite primary key that would exceed older MySQL index limits.
|
||||
artifact_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
thread_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
thread_title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
thread_type: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
source_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
strategy_project_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
strategy_name: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
strategy_stale: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
strategy_source_kind: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
download_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
path: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
mime_type: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
modified_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False)
|
||||
file_kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
sync_generation: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||
indexed_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ArtifactLibrarySyncStateRow(Base):
|
||||
"""Per-user lease and last-published generation metadata."""
|
||||
|
||||
__tablename__ = "artifact_library_sync_states"
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="idle")
|
||||
generation: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
+346
@@ -0,0 +1,346 @@
|
||||
"""SQLAlchemy-backed generated-file library index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import and_, delete, func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
|
||||
from deerflow.persistence.artifact_library.model import (
|
||||
ArtifactLibraryItemRow,
|
||||
ArtifactLibrarySyncStateRow,
|
||||
)
|
||||
|
||||
_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
def _artifact_id(user_id: str, item: dict[str, Any]) -> str:
|
||||
identity = "\0".join((user_id, str(item.get("thread_id") or ""), str(item.get("path") or "")))
|
||||
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _parse_datetime(value: object) -> datetime:
|
||||
if isinstance(value, datetime):
|
||||
parsed = value
|
||||
else:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return _EPOCH
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return _EPOCH
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _state_dict(row: ArtifactLibrarySyncStateRow | None) -> dict[str, Any]:
|
||||
if row is None:
|
||||
return {
|
||||
"status": "idle",
|
||||
"generation": None,
|
||||
"started_at": None,
|
||||
"completed_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
return {
|
||||
"status": row.status,
|
||||
"generation": row.generation,
|
||||
"started_at": _iso(row.started_at),
|
||||
"completed_at": _iso(row.completed_at),
|
||||
"last_error": row.last_error,
|
||||
}
|
||||
|
||||
|
||||
def _item_dict(row: ArtifactLibraryItemRow) -> dict[str, Any]:
|
||||
return {
|
||||
"thread_id": row.thread_id,
|
||||
"thread_title": row.thread_title,
|
||||
"agent_id": row.agent_id,
|
||||
"agent_name": row.agent_name,
|
||||
"thread_type": row.thread_type,
|
||||
"source_type": row.source_type,
|
||||
"strategy_project_id": row.strategy_project_id,
|
||||
"strategy_name": row.strategy_name,
|
||||
"strategy_stale": row.strategy_stale,
|
||||
"strategy_source_kind": row.strategy_source_kind,
|
||||
"download_url": row.download_url,
|
||||
"path": row.path,
|
||||
"name": row.name,
|
||||
"size_bytes": row.size_bytes,
|
||||
"mime_type": row.mime_type,
|
||||
"modified_at": row.modified_at.isoformat(),
|
||||
"file_kind": row.file_kind,
|
||||
}
|
||||
|
||||
|
||||
def _apply_item(
|
||||
row: ArtifactLibraryItemRow,
|
||||
*,
|
||||
user_id: str,
|
||||
generation: str,
|
||||
item: dict[str, Any],
|
||||
indexed_at: datetime,
|
||||
) -> None:
|
||||
row.user_id = user_id
|
||||
row.thread_id = str(item.get("thread_id") or "")[:128]
|
||||
row.thread_title = str(item.get("thread_title") or "")[:512]
|
||||
row.agent_id = str(item.get("agent_id"))[:128] if item.get("agent_id") else None
|
||||
row.agent_name = str(item.get("agent_name"))[:512] if item.get("agent_name") else None
|
||||
row.thread_type = str(item.get("thread_type"))[:64] if item.get("thread_type") else None
|
||||
row.source_type = str(item.get("source_type") or "general_qa")[:32]
|
||||
row.strategy_project_id = str(item.get("strategy_project_id"))[:128] if item.get("strategy_project_id") else None
|
||||
row.strategy_name = str(item.get("strategy_name"))[:512] if item.get("strategy_name") else None
|
||||
row.strategy_stale = item.get("strategy_stale") is True
|
||||
row.strategy_source_kind = str(item.get("strategy_source_kind"))[:32] if item.get("strategy_source_kind") else None
|
||||
row.download_url = str(item.get("download_url")) if item.get("download_url") else None
|
||||
row.path = str(item.get("path") or "")
|
||||
row.name = str(item.get("name") or "")[:512]
|
||||
row.size_bytes = max(0, int(item.get("size_bytes") or 0))
|
||||
row.mime_type = str(item.get("mime_type"))[:255] if item.get("mime_type") else None
|
||||
row.modified_at = _parse_datetime(item.get("modified_at"))
|
||||
row.file_kind = str(item.get("file_kind") or "other")[:32]
|
||||
row.sync_generation = generation
|
||||
row.indexed_at = indexed_at
|
||||
|
||||
|
||||
class SqlArtifactLibraryStore(ArtifactLibraryStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def claim_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
force: bool = False,
|
||||
stale_after_seconds: float = 15.0,
|
||||
lease_seconds: float = 300.0,
|
||||
) -> str | None:
|
||||
now = datetime.now(UTC)
|
||||
generation = str(uuid4())
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
async with session.begin():
|
||||
claim_filters = [
|
||||
ArtifactLibrarySyncStateRow.user_id == user_id,
|
||||
# An expired (or legacy missing) lease may be reclaimed.
|
||||
# A live lease remains exclusive even for force=True.
|
||||
or_(
|
||||
ArtifactLibrarySyncStateRow.status != "syncing",
|
||||
ArtifactLibrarySyncStateRow.lease_expires_at.is_(None),
|
||||
ArtifactLibrarySyncStateRow.lease_expires_at <= now,
|
||||
),
|
||||
]
|
||||
if not force:
|
||||
stale_before = now - timedelta(seconds=stale_after_seconds)
|
||||
claim_filters.append(
|
||||
or_(
|
||||
and_(
|
||||
ArtifactLibrarySyncStateRow.status == "error",
|
||||
or_(
|
||||
ArtifactLibrarySyncStateRow.started_at.is_(None),
|
||||
ArtifactLibrarySyncStateRow.started_at <= stale_before,
|
||||
),
|
||||
),
|
||||
and_(
|
||||
ArtifactLibrarySyncStateRow.status != "error",
|
||||
or_(
|
||||
ArtifactLibrarySyncStateRow.completed_at.is_(None),
|
||||
ArtifactLibrarySyncStateRow.completed_at <= stale_before,
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# A conditional UPDATE is the portable compare-and-swap.
|
||||
# SQLite ignores SELECT ... FOR UPDATE, whereas this write
|
||||
# serializes claimers there and also works on MySQL,
|
||||
# PostgreSQL, and openGauss.
|
||||
result = await session.execute(
|
||||
update(ArtifactLibrarySyncStateRow)
|
||||
.where(*claim_filters)
|
||||
.values(
|
||||
status="syncing",
|
||||
generation=generation,
|
||||
started_at=now,
|
||||
lease_expires_at=now + timedelta(seconds=lease_seconds),
|
||||
last_error=None,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 1:
|
||||
return generation
|
||||
|
||||
row = await session.get(ArtifactLibrarySyncStateRow, user_id)
|
||||
if row is not None:
|
||||
return None
|
||||
session.add(
|
||||
ArtifactLibrarySyncStateRow(
|
||||
user_id=user_id,
|
||||
status="syncing",
|
||||
generation=generation,
|
||||
started_at=now,
|
||||
lease_expires_at=now + timedelta(seconds=lease_seconds),
|
||||
last_error=None,
|
||||
)
|
||||
)
|
||||
return generation
|
||||
except IntegrityError:
|
||||
# Two workers can race while inserting a user's first state row.
|
||||
# The winner owns the lease; the loser simply serves the same
|
||||
# persisted index without starting a duplicate filesystem scan.
|
||||
async with self._sf() as session:
|
||||
if await session.get(ArtifactLibrarySyncStateRow, user_id) is not None:
|
||||
return None
|
||||
raise
|
||||
|
||||
async def finish_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
generation: str,
|
||||
items: list[dict[str, Any]],
|
||||
) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
deduped = {_artifact_id(user_id, item): item for item in items}
|
||||
async with self._sf() as session:
|
||||
async with session.begin():
|
||||
# Write first so SQLite also serializes this publisher against
|
||||
# a lease takeover. Requiring status=syncing makes completion
|
||||
# one-shot even if the same callback is accidentally invoked
|
||||
# twice with an old generation token.
|
||||
fence = await session.execute(
|
||||
update(ArtifactLibrarySyncStateRow)
|
||||
.where(
|
||||
ArtifactLibrarySyncStateRow.user_id == user_id,
|
||||
ArtifactLibrarySyncStateRow.generation == generation,
|
||||
ArtifactLibrarySyncStateRow.status == "syncing",
|
||||
)
|
||||
.values(lease_expires_at=now + timedelta(seconds=300))
|
||||
)
|
||||
if fence.rowcount != 1:
|
||||
return False
|
||||
|
||||
existing_result = await session.execute(select(ArtifactLibraryItemRow).where(ArtifactLibraryItemRow.user_id == user_id))
|
||||
existing = {row.artifact_id: row for row in existing_result.scalars()}
|
||||
for artifact_id, item in deduped.items():
|
||||
row = existing.get(artifact_id)
|
||||
if row is None:
|
||||
row = ArtifactLibraryItemRow(
|
||||
artifact_id=artifact_id,
|
||||
user_id=user_id,
|
||||
thread_id="",
|
||||
thread_title="",
|
||||
source_type="general_qa",
|
||||
path="",
|
||||
name="",
|
||||
size_bytes=0,
|
||||
modified_at=_EPOCH,
|
||||
file_kind="other",
|
||||
sync_generation=generation,
|
||||
indexed_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
_apply_item(
|
||||
row,
|
||||
user_id=user_id,
|
||||
generation=generation,
|
||||
item=item,
|
||||
indexed_at=now,
|
||||
)
|
||||
await session.flush()
|
||||
await session.execute(
|
||||
delete(ArtifactLibraryItemRow).where(
|
||||
ArtifactLibraryItemRow.user_id == user_id,
|
||||
ArtifactLibraryItemRow.sync_generation != generation,
|
||||
)
|
||||
)
|
||||
await session.execute(
|
||||
update(ArtifactLibrarySyncStateRow)
|
||||
.where(
|
||||
ArtifactLibrarySyncStateRow.user_id == user_id,
|
||||
ArtifactLibrarySyncStateRow.generation == generation,
|
||||
ArtifactLibrarySyncStateRow.status == "syncing",
|
||||
)
|
||||
.values(
|
||||
status="ready",
|
||||
completed_at=now,
|
||||
lease_expires_at=None,
|
||||
last_error=None,
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
async with session.begin():
|
||||
result = await session.execute(
|
||||
update(ArtifactLibrarySyncStateRow)
|
||||
.where(
|
||||
ArtifactLibrarySyncStateRow.user_id == user_id,
|
||||
ArtifactLibrarySyncStateRow.generation == generation,
|
||||
ArtifactLibrarySyncStateRow.status == "syncing",
|
||||
)
|
||||
.values(
|
||||
status="error",
|
||||
lease_expires_at=None,
|
||||
last_error=(error or "Artifact library refresh failed")[:1024],
|
||||
)
|
||||
)
|
||||
return result.rowcount == 1
|
||||
|
||||
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArtifactLibrarySyncStateRow, user_id)
|
||||
return _state_dict(row)
|
||||
|
||||
async def list_items(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
query: str | None = None,
|
||||
file_kind: str = "all",
|
||||
source_type: str = "all",
|
||||
offset: int = 0,
|
||||
limit: int = 50,
|
||||
) -> tuple[int, list[dict[str, Any]]]:
|
||||
filters = [ArtifactLibraryItemRow.user_id == user_id]
|
||||
if file_kind != "all":
|
||||
filters.append(ArtifactLibraryItemRow.file_kind == file_kind)
|
||||
if source_type != "all":
|
||||
filters.append(ArtifactLibraryItemRow.source_type == source_type)
|
||||
normalized = (query or "").strip().casefold()
|
||||
if normalized:
|
||||
searchable = (
|
||||
ArtifactLibraryItemRow.name,
|
||||
ArtifactLibraryItemRow.thread_title,
|
||||
ArtifactLibraryItemRow.agent_name,
|
||||
ArtifactLibraryItemRow.strategy_name,
|
||||
ArtifactLibraryItemRow.strategy_project_id,
|
||||
)
|
||||
filters.append(or_(*(func.lower(func.coalesce(column, "")).contains(normalized, autoescape=True) for column in searchable)))
|
||||
|
||||
count_stmt = select(func.count()).select_from(ArtifactLibraryItemRow).where(*filters)
|
||||
page_stmt = (
|
||||
select(ArtifactLibraryItemRow)
|
||||
.where(*filters)
|
||||
.order_by(
|
||||
ArtifactLibraryItemRow.modified_at.desc(),
|
||||
ArtifactLibraryItemRow.artifact_id.desc(),
|
||||
)
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
total = int((await session.execute(count_stmt)).scalar_one())
|
||||
rows = (await session.execute(page_stmt)).scalars().all()
|
||||
return total, [_item_dict(row) for row in rows]
|
||||
+18
-1
@@ -1,4 +1,21 @@
|
||||
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
|
||||
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
|
||||
|
||||
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
|
||||
_assistant_knowledge_store = None
|
||||
|
||||
|
||||
def register_assistant_knowledge_store(store) -> None:
|
||||
global _assistant_knowledge_store
|
||||
_assistant_knowledge_store = store
|
||||
|
||||
|
||||
def get_assistant_knowledge_store():
|
||||
return _assistant_knowledge_store
|
||||
|
||||
|
||||
__all__ = [
|
||||
name
|
||||
for name in globals()
|
||||
if name.startswith("AssistantKnowledge")
|
||||
or name in {"make_assistant_knowledge_store", "register_assistant_knowledge_store", "get_assistant_knowledge_store"}
|
||||
]
|
||||
+1012
-146
File diff suppressed because it is too large.
Load diff
+2
-2
@@ -1,7 +1,7 @@
|
||||
"""ORM model for the global business → agent / chain mapping (业务链条管理).
|
||||
|
||||
A "business mapping" row binds a fixed business code (3Q / 6BF / 6-1…6-4 / 7BF /
|
||||
8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
|
||||
A "business mapping" row binds a fixed business code (3Q / 6BF / 3q-0…3q-6 /
|
||||
6-1…6-4 / 7BF / 7-1…7-4 / 8BF / AFX) to an optional 智能体 (``agent_id``) and an optional 业务链条
|
||||
(``chain_id`` → a ``roundtable_chains`` row, which must be **public** so every
|
||||
account can resolve it). Used to auto-pick the business chain when a task page
|
||||
launches the multi-agent roundtable.
|
||||
|
||||
+8
-1
@@ -21,7 +21,7 @@ _UNSET = object()
|
||||
# Fixed business rows (业务码 → 展示名). Seeded on first boot; agent/chain left empty.
|
||||
# 8BF 页面尚未建好,先占位保留行。顺序即管理表的展示顺序。
|
||||
# AFX = goPath=agentfx 深链(新系统 iframe 嵌入的智能体问答,任务详情取本系统 cop-task-detail)。
|
||||
# 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
|
||||
# 3q-0…3q-6 / 6-1…6-4 / 7-1…7-4 = 独立配置位(与 3Q/6BF/7BF 等无关),固定排在表尾。
|
||||
DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
{"business_code": "3Q", "label": "3QFX"},
|
||||
{"business_code": "6BF", "label": "任务FX"},
|
||||
@@ -29,6 +29,13 @@ DEFAULT_BUSINESS_ROWS: list[dict[str, Any]] = [
|
||||
{"business_code": "8BF", "label": "8BF"},
|
||||
{"business_code": "AFX", "label": "AGENTFX"},
|
||||
# 独立配置位,与上方业务码无关;固定排在表尾。
|
||||
{"business_code": "3q-0", "label": "3q-0"},
|
||||
{"business_code": "3q-1", "label": "3q-1"},
|
||||
{"business_code": "3q-2", "label": "3q-2"},
|
||||
{"business_code": "3q-3", "label": "3q-3"},
|
||||
{"business_code": "3q-4", "label": "3q-4"},
|
||||
{"business_code": "3q-5", "label": "3q-5"},
|
||||
{"business_code": "3q-6", "label": "3q-6"},
|
||||
{"business_code": "6-1", "label": "6-1"},
|
||||
{"business_code": "6-2", "label": "6-2"},
|
||||
{"business_code": "6-3", "label": "6-3"},
|
||||
|
||||
+5
-1
@@ -9,13 +9,17 @@ from deerflow.persistence.compact_menu_overrides.keys import (
|
||||
normalize_compact_menu_type,
|
||||
)
|
||||
from deerflow.persistence.compact_menu_overrides.memory import MemoryCompactMenuOverrideStore
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
|
||||
from deerflow.persistence.compact_menu_overrides.model import (
|
||||
CompactMenuOverrideRow,
|
||||
UserCompactMenuOverrideConfigRow,
|
||||
)
|
||||
from deerflow.persistence.compact_menu_overrides.sql import CompactMenuOverrideRepository
|
||||
|
||||
__all__ = [
|
||||
"ALLOWED_COMPACT_MENU_TYPES",
|
||||
"CompactMenuOverrideRepository",
|
||||
"CompactMenuOverrideRow",
|
||||
"UserCompactMenuOverrideConfigRow",
|
||||
"CompactMenuOverrideStore",
|
||||
"DEFAULT_COMPACT_MENU_TYPE",
|
||||
"MemoryCompactMenuOverrideStore",
|
||||
|
||||
+24
-1
@@ -1,4 +1,10 @@
|
||||
"""Abstract compact-sidebar menu override persistence API."""
|
||||
"""Abstract compact-sidebar menu override persistence API.
|
||||
|
||||
The regular rows are the administrator's default for a compact-menu variant.
|
||||
Personal layouts live in a separate complete snapshot per user and variant;
|
||||
the router overlays that snapshot while allowing newly-added default node ids
|
||||
to flow through.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -22,3 +28,20 @@ class CompactMenuOverrideStore(ABC):
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Atomically replace the compact-menu override set for one type."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
|
||||
"""Return one user's snapshot for a type, or ``None`` if absent."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
menu_type: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""Replace one user's snapshot; an empty list resets to defaults."""
|
||||
raise NotImplementedError
|
||||
+34
@@ -38,6 +38,7 @@ def _public(row: dict[str, Any]) -> dict[str, Any]:
|
||||
class MemoryCompactMenuOverrideStore(CompactMenuOverrideStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._user_rows: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
||||
|
||||
async def list_overrides(self, menu_type: str | None = None) -> list[dict[str, Any]]:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
@@ -62,3 +63,36 @@ class MemoryCompactMenuOverrideStore(CompactMenuOverrideStore):
|
||||
stored = _normalize(r, t, updated_by)
|
||||
self._rows[str(stored["node_id"])] = stored
|
||||
return await self.list_overrides(t)
|
||||
|
||||
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
rows = self._user_rows.get((str(user_id), t))
|
||||
return [dict(r) for r in rows] if rows is not None else None
|
||||
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
menu_type: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
key = (str(user_id), t)
|
||||
if not rows:
|
||||
self._user_rows.pop(key, None)
|
||||
return None
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
node_id = row.get("node_id")
|
||||
if not node_id:
|
||||
continue
|
||||
stored = _normalize(row, t, updated_by or str(user_id))
|
||||
# Expose the public id in a personal document just like the API.
|
||||
public = _public(stored)
|
||||
by_id[str(public["node_id"])] = public
|
||||
if not by_id:
|
||||
self._user_rows.pop(key, None)
|
||||
return None
|
||||
self._user_rows[key] = [dict(r) for r in by_id.values()]
|
||||
return [dict(r) for r in self._user_rows[key]]
|
||||
+15
-1
@@ -12,7 +12,7 @@ from sqlalchemy import Index, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class CompactMenuOverrideRow(Base):
|
||||
@@ -28,3 +28,17 @@ class CompactMenuOverrideRow(Base):
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
|
||||
class UserCompactMenuOverrideConfigRow(Base):
|
||||
"""A user's complete snapshot for one compact-menu variant."""
|
||||
|
||||
__tablename__ = "user_compact_menu_override_configs"
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
menu_type: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
overrides: Mapped[list[dict]] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
+88
-2
@@ -2,7 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, or_, select
|
||||
@@ -15,7 +15,10 @@ from deerflow.persistence.compact_menu_overrides.keys import (
|
||||
public_node_id,
|
||||
storage_node_id,
|
||||
)
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
|
||||
from deerflow.persistence.compact_menu_overrides.model import (
|
||||
CompactMenuOverrideRow,
|
||||
UserCompactMenuOverrideConfigRow,
|
||||
)
|
||||
|
||||
|
||||
def _to_dict(row: CompactMenuOverrideRow) -> dict[str, Any]:
|
||||
@@ -42,6 +45,41 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C
|
||||
)
|
||||
|
||||
|
||||
def _normalize_user_row(data: dict[str, Any], menu_type: str) -> dict[str, Any]:
|
||||
placement = str(data.get("placement") or "sidebar")
|
||||
return {
|
||||
"node_id": public_node_id(str(data["node_id"]), menu_type),
|
||||
"menu_type": menu_type,
|
||||
"placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar",
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"label": (data.get("label") or None),
|
||||
}
|
||||
|
||||
|
||||
def _user_rows(row: UserCompactMenuOverrideConfigRow) -> list[dict[str, Any]]:
|
||||
value = row.overrides
|
||||
if isinstance(value, str):
|
||||
import json
|
||||
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
value = []
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
menu_type = normalize_compact_menu_type(row.menu_type)
|
||||
updated_at = row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at
|
||||
return [
|
||||
{
|
||||
**_normalize_user_row(item, menu_type),
|
||||
"updated_by": row.updated_by,
|
||||
"updated_at": updated_at,
|
||||
}
|
||||
for item in value
|
||||
if isinstance(item, dict) and item.get("node_id")
|
||||
]
|
||||
|
||||
|
||||
class CompactMenuOverrideRepository(CompactMenuOverrideStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
@@ -76,6 +114,54 @@ class CompactMenuOverrideRepository(CompactMenuOverrideStore):
|
||||
await session.refresh(row)
|
||||
return [_to_dict(row) for row in new_rows]
|
||||
|
||||
async def list_user_overrides(self, user_id: str, menu_type: str | None = None) -> list[dict[str, Any]] | None:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserCompactMenuOverrideConfigRow, (str(user_id), t))
|
||||
return _user_rows(row) if row is not None else None
|
||||
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
menu_type: str | None = None,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
t = normalize_compact_menu_type(menu_type)
|
||||
key = str(user_id)
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for data in rows:
|
||||
node_id = data.get("node_id")
|
||||
if node_id:
|
||||
normalized = _normalize_user_row(data, t)
|
||||
by_id[normalized["node_id"]] = normalized
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserCompactMenuOverrideConfigRow, (key, t))
|
||||
if not by_id:
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
payload = list(by_id.values())
|
||||
if row is None:
|
||||
row = UserCompactMenuOverrideConfigRow(
|
||||
user_id=key,
|
||||
menu_type=t,
|
||||
overrides=payload,
|
||||
updated_by=updated_by or key,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.overrides = payload
|
||||
row.updated_by = updated_by or key
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
# ``expire_on_commit=False`` keeps this writer-side row usable and
|
||||
# avoids a replica-lag-prone post-commit refresh.
|
||||
return _user_rows(row)
|
||||
|
||||
|
||||
def _menu_type_filter(menu_type: str):
|
||||
if menu_type == DEFAULT_COMPACT_MENU_TYPE:
|
||||
|
||||
@@ -215,7 +215,7 @@ async def _auto_create_mysql_db(url: str) -> None:
|
||||
await maint_engine.dispose()
|
||||
|
||||
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
"""Add ORM-declared columns that are missing from already-existing tables.
|
||||
|
||||
``Base.metadata.create_all`` creates brand-new tables but never patches
|
||||
@@ -226,8 +226,8 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
|
||||
This sweep compares each existing table against the ORM metadata and issues
|
||||
``ALTER TABLE … ADD COLUMN`` for every missing column that can be added
|
||||
safely (nullable, or carrying a server_default). Indexes covering a newly
|
||||
added column are created as well. Table-level ``UniqueConstraint`` objects
|
||||
safely (nullable, or carrying a server_default). Missing ORM indexes are
|
||||
created as well. Table-level ``UniqueConstraint`` objects
|
||||
that are declared on the model but missing from the table are also created
|
||||
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
|
||||
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
|
||||
@@ -261,8 +261,7 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
if not missing:
|
||||
continue
|
||||
|
||||
added: set[str] = set()
|
||||
for col in missing:
|
||||
for col in missing:
|
||||
if not col.nullable and col.server_default is None:
|
||||
logger.warning(
|
||||
"Cannot auto-add NOT NULL column without server_default: %s.%s — add it manually",
|
||||
@@ -271,24 +270,21 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
)
|
||||
continue
|
||||
try:
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
added.add(col.name)
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
spec = CreateColumn(col).compile(dialect=sync_conn.dialect).string
|
||||
sync_conn.exec_driver_sql(f"ALTER TABLE {preparer.quote(table.name)} ADD COLUMN {spec}")
|
||||
logger.info("Auto-added missing column %s.%s", table.name, col.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-add column %s.%s", table.name, col.name)
|
||||
|
||||
if not added:
|
||||
continue
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx or not any(c.name in added for c in idx.columns):
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
try:
|
||||
existing_idx = {ix["name"] for ix in inspector.get_indexes(table.name)}
|
||||
except Exception:
|
||||
existing_idx = set()
|
||||
for idx in table.indexes:
|
||||
if idx.name in existing_idx:
|
||||
continue
|
||||
try:
|
||||
idx.create(sync_conn)
|
||||
logger.info("Auto-created index %s on %s", idx.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to auto-create index %s on %s", idx.name, table.name)
|
||||
@@ -332,15 +328,111 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
|
||||
f"ON {preparer.quote(table.name)} ({cols})"
|
||||
)
|
||||
logger.info("Auto-added unique index %s on %s", uq.name, table.name)
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
except Exception:
|
||||
logger.exception("Failed to add unique index %s on %s", uq.name, table.name)
|
||||
|
||||
|
||||
def _backfill_thread_meta_task_projections_sync(sync_conn) -> None:
|
||||
"""Populate task lookup columns added by :func:`_ensure_orm_columns_sync`.
|
||||
|
||||
Alembic auto-upgrade is intentionally disabled in deployed Gateway
|
||||
processes, while the ORM column synchronizer is always run. The latter
|
||||
can add the task lookup columns and indexes but cannot derive values from
|
||||
JSON metadata. Backfill task rows once at startup so an upgrade never
|
||||
temporarily hides every historical task conversation from ``/threads/search``.
|
||||
"""
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
table_name = "threads_meta"
|
||||
required_columns = {"metadata_json", "task_id", "agent_id", "action_id"}
|
||||
try:
|
||||
inspector = sa_inspect(sync_conn)
|
||||
if table_name not in inspector.get_table_names():
|
||||
return
|
||||
columns = {column["name"] for column in inspector.get_columns(table_name)}
|
||||
if not required_columns <= columns:
|
||||
return
|
||||
|
||||
dialect = sync_conn.dialect.name
|
||||
if dialect == "sqlite":
|
||||
task_id = "json_extract(metadata_json, '$.taskId')"
|
||||
agent_id = "json_extract(metadata_json, '$.agent_id')"
|
||||
action_id = "json_extract(metadata_json, '$.actionId')"
|
||||
valid_json = "json_valid(metadata_json)"
|
||||
elif dialect == "mysql":
|
||||
task_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.taskId'))"
|
||||
agent_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.agent_id'))"
|
||||
action_id = "JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.actionId'))"
|
||||
valid_json = "JSON_VALID(metadata_json)"
|
||||
elif dialect in {"postgresql", "opengauss"}:
|
||||
task_id = "metadata_json ->> 'taskId'"
|
||||
agent_id = "metadata_json ->> 'agent_id'"
|
||||
action_id = "metadata_json ->> 'actionId'"
|
||||
valid_json = "TRUE"
|
||||
else:
|
||||
return
|
||||
|
||||
result = sync_conn.execute(
|
||||
text(
|
||||
"UPDATE threads_meta "
|
||||
f"SET task_id = {task_id}, agent_id = {agent_id}, action_id = {action_id} "
|
||||
f"WHERE task_id IS NULL AND {valid_json} AND {task_id} IS NOT NULL"
|
||||
)
|
||||
)
|
||||
if result.rowcount:
|
||||
logger.info("Backfilled task lookup projections for %s historical thread(s)", result.rowcount)
|
||||
except Exception:
|
||||
# A malformed legacy JSON row or a backend-specific JSON function must
|
||||
# not prevent startup. The generic metadata search remains available,
|
||||
# and the failure is visible to the operator for remediation.
|
||||
logger.exception("Failed to backfill thread task lookup projections")
|
||||
|
||||
|
||||
def _is_mysql_unknown_database_error(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return "unknown database" in message or "1049" in message
|
||||
|
||||
|
||||
def _acquire_gauss_mysql_schema_bootstrap_lock():
|
||||
"""Serialize Dolphin schema bootstrap across workers on this host."""
|
||||
|
||||
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
|
||||
|
||||
if not is_gauss_mysql_product():
|
||||
return None
|
||||
try:
|
||||
import fcntl
|
||||
except ImportError:
|
||||
# Production offline deployments are Linux. Keeping a no-lock fallback
|
||||
# makes imports and unit tests work on Windows without changing MySQL.
|
||||
logger.warning("fcntl is unavailable; openGauss schema bootstrap is not cross-process locked")
|
||||
return None
|
||||
|
||||
try:
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
home = runtime_home()
|
||||
except Exception:
|
||||
home = Path(os.getenv("DEER_FLOW_HOME", ".deer-flow")).resolve()
|
||||
lock_path = home / "data" / "gauss-schema-bootstrap.lock"
|
||||
lock_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = lock_path.open("a+", encoding="utf-8")
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||
logger.info("Acquired openGauss schema bootstrap lock: %s", lock_path)
|
||||
return handle
|
||||
|
||||
|
||||
def _release_gauss_mysql_schema_bootstrap_lock(handle) -> None: # noqa: ANN001
|
||||
if handle is None:
|
||||
return
|
||||
import fcntl
|
||||
|
||||
try:
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
finally:
|
||||
handle.close()
|
||||
|
||||
|
||||
async def init_engine(
|
||||
backend: str,
|
||||
*,
|
||||
@@ -371,6 +463,10 @@ async def init_engine(
|
||||
logger.info("Persistence backend=memory -- ORM engine not initialized")
|
||||
return
|
||||
|
||||
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
|
||||
|
||||
gauss_mysql = backend == "mysql" and is_gauss_mysql_product()
|
||||
|
||||
# Remember the loop that owns the engine's connection pool, so off-loop
|
||||
# best-effort writes (metrics from worker threads, etc.) can be routed here
|
||||
# instead of reusing pooled connections cross-loop.
|
||||
@@ -394,6 +490,10 @@ async def init_engine(
|
||||
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
|
||||
"is not installed.\nInstall it with:\n uv sync --extra gauss"
|
||||
)
|
||||
if backend == "gauss":
|
||||
from deerflow.persistence.gauss_compat import install_gauss_dialect_compatibility
|
||||
|
||||
install_gauss_dialect_compatibility()
|
||||
if backend == "mysql":
|
||||
_add_runtime_vendor_path()
|
||||
driver = _mysql_driver_name(url)
|
||||
@@ -403,6 +503,10 @@ async def init_engine(
|
||||
"Install asyncmy, or run switch_database_to_mysql.sh so it can install asyncmy into "
|
||||
f"{Path('.deer-flow/python-vendor')} from the offline wheelhouse."
|
||||
)
|
||||
if gauss_mysql:
|
||||
from deerflow.persistence.gauss_mysql_compat import install_gauss_mysql_dialect_compatibility
|
||||
|
||||
install_gauss_mysql_dialect_compatibility(driver)
|
||||
|
||||
if backend == "sqlite":
|
||||
import os
|
||||
@@ -474,48 +578,98 @@ async def init_engine(
|
||||
# This is expected during initial scaffolding or minimal installs.
|
||||
logger.debug("deerflow.persistence.models not found; skipping auto-create tables")
|
||||
|
||||
async def _create_all_with_product_integrity(conn) -> None: # noqa: ANN001
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
if gauss_mysql:
|
||||
from deerflow.persistence.gauss_schema import ensure_gauss_schema_sync
|
||||
|
||||
await conn.run_sync(ensure_gauss_schema_sync, Base.metadata)
|
||||
|
||||
# Schema bootstrap is deliberately best-effort. A single incompatible or
|
||||
# permission-restricted table must not make the whole Gateway unavailable:
|
||||
# routes backed by healthy existing tables can still work, while a route
|
||||
# whose table is missing will report its own request-time error. Keep the
|
||||
# auto-create-database retries, but contain failures from both the initial
|
||||
# create_all and the retry path.
|
||||
schema_bootstrap_lock = None
|
||||
if gauss_mysql:
|
||||
try:
|
||||
schema_bootstrap_lock = _acquire_gauss_mysql_schema_bootstrap_lock()
|
||||
except Exception:
|
||||
# A lock-file permission problem is operationally important, but
|
||||
# the global startup invariant still requires the Gateway to come
|
||||
# up and report schema errors at the affected routes.
|
||||
logger.exception("Could not acquire openGauss schema bootstrap lock; continuing without the host lock")
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
except Exception as exc:
|
||||
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
|
||||
# Database not yet created — attempt to auto-create it, then retry.
|
||||
await _auto_create_postgres_db(url)
|
||||
# Rebuild engine against the now-existing database
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, pool_recycle=pool_recycle, json_serializer=_json_serializer)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
elif backend == "mysql" and _is_mysql_unknown_database_error(exc):
|
||||
# Database not yet created. Create it, rebuild the app engine
|
||||
# against the target DB, then let SQLAlchemy create all tables.
|
||||
await _auto_create_mysql_db(url)
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
else:
|
||||
raise
|
||||
await _create_all_with_product_integrity(conn)
|
||||
except Exception as exc:
|
||||
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
|
||||
# Database not yet created — attempt to auto-create it, then retry.
|
||||
await _auto_create_postgres_db(url)
|
||||
# Rebuild engine against the now-existing database.
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await _create_all_with_product_integrity(conn)
|
||||
elif backend == "mysql" and not gauss_mysql and _is_mysql_unknown_database_error(exc):
|
||||
# Database not yet created. Create it, rebuild the app engine
|
||||
# against the target DB, then let SQLAlchemy create all tables.
|
||||
await _auto_create_mysql_db(url)
|
||||
await _engine.dispose()
|
||||
_engine = create_async_engine(
|
||||
url,
|
||||
echo=echo,
|
||||
pool_size=pool_size,
|
||||
max_overflow=max_overflow,
|
||||
pool_timeout=pool_timeout,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=pool_recycle,
|
||||
json_serializer=_json_serializer,
|
||||
)
|
||||
_session_factory = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
async with _engine.begin() as conn:
|
||||
await _create_all_with_product_integrity(conn)
|
||||
else:
|
||||
raise
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Automatic ORM schema creation failed; continuing Gateway startup "
|
||||
"with the existing database schema"
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
_release_gauss_mysql_schema_bootstrap_lock(schema_bootstrap_lock)
|
||||
except Exception:
|
||||
logger.exception("Could not release openGauss schema bootstrap lock")
|
||||
|
||||
# Patch existing tables with ORM-declared columns that ``create_all``
|
||||
# cannot add (it only creates brand-new tables). Best-effort: a failure is
|
||||
# logged and startup continues on the current schema.
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(_ensure_orm_columns_sync)
|
||||
except Exception:
|
||||
logger.exception("Auto-adding missing ORM columns failed; continuing with current schema")
|
||||
if gauss_mysql:
|
||||
# Dolphin returns PostgreSQL-style DDL from SHOW CREATE TABLE. The
|
||||
# stock MySQL inspector misreads it and may issue duplicate ALTERs,
|
||||
# aborting the transaction with error 322. The native-catalog helper
|
||||
# above is the only schema repair path for this product.
|
||||
logger.info("Skipping MySQL reflection and JSON backfill for openGauss Dolphin")
|
||||
else:
|
||||
try:
|
||||
async with _engine.begin() as conn:
|
||||
await conn.run_sync(_ensure_orm_columns_sync)
|
||||
await conn.run_sync(_backfill_thread_meta_task_projections_sync)
|
||||
except Exception:
|
||||
logger.exception("Auto-updating ORM columns or task lookup projections failed; continuing with current schema")
|
||||
|
||||
# Apply in-place schema fixes (column widenings, etc.) that ``create_all``
|
||||
# cannot do on its own. Failures don't abort startup — see the helper.
|
||||
|
||||
+38
-24
@@ -1,38 +1,52 @@
|
||||
"""ORM model for user feedback on runs."""
|
||||
"""ORM model for user feedback on assistant answers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from sqlalchemy import Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
# Prefix-unique instead of UniqueConstraint: a plain unique over three
|
||||
# String(64) columns is (64+64+64)*4 = 768 bytes — one byte over MySQL's
|
||||
# legacy 767-byte index limit (utf8mb4). Capping user_id's key part at 63
|
||||
# chars (ids are 36-char UUIDs in practice) brings it to 764 bytes.
|
||||
# mysql_length is ignored on SQLite/Postgres.
|
||||
__table_args__ = (Index("uq_feedback_thread_run_user", "thread_id", "run_id", "user_id", unique=True, mysql_length={"user_id": 63}),)
|
||||
class FeedbackRow(Base):
|
||||
__tablename__ = "feedback"
|
||||
|
||||
# ``target_key`` is a SHA-256 digest of user/thread/message/agent. A
|
||||
# digest keeps the idempotency index well below MySQL's legacy utf8mb4
|
||||
# key-size limit; indexing all four source strings would exceed it.
|
||||
__table_args__ = (
|
||||
Index("uq_feedback_target_key", "target_key", unique=True),
|
||||
Index("ix_feedback_agent_created", "agent_id", "created_at"),
|
||||
)
|
||||
|
||||
feedback_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
message_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# message_id is an optional RunEventStore event identifier —
|
||||
# allows feedback to target a specific message or the entire run
|
||||
thread_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
message_id: Mapped[str | None] = mapped_column(String(64))
|
||||
# ``message_id`` is the LangChain/checkpoint message id serialized inside
|
||||
# a RunEventStore ``llm.ai.response`` event. It stays nullable solely for
|
||||
# legacy run-level feedback rows created before answer-level feedback.
|
||||
|
||||
agent_id: Mapped[str] = mapped_column(String(128), nullable=False, default="lead_agent")
|
||||
target_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
|
||||
rating: Mapped[int] = mapped_column(nullable=False)
|
||||
# +1 (thumbs-up) or -1 (thumbs-down)
|
||||
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional text feedback from the user
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
# Optional user detail. The API exposes this as ``detail`` and retains
|
||||
# ``comment`` as a compatibility alias for older clients.
|
||||
|
||||
reasons: Mapped[list[str]] = mapped_column(PortableJSON(), default=list)
|
||||
# Stable machine-readable reason codes for negative feedback.
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+225
@@ -0,0 +1,225 @@
|
||||
"""Transparent user-feedback scoring for assistant answers.
|
||||
|
||||
This module deliberately uses no model or heuristic content grading. It
|
||||
summarizes explicit thumbs feedback and selected problem reasons only. The
|
||||
result is an answer-quality signal, not a complete evaluation of an agent's
|
||||
planning, reasoning trajectory, tool use, latency, reliability, or safety.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
FEEDBACK_REASONS = (
|
||||
"incorrect_or_incomplete",
|
||||
"unmet_expectations",
|
||||
"slow_or_problematic",
|
||||
"style_or_tone",
|
||||
"safety_or_legal",
|
||||
"other",
|
||||
)
|
||||
|
||||
# The five response-quality dimensions and weights are fixed product policy.
|
||||
# A negative reason is evidence only for the explicitly mapped dimensions;
|
||||
# operational, communication, safety, and uncategorized complaints remain in
|
||||
# the problem breakdown instead of being mislabeled as task-quality evidence.
|
||||
DIMENSION_CONFIG: dict[str, dict[str, Any]] = {
|
||||
"correctness": {
|
||||
"weight": 0.30,
|
||||
"mapped_reasons": ["incorrect_or_incomplete"],
|
||||
},
|
||||
"completeness": {
|
||||
"weight": 0.20,
|
||||
"mapped_reasons": ["incorrect_or_incomplete"],
|
||||
},
|
||||
"evidence_grounding": {
|
||||
"weight": 0.20,
|
||||
"mapped_reasons": ["incorrect_or_incomplete"],
|
||||
},
|
||||
"relevance": {
|
||||
"weight": 0.15,
|
||||
"mapped_reasons": ["unmet_expectations"],
|
||||
},
|
||||
"actionability": {
|
||||
"weight": 0.15,
|
||||
"mapped_reasons": ["unmet_expectations"],
|
||||
},
|
||||
}
|
||||
|
||||
# Beta(8, 2): prior mean 80/100 with ten equivalent observations. This keeps
|
||||
# one or two votes from producing an extreme public score while real evidence
|
||||
# steadily dominates. With no real sample the API returns ``score=null`` so
|
||||
# the prior is never presented as earned performance.
|
||||
PRIOR_POSITIVE = 8
|
||||
PRIOR_NEGATIVE = 2
|
||||
PRIOR_EQUIVALENT_SAMPLES = PRIOR_POSITIVE + PRIOR_NEGATIVE
|
||||
PRIOR_MEAN = PRIOR_POSITIVE / PRIOR_EQUIVALENT_SAMPLES
|
||||
|
||||
|
||||
def _posterior_score(successes: int, failures: int) -> float | None:
|
||||
evidence_count = successes + failures
|
||||
if evidence_count == 0:
|
||||
return None
|
||||
value = 100.0 * (successes + PRIOR_POSITIVE) / (
|
||||
evidence_count + PRIOR_EQUIVALENT_SAMPLES
|
||||
)
|
||||
return round(value, 1)
|
||||
|
||||
|
||||
def _score_band(score: float | None) -> dict[str, Any]:
|
||||
if score is None:
|
||||
return {
|
||||
"key": "insufficient_data",
|
||||
"label": "数据不足",
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
}
|
||||
if score >= 90:
|
||||
return {
|
||||
"key": "excellent",
|
||||
"label": "优秀",
|
||||
"min": 90,
|
||||
"max": 100,
|
||||
}
|
||||
if score >= 80:
|
||||
return {
|
||||
"key": "good",
|
||||
"label": "良好",
|
||||
"min": 80,
|
||||
"max": 89.9,
|
||||
}
|
||||
if score >= 70:
|
||||
return {
|
||||
"key": "needs_attention",
|
||||
"label": "需关注",
|
||||
"min": 70,
|
||||
"max": 79.9,
|
||||
}
|
||||
return {
|
||||
"key": "poor",
|
||||
"label": "较差",
|
||||
"min": 0,
|
||||
"max": 69.9,
|
||||
}
|
||||
|
||||
|
||||
def _confidence_level(sample_count: int) -> str:
|
||||
if sample_count == 0:
|
||||
return "none"
|
||||
if sample_count < 10:
|
||||
return "low"
|
||||
if sample_count < 30:
|
||||
return "medium"
|
||||
return "high"
|
||||
|
||||
|
||||
def calculate_answer_quality_score(
|
||||
rows: Iterable[Mapping[str, Any]],
|
||||
*,
|
||||
agent_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Aggregate explicit answer feedback into a transparent 0-100 score."""
|
||||
|
||||
records = list(rows)
|
||||
positive_count = sum(1 for row in records if int(row.get("rating") or 0) == 1)
|
||||
negative_rows = [row for row in records if int(row.get("rating") or 0) == -1]
|
||||
negative_count = len(negative_rows)
|
||||
sample_count = positive_count + negative_count
|
||||
|
||||
reason_counts = {reason: 0 for reason in FEEDBACK_REASONS}
|
||||
negative_reason_sets: list[set[str]] = []
|
||||
for row in negative_rows:
|
||||
raw_reasons = row.get("reasons")
|
||||
if not isinstance(raw_reasons, list):
|
||||
raw_reasons = []
|
||||
reason_set = {
|
||||
str(reason)
|
||||
for reason in raw_reasons
|
||||
if str(reason) in reason_counts
|
||||
}
|
||||
negative_reason_sets.append(reason_set)
|
||||
for reason in reason_set:
|
||||
reason_counts[reason] += 1
|
||||
|
||||
dimensions: dict[str, dict[str, Any]] = {}
|
||||
weighted_total = 0.0
|
||||
available_weight = 0.0
|
||||
for name, config in DIMENSION_CONFIG.items():
|
||||
mapped = set(config["mapped_reasons"])
|
||||
failures = sum(1 for reasons in negative_reason_sets if reasons & mapped)
|
||||
evidence_count = positive_count + failures
|
||||
dimension_score = _posterior_score(positive_count, failures)
|
||||
weight = float(config["weight"])
|
||||
dimensions[name] = {
|
||||
"weight": weight,
|
||||
"score": dimension_score,
|
||||
"evidence_count": evidence_count,
|
||||
"positive_count": positive_count,
|
||||
"negative_count": failures,
|
||||
"mapped_reasons": list(config["mapped_reasons"]),
|
||||
}
|
||||
if dimension_score is not None:
|
||||
weighted_total += weight * dimension_score
|
||||
available_weight += weight
|
||||
|
||||
task_score = (
|
||||
round(weighted_total / available_weight, 1)
|
||||
if available_weight > 0
|
||||
else None
|
||||
)
|
||||
score = _posterior_score(positive_count, negative_count)
|
||||
task_success_rate = (
|
||||
round(positive_count / sample_count, 4) if sample_count else None
|
||||
)
|
||||
reason_breakdown = {
|
||||
reason: {
|
||||
"count": count,
|
||||
"rate": round(count / negative_count, 4) if negative_count else 0.0,
|
||||
}
|
||||
for reason, count in reason_counts.items()
|
||||
}
|
||||
|
||||
return {
|
||||
"agent_id": agent_id,
|
||||
"score": score,
|
||||
"score_band": _score_band(score),
|
||||
"confidence": round(sample_count / (sample_count + 10), 4),
|
||||
"confidence_level": _confidence_level(sample_count),
|
||||
"sample_count": sample_count,
|
||||
"positive_count": positive_count,
|
||||
"negative_count": negative_count,
|
||||
"task_success_rate": task_success_rate,
|
||||
"task_score": task_score,
|
||||
"dimensions": dimensions,
|
||||
"reason_breakdown": reason_breakdown,
|
||||
"prior": {
|
||||
"positive": PRIOR_POSITIVE,
|
||||
"negative": PRIOR_NEGATIVE,
|
||||
"mean": PRIOR_MEAN,
|
||||
"equivalent_samples": PRIOR_EQUIVALENT_SAMPLES,
|
||||
},
|
||||
"formula": {
|
||||
"scope": "user_feedback_answer_quality",
|
||||
"overall": (
|
||||
"100 * (positive_count + 8) / (sample_count + 10); "
|
||||
"null when sample_count is zero"
|
||||
),
|
||||
"confidence": "sample_count / (sample_count + 10)",
|
||||
"dimension": (
|
||||
"100 * (positive_count + 8) / "
|
||||
"(positive_count + mapped_negative_count + 10); "
|
||||
"null when dimension evidence_count is zero"
|
||||
),
|
||||
"task_score": (
|
||||
"weighted mean of evidenced dimensions using weights "
|
||||
"30/20/20/15/15, renormalized over available evidence"
|
||||
),
|
||||
"reason_rate_denominator": "negative_count",
|
||||
"limitations": (
|
||||
"This is not a complete agent-capability score and does not "
|
||||
"grade hidden reasoning, planning, skill/tool use, latency, "
|
||||
"reliability, or safety without separate evidence."
|
||||
),
|
||||
},
|
||||
}
|
||||
+279
-88
@@ -1,17 +1,23 @@
|
||||
"""SQLAlchemy-backed feedback storage.
|
||||
"""SQLAlchemy-backed assistant-answer feedback storage.
|
||||
|
||||
Each method acquires its own short-lived session.
|
||||
Each method acquires its own short-lived session. Writes are idempotent on a
|
||||
SHA-256 target key derived from user/thread/message/agent; the digest avoids
|
||||
oversized compound indexes on legacy MySQL utf8mb4 deployments.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import case, func, select
|
||||
from sqlalchemy import case, delete, func, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.feedback.scoring import FEEDBACK_REASONS, calculate_answer_quality_score
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
|
||||
@@ -20,12 +26,51 @@ class FeedbackRepository:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _row_to_dict(row: FeedbackRow) -> dict:
|
||||
d = row.to_dict()
|
||||
val = d.get("created_at")
|
||||
if isinstance(val, datetime):
|
||||
d["created_at"] = val.isoformat()
|
||||
return d
|
||||
def _row_to_dict(row: FeedbackRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
for key in ("created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
data[key] = value.isoformat()
|
||||
data["reasons"] = list(data.get("reasons") or [])
|
||||
data["detail"] = data.get("comment")
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def make_target_key(
|
||||
*,
|
||||
user_id: str | None,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None,
|
||||
agent_id: str,
|
||||
) -> str:
|
||||
"""Build the portable idempotency identity for one answer rating.
|
||||
|
||||
Historical run-level callers have no message id. Their deterministic
|
||||
``run:<id>`` surrogate preserves the old one-rating-per-run behavior.
|
||||
"""
|
||||
|
||||
message_key = message_id or f"run:{run_id}"
|
||||
raw = "\x1f".join(
|
||||
(
|
||||
user_id or "",
|
||||
thread_id,
|
||||
message_key,
|
||||
agent_id or "lead_agent",
|
||||
)
|
||||
)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _normalize_reasons(reasons: list[str] | None) -> list[str]:
|
||||
allowed = set(FEEDBACK_REASONS)
|
||||
normalized: list[str] = []
|
||||
for reason in reasons or []:
|
||||
value = str(reason).strip()
|
||||
if value in allowed and value not in normalized:
|
||||
normalized.append(value)
|
||||
return normalized
|
||||
|
||||
async def create(
|
||||
self,
|
||||
@@ -35,34 +80,29 @@ class FeedbackRepository:
|
||||
rating: int,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
message_id: str | None = None,
|
||||
agent_id: str = "lead_agent",
|
||||
reasons: list[str] | None = None,
|
||||
comment: str | None = None,
|
||||
) -> dict:
|
||||
"""Create a feedback record. rating must be +1 or -1."""
|
||||
if rating not in (1, -1):
|
||||
raise ValueError(f"rating must be +1 or -1, got {rating}")
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.create")
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
) -> dict[str, Any]:
|
||||
"""Create or update the same answer target idempotently."""
|
||||
|
||||
return await self.upsert(
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
message_id=message_id,
|
||||
rating=rating,
|
||||
user_id=user_id,
|
||||
message_id=message_id,
|
||||
agent_id=agent_id,
|
||||
reasons=reasons,
|
||||
comment=comment,
|
||||
created_at=datetime.now(UTC),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
feedback_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict | None:
|
||||
) -> dict[str, Any] | None:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.get")
|
||||
async with self._sf() as session:
|
||||
row = await session.get(FeedbackRow, feedback_id)
|
||||
@@ -72,38 +112,101 @@ class FeedbackRepository:
|
||||
return None
|
||||
return self._row_to_dict(row)
|
||||
|
||||
async def get_by_target(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None,
|
||||
agent_id: str,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict[str, Any] | None:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.get_by_target",
|
||||
)
|
||||
target_key = self.make_target_key(
|
||||
user_id=resolved_user_id,
|
||||
thread_id=thread_id,
|
||||
run_id=run_id,
|
||||
message_id=message_id,
|
||||
agent_id=agent_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = await session.scalar(
|
||||
select(FeedbackRow).where(FeedbackRow.target_key == target_key)
|
||||
)
|
||||
return self._row_to_dict(row) if row is not None else None
|
||||
|
||||
async def list_by_run(
|
||||
self,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
message_id: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_run")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
|
||||
) -> list[dict[str, Any]]:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.list_by_run",
|
||||
)
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
)
|
||||
if message_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.message_id == message_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_by_thread(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
limit: int = 1000,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict]:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread")
|
||||
) -> list[dict[str, Any]]:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.list_by_thread",
|
||||
)
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
stmt = stmt.order_by(FeedbackRow.created_at.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_by_message_ids(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
message_ids: list[str],
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Load current-user feedback for a bounded visible-message batch."""
|
||||
|
||||
if not message_ids:
|
||||
return []
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.list_by_message_ids",
|
||||
)
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.message_id.in_(message_ids),
|
||||
)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
@@ -111,7 +214,10 @@ class FeedbackRepository:
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete")
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.delete",
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = await session.get(FeedbackRow, feedback_id)
|
||||
if row is None:
|
||||
@@ -129,84 +235,152 @@ class FeedbackRepository:
|
||||
thread_id: str,
|
||||
rating: int,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
message_id: str | None = None,
|
||||
agent_id: str = "lead_agent",
|
||||
reasons: list[str] | None = None,
|
||||
comment: str | None = None,
|
||||
) -> dict:
|
||||
"""Create or update feedback for (thread_id, run_id, user_id). rating must be +1 or -1."""
|
||||
) -> dict[str, Any]:
|
||||
"""Create or update feedback for one user/thread/message/agent target."""
|
||||
|
||||
if rating not in (1, -1):
|
||||
raise ValueError(f"rating must be +1 or -1, got {rating}")
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.upsert")
|
||||
async with self._sf() as session:
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is not None:
|
||||
row.rating = rating
|
||||
row.comment = comment
|
||||
row.created_at = datetime.now(UTC)
|
||||
else:
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
rating=rating,
|
||||
comment=comment,
|
||||
created_at=datetime.now(UTC),
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.upsert",
|
||||
)
|
||||
normalized_agent_id = str(agent_id or "lead_agent")[:128]
|
||||
normalized_reasons = self._normalize_reasons(reasons)
|
||||
target_key = self.make_target_key(
|
||||
user_id=resolved_user_id,
|
||||
thread_id=thread_id,
|
||||
run_id=run_id,
|
||||
message_id=message_id,
|
||||
agent_id=normalized_agent_id,
|
||||
)
|
||||
|
||||
# The unique digest handles cross-worker races. On a simultaneous
|
||||
# insert, retry as an update in a fresh transaction.
|
||||
for attempt in range(2):
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
row = await session.scalar(
|
||||
select(FeedbackRow).where(FeedbackRow.target_key == target_key)
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return self._row_to_dict(row)
|
||||
if row is None:
|
||||
row = FeedbackRow(
|
||||
feedback_id=str(uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
thread_id=thread_id,
|
||||
user_id=resolved_user_id,
|
||||
message_id=message_id,
|
||||
agent_id=normalized_agent_id,
|
||||
target_key=target_key,
|
||||
rating=rating,
|
||||
reasons=normalized_reasons,
|
||||
comment=comment,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.run_id = run_id
|
||||
row.rating = rating
|
||||
row.reasons = normalized_reasons
|
||||
row.comment = comment
|
||||
row.updated_at = now
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
if attempt == 0:
|
||||
continue
|
||||
raise
|
||||
# Runtime factories use expire_on_commit=False. Avoid a
|
||||
# post-commit refresh, unsafe behind a read replica.
|
||||
return self._row_to_dict(row)
|
||||
raise RuntimeError("unreachable feedback upsert retry state")
|
||||
|
||||
async def delete_by_run(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None = None,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
"""Delete the current user's feedback for a run. Returns True if a record was deleted."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.delete_by_run")
|
||||
"""Delete this user's feedback for a run or one message in the run."""
|
||||
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.delete_by_run",
|
||||
)
|
||||
conditions = [
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
]
|
||||
if message_id is not None:
|
||||
conditions.append(FeedbackRow.message_id == message_id)
|
||||
async with self._sf() as session:
|
||||
stmt = select(FeedbackRow).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
FeedbackRow.user_id == resolved_user_id,
|
||||
)
|
||||
result = await session.execute(stmt)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
result = await session.execute(delete(FeedbackRow).where(*conditions))
|
||||
await session.commit()
|
||||
return True
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def delete_by_target(
|
||||
self,
|
||||
*,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str,
|
||||
agent_id: str,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> bool:
|
||||
resolved_user_id = resolve_user_id(
|
||||
user_id,
|
||||
method_name="FeedbackRepository.delete_by_target",
|
||||
)
|
||||
target_key = self.make_target_key(
|
||||
user_id=resolved_user_id,
|
||||
thread_id=thread_id,
|
||||
run_id=run_id,
|
||||
message_id=message_id,
|
||||
agent_id=agent_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(FeedbackRow).where(FeedbackRow.target_key == target_key)
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def list_by_thread_grouped(
|
||||
self,
|
||||
thread_id: str,
|
||||
*,
|
||||
user_id: str | None | _AutoSentinel = AUTO,
|
||||
) -> dict[str, dict]:
|
||||
"""Return feedback grouped by run_id for a thread: {run_id: feedback_dict}."""
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="FeedbackRepository.list_by_thread_grouped")
|
||||
stmt = select(FeedbackRow).where(FeedbackRow.thread_id == thread_id)
|
||||
if resolved_user_id is not None:
|
||||
stmt = stmt.where(FeedbackRow.user_id == resolved_user_id)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return {row.run_id: self._row_to_dict(row) for row in result.scalars()}
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""Legacy run-keyed view, retaining the newest row for each run."""
|
||||
|
||||
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict:
|
||||
records = await self.list_by_thread(thread_id, user_id=user_id)
|
||||
return {record["run_id"]: record for record in records}
|
||||
|
||||
async def aggregate_by_run(self, thread_id: str, run_id: str) -> dict[str, Any]:
|
||||
"""Aggregate feedback stats for a run using database-side counting."""
|
||||
|
||||
stmt = select(
|
||||
func.count().label("total"),
|
||||
func.coalesce(func.sum(case((FeedbackRow.rating == 1, 1), else_=0)), 0).label("positive"),
|
||||
func.coalesce(func.sum(case((FeedbackRow.rating == -1, 1), else_=0)), 0).label("negative"),
|
||||
).where(FeedbackRow.thread_id == thread_id, FeedbackRow.run_id == run_id)
|
||||
func.coalesce(
|
||||
func.sum(case((FeedbackRow.rating == 1, 1), else_=0)),
|
||||
0,
|
||||
).label("positive"),
|
||||
func.coalesce(
|
||||
func.sum(case((FeedbackRow.rating == -1, 1), else_=0)),
|
||||
0,
|
||||
).label("negative"),
|
||||
).where(
|
||||
FeedbackRow.thread_id == thread_id,
|
||||
FeedbackRow.run_id == run_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).one()
|
||||
return {
|
||||
@@ -215,3 +389,20 @@ class FeedbackRepository:
|
||||
"positive": row.positive,
|
||||
"negative": row.negative,
|
||||
}
|
||||
|
||||
async def aggregate_by_agent(self, agent_id: str) -> dict[str, Any]:
|
||||
"""Return the documented user-feedback answer-quality score."""
|
||||
|
||||
normalized_agent_id = str(agent_id or "lead_agent")[:128]
|
||||
stmt = select(FeedbackRow.rating, FeedbackRow.reasons).where(
|
||||
FeedbackRow.agent_id == normalized_agent_id
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return calculate_answer_quality_score(
|
||||
(
|
||||
{"rating": row.rating, "reasons": list(row.reasons or [])}
|
||||
for row in rows
|
||||
),
|
||||
agent_id=normalized_agent_id,
|
||||
)
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
"""Force-situation task detail persistence(力量态势任务详情)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.force_situation.base import ForceSituationStore
|
||||
from deerflow.persistence.force_situation.memory import MemoryForceSituationStore
|
||||
from deerflow.persistence.force_situation.model import (
|
||||
ForceSituationDetailRow,
|
||||
ForceSituationDetailVersionRow,
|
||||
)
|
||||
from deerflow.persistence.force_situation.sql import ForceSituationRepository
|
||||
|
||||
__all__ = [
|
||||
"ForceSituationDetailRow",
|
||||
"ForceSituationDetailVersionRow",
|
||||
"ForceSituationRepository",
|
||||
"ForceSituationStore",
|
||||
"MemoryForceSituationStore",
|
||||
"make_force_situation_store",
|
||||
]
|
||||
|
||||
|
||||
def make_force_situation_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryForceSituationStore()
|
||||
return ForceSituationRepository(session_factory)
|
||||
+44
@@ -0,0 +1,44 @@
|
||||
"""Persistence contract for force-situation task details(力量态势任务详情).
|
||||
|
||||
Each task owns one shared JSON document; the store never inspects the payload
|
||||
structure — fields are owned by the front-end and may evolve freely.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ForceSituationStore(ABC):
|
||||
"""Shared force-situation detail store (no user-level partitioning)."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_detail(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""Return ``{task_id, detail, updated_at, ...}`` or ``None`` when absent."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def upsert_detail(
|
||||
self,
|
||||
task_id: str,
|
||||
detail: dict[str, Any],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Replace the task's detail document wholesale; create when absent.
|
||||
|
||||
Every write also appends one version row(时间轴刻度).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_versions(self, task_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
"""Version list for the timeline, ascending by version
|
||||
(``[{version, created_at, updated_by}]``), newest last."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_version(self, task_id: str, version: int) -> dict[str, Any] | None:
|
||||
"""Return one historical version record or ``None`` when absent."""
|
||||
raise NotImplementedError
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
"""In-memory force-situation detail store (tests / no-database mode)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.force_situation.base import ForceSituationStore
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(UTC).astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
class MemoryForceSituationStore(ForceSituationStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._versions: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
async def get_detail(self, task_id: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(task_id)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def upsert_detail(
|
||||
self,
|
||||
task_id: str,
|
||||
detail: dict[str, Any],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
now = _now_iso()
|
||||
row = self._rows.get(task_id)
|
||||
if row is None:
|
||||
row = {"task_id": task_id, "created_at": now}
|
||||
self._rows[task_id] = row
|
||||
row.update(
|
||||
{
|
||||
"detail": detail,
|
||||
"updated_by": updated_by,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
versions = self._versions.setdefault(task_id, [])
|
||||
versions.append(
|
||||
{
|
||||
"version": len(versions) + 1,
|
||||
"detail": detail,
|
||||
"updated_by": updated_by,
|
||||
"created_at": now,
|
||||
}
|
||||
)
|
||||
return dict(row)
|
||||
|
||||
async def list_versions(self, task_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
versions = self._versions.get(task_id, [])
|
||||
trimmed = versions[-limit:] if limit else versions
|
||||
return [{key: row.get(key) for key in ("version", "created_at", "updated_by")} for row in trimmed]
|
||||
|
||||
async def get_version(self, task_id: str, version: int) -> dict[str, Any] | None:
|
||||
for row in self._versions.get(task_id, []):
|
||||
if row.get("version") == version:
|
||||
return dict(row)
|
||||
return None
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""ORM model for force-situation task details(力量态势任务详情).
|
||||
|
||||
新类型任务的详情文档:一个任务一份自由 JSON(含时间轴、任务衔接、页面单块内容等),
|
||||
结构由前端与产出方约定、后续可变,数据库不定死字段(整体存 ``detail_json``)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class ForceSituationDetailRow(Base):
|
||||
"""One free-form JSON detail document per external task id (current state)."""
|
||||
|
||||
__tablename__ = "force_situation_details"
|
||||
|
||||
task_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
detail_json: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="{}")
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class ForceSituationDetailVersionRow(Base):
|
||||
"""Version history for one task's detail document(时间轴 = 每次写入的版本).
|
||||
|
||||
Every PUT appends one version row; ``version`` is a 1-based sequence per task.
|
||||
"""
|
||||
|
||||
__tablename__ = "force_situation_detail_versions"
|
||||
|
||||
id: Mapped[int] = mapped_column(autoincrement=True, primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
version: Mapped[int] = mapped_column(nullable=False, default=1)
|
||||
detail_json: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="{}")
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
"""SQL repository for force-situation task details."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.force_situation.base import ForceSituationStore
|
||||
from deerflow.persistence.force_situation.model import (
|
||||
ForceSituationDetailRow,
|
||||
ForceSituationDetailVersionRow,
|
||||
)
|
||||
|
||||
|
||||
def _to_record(row: ForceSituationDetailRow) -> dict[str, Any]:
|
||||
try:
|
||||
detail = json.loads(row.detail_json or "{}")
|
||||
except json.JSONDecodeError:
|
||||
detail = {}
|
||||
return {
|
||||
"task_id": row.task_id,
|
||||
"detail": detail,
|
||||
"updated_by": row.updated_by,
|
||||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||||
"updated_at": row.updated_at.isoformat() if row.updated_at else None,
|
||||
}
|
||||
|
||||
|
||||
def _version_record(row: ForceSituationDetailVersionRow, *, with_detail: bool = False) -> dict[str, Any]:
|
||||
record: dict[str, Any] = {
|
||||
"version": row.version,
|
||||
"created_at": row.created_at.isoformat() if row.created_at else None,
|
||||
"updated_by": row.updated_by,
|
||||
}
|
||||
if with_detail:
|
||||
try:
|
||||
record["detail"] = json.loads(row.detail_json or "{}")
|
||||
except json.JSONDecodeError:
|
||||
record["detail"] = {}
|
||||
return record
|
||||
|
||||
|
||||
class ForceSituationRepository(ForceSituationStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._session_factory = session_factory
|
||||
|
||||
async def get_detail(self, task_id: str) -> dict[str, Any] | None:
|
||||
async with self._session_factory() as session:
|
||||
row = await session.get(ForceSituationDetailRow, task_id)
|
||||
return _to_record(row) if row else None
|
||||
|
||||
async def upsert_detail(
|
||||
self,
|
||||
task_id: str,
|
||||
detail: dict[str, Any],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
async with self._session_factory() as session:
|
||||
row = await session.get(ForceSituationDetailRow, task_id)
|
||||
if row is None:
|
||||
row = ForceSituationDetailRow(task_id=task_id, updated_by=updated_by)
|
||||
session.add(row)
|
||||
detail_json = json.dumps(detail, ensure_ascii=False)
|
||||
row.detail_json = detail_json
|
||||
row.updated_by = updated_by
|
||||
# 版本历史:每次写入追加一条(时间轴刻度)。
|
||||
max_version = (
|
||||
await session.execute(
|
||||
select(func.max(ForceSituationDetailVersionRow.version)).where(
|
||||
ForceSituationDetailVersionRow.task_id == task_id
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
session.add(
|
||||
ForceSituationDetailVersionRow(
|
||||
task_id=task_id,
|
||||
version=int(max_version) + 1,
|
||||
detail_json=detail_json,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_record(row)
|
||||
|
||||
async def list_versions(self, task_id: str, *, limit: int = 100) -> list[dict[str, Any]]:
|
||||
async with self._session_factory() as session:
|
||||
rows = (
|
||||
(
|
||||
await session.execute(
|
||||
select(ForceSituationDetailVersionRow)
|
||||
.where(ForceSituationDetailVersionRow.task_id == task_id)
|
||||
.order_by(ForceSituationDetailVersionRow.version.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return [_version_record(row) for row in reversed(rows)]
|
||||
|
||||
async def get_version(self, task_id: str, version: int) -> dict[str, Any] | None:
|
||||
async with self._session_factory() as session:
|
||||
row = (
|
||||
(
|
||||
await session.execute(
|
||||
select(ForceSituationDetailVersionRow).where(
|
||||
ForceSituationDetailVersionRow.task_id == task_id,
|
||||
ForceSituationDetailVersionRow.version == version,
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
return _version_record(row, with_detail=True) if row else None
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
"""Runtime compatibility fixes for the official openGauss SQLAlchemy dialect."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy.dialects.postgresql.base import PGTypeCompiler
|
||||
|
||||
|
||||
class DeerFlowOpenGaussTypeCompiler(PGTypeCompiler):
|
||||
"""Render variable strings without a length for stricter GaussDB variants.
|
||||
|
||||
Some GaussDB PostgreSQL-compatible deployments reject ``VARCHAR(n)`` at
|
||||
the numeric length token even though ordinary PostgreSQL accepts it. A
|
||||
length-less ``VARCHAR`` keeps the same variable-length string semantics on
|
||||
openGauss and avoids coupling the schema to that parser difference.
|
||||
"""
|
||||
|
||||
def visit_string(self, type_, **kw): # noqa: ANN001
|
||||
return "VARCHAR"
|
||||
|
||||
def visit_VARCHAR(self, type_, **kw): # noqa: N802, ANN001
|
||||
return "VARCHAR"
|
||||
|
||||
|
||||
def install_gauss_dialect_compatibility() -> None:
|
||||
"""Install the compiler only on the openGauss async dialect class."""
|
||||
from opengauss_sqlalchemy.asyncpg import OpenGaussDialect_asyncpg
|
||||
|
||||
OpenGaussDialect_asyncpg.type_compiler_cls = DeerFlowOpenGaussTypeCompiler
|
||||
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
"""Compatibility helpers for openGauss/Dolphin's MySQL protocol.
|
||||
|
||||
The Dolphin endpoint deliberately presents itself as MySQL, but a few pieces
|
||||
of server metadata and DDL reflection differ from real MySQL. These patches
|
||||
are installed only when the deployment explicitly selects the ``gauss_mysql``
|
||||
database product. A normal MySQL deployment never imports or installs them.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
GAUSS_MYSQL_PRODUCT = "gauss_mysql"
|
||||
GAUSS_MYSQL_SQLALCHEMY_VERSION = (5, 7, 20)
|
||||
_DOLPHIN_INIT_COMMAND = "SET standard_conforming_strings TO off"
|
||||
|
||||
|
||||
def is_gauss_mysql_product() -> bool:
|
||||
"""Return whether this process was started for the Dolphin endpoint."""
|
||||
|
||||
return os.getenv("DEERFLOW_DATABASE_PRODUCT", "").strip().lower() == GAUSS_MYSQL_PRODUCT
|
||||
|
||||
|
||||
def is_gauss_mysql_dialect(dialect: object) -> bool:
|
||||
"""Return whether the selected SQLAlchemy dialect has the Dolphin hook."""
|
||||
|
||||
return bool(getattr(dialect, "_deerflow_gauss_mysql_compat_installed", False))
|
||||
|
||||
|
||||
def _decode_scalar(value: Any) -> str:
|
||||
if isinstance(value, bytes):
|
||||
return value.decode(errors="replace")
|
||||
return str(value)
|
||||
|
||||
|
||||
def _gauss_mysql_default_schema_name(self, connection): # noqa: ANN001, ARG001
|
||||
"""Return Dolphin's current schema rather than its physical database."""
|
||||
|
||||
schema = connection.exec_driver_sql("SELECT current_schema()").scalar()
|
||||
if schema is None:
|
||||
raise RuntimeError("GaussDB did not return current_schema()")
|
||||
return _decode_scalar(schema)
|
||||
|
||||
|
||||
def _gauss_mysql_server_version_info(self, connection): # noqa: ANN001
|
||||
"""Expose a stable MySQL capability version for a Dolphin connection."""
|
||||
|
||||
dbapi_connection = connection.connection
|
||||
cursor = dbapi_connection.cursor()
|
||||
try:
|
||||
cursor.execute("SELECT VERSION()")
|
||||
row = cursor.fetchone()
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
raw_version = "" if row is None or row[0] is None else _decode_scalar(row[0])
|
||||
version = GAUSS_MYSQL_SQLALCHEMY_VERSION
|
||||
self._deerflow_gauss_server_version = raw_version
|
||||
self._set_mariadb(False, version)
|
||||
self._mariadb_normalized_version_info = version
|
||||
self.server_version_info = version
|
||||
logger.info(
|
||||
"GaussDB MySQL compatibility enabled: server=%r, SQLAlchemy capability=%s",
|
||||
raw_version,
|
||||
".".join(map(str, version)),
|
||||
)
|
||||
return version
|
||||
|
||||
|
||||
def _gauss_mysql_isolation_level(self, dbapi_connection): # noqa: ANN001, ARG001
|
||||
"""Read isolation without comparing an unparseable vendor version."""
|
||||
|
||||
cursor = dbapi_connection.cursor()
|
||||
try:
|
||||
cursor.execute("SELECT @@transaction_isolation")
|
||||
row = cursor.fetchone()
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
if row is None or row[0] is None:
|
||||
raise NotImplementedError("GaussDB did not return a transaction isolation level")
|
||||
return _decode_scalar(row[0]).upper().replace("-", " ")
|
||||
|
||||
|
||||
def _gauss_mysql_has_table(self, connection, table_name, schema=None, **kw): # noqa: ANN001, ARG001
|
||||
"""Check relation existence without a failing ``DESCRIBE`` transaction."""
|
||||
|
||||
self._ensure_has_table_connection(connection)
|
||||
if schema is None:
|
||||
schema = self.default_schema_name
|
||||
if schema is None:
|
||||
raise ValueError("GaussDB MySQL table check requires a database name")
|
||||
|
||||
statement = "SELECT 1 FROM pg_catalog.pg_class AS c JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace WHERE n.nspname = %s AND c.relname = %s AND c.relkind IN ('r', 'p', 'v', 'm', 'f') LIMIT 1"
|
||||
with connection.exec_driver_sql(
|
||||
statement,
|
||||
(schema, table_name),
|
||||
execution_options={"skip_user_error_events": True},
|
||||
) as result:
|
||||
return result.fetchone() is not None
|
||||
|
||||
|
||||
def _gauss_mysql_create_connect_args(self, url): # noqa: ANN001
|
||||
"""Add the Dolphin session setting required by PyMySQL-compatible drivers."""
|
||||
|
||||
args, kwargs = self._deerflow_gauss_mysql_original_create_connect_args(url)
|
||||
existing = kwargs.get("init_command")
|
||||
if existing and existing.strip().rstrip(";").lower() != _DOLPHIN_INIT_COMMAND.lower():
|
||||
raise ValueError(
|
||||
"gauss_mysql requires init_command='SET standard_conforming_strings TO off'; "
|
||||
"remove the conflicting init_command from the database URL"
|
||||
)
|
||||
kwargs["init_command"] = _DOLPHIN_INIT_COMMAND
|
||||
return args, kwargs
|
||||
|
||||
|
||||
def _unescape_gauss_mysql_json_container(value: str) -> Any:
|
||||
"""Undo exactly one complete MySQL SQL-string escape layer."""
|
||||
|
||||
text = value.strip()
|
||||
if not text.startswith(("[", "{")) or "\\" not in text:
|
||||
raise ValueError("Not an escaped JSON container")
|
||||
escapes = {
|
||||
"0": "\x00",
|
||||
"n": "\n",
|
||||
"r": "\r",
|
||||
"Z": "\x1a",
|
||||
"\\": "\\",
|
||||
"'": "'",
|
||||
'"': '"',
|
||||
}
|
||||
output: list[str] = []
|
||||
position = 0
|
||||
while position < len(text):
|
||||
character = text[position]
|
||||
if character == "\\":
|
||||
position += 1
|
||||
if position == len(text) or text[position] not in escapes:
|
||||
raise ValueError("Unknown or incomplete MySQL escape")
|
||||
output.append(escapes[text[position]])
|
||||
elif character in {'"', "'"}:
|
||||
raise ValueError("Mixed escaped and unescaped quotes")
|
||||
else:
|
||||
output.append(character)
|
||||
position += 1
|
||||
decoded = json.loads("".join(output))
|
||||
if not isinstance(decoded, (dict, list)):
|
||||
raise ValueError("Escaped legacy JSON must be an object or array")
|
||||
return decoded
|
||||
|
||||
|
||||
def _is_json_domain(value: Any) -> bool:
|
||||
"""Recursively reject Python-only values accepted by ``literal_eval``."""
|
||||
|
||||
if value is None or isinstance(value, (str, bool, int)):
|
||||
return True
|
||||
if isinstance(value, float):
|
||||
return math.isfinite(value)
|
||||
if isinstance(value, list):
|
||||
return all(_is_json_domain(item) for item in value)
|
||||
if isinstance(value, dict):
|
||||
return all(isinstance(key, str) and _is_json_domain(item) for key, item in value.items())
|
||||
return False
|
||||
|
||||
|
||||
def decode_gauss_mysql_json(value: Any) -> Any:
|
||||
"""Decode strict JSON plus the two legacy Dolphin encodings we shipped."""
|
||||
|
||||
if isinstance(value, memoryview):
|
||||
value = value.tobytes()
|
||||
if isinstance(value, bytes):
|
||||
value = value.decode("utf-8")
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
try:
|
||||
decoded = ast.literal_eval(value)
|
||||
except (SyntaxError, ValueError):
|
||||
try:
|
||||
return _unescape_gauss_mysql_json_container(value)
|
||||
except (TypeError, ValueError):
|
||||
preview = value[:160].replace("\n", "\\n")
|
||||
raise ValueError(f"GaussDB MySQL JSON value is neither valid JSON, a supported legacy literal, nor a single SQL-escaped JSON container: {preview!r}") from None
|
||||
if not _is_json_domain(decoded):
|
||||
raise ValueError(f"GaussDB MySQL JSON legacy literal decoded to an unsupported type: {type(decoded).__name__}")
|
||||
return decoded
|
||||
|
||||
|
||||
def install_gauss_mysql_dialect_compatibility(driver: str = "aiomysql") -> None:
|
||||
"""Patch only the selected async MySQL dialect class, once per process."""
|
||||
|
||||
if driver == "aiomysql":
|
||||
from sqlalchemy.dialects.mysql.aiomysql import MySQLDialect_aiomysql as dialect_class
|
||||
elif driver == "asyncmy":
|
||||
from sqlalchemy.dialects.mysql.asyncmy import MySQLDialect_asyncmy as dialect_class
|
||||
else:
|
||||
raise ValueError(f"Unsupported GaussDB MySQL driver: {driver}")
|
||||
|
||||
if getattr(dialect_class, "_deerflow_gauss_mysql_compat_installed", False):
|
||||
return
|
||||
|
||||
dialect_class._deerflow_gauss_mysql_original_create_connect_args = dialect_class.create_connect_args
|
||||
dialect_class.create_connect_args = _gauss_mysql_create_connect_args
|
||||
dialect_class._get_server_version_info = _gauss_mysql_server_version_info
|
||||
dialect_class._get_default_schema_name = _gauss_mysql_default_schema_name
|
||||
dialect_class.get_isolation_level = _gauss_mysql_isolation_level
|
||||
dialect_class.has_table = _gauss_mysql_has_table
|
||||
dialect_class._deerflow_gauss_mysql_compat_installed = True
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
"""Conservatively extend ORM tables through openGauss native catalogs.
|
||||
|
||||
This is used only through Dolphin's MySQL protocol. The stock MySQL inspector
|
||||
cannot parse Dolphin's PostgreSQL-style ``SHOW CREATE TABLE`` result and can
|
||||
therefore report existing columns as missing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from sqlalchemy import CheckConstraint, ForeignKeyConstraint, MetaData, UniqueConstraint
|
||||
from sqlalchemy.engine import Connection
|
||||
from sqlalchemy.schema import CreateColumn, CreateIndex
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _string(value: object) -> str:
|
||||
return value.decode("utf-8") if isinstance(value, bytes) else str(value)
|
||||
|
||||
|
||||
def ensure_gauss_schema_sync(connection: Connection, metadata: MetaData) -> None:
|
||||
"""Add only safe missing columns and indexes involving those columns.
|
||||
|
||||
Existing columns and unrelated indexes are never altered. A missing
|
||||
column without a nullable definition or database-side default cannot be
|
||||
backfilled reliably and stops this repair before any DDL is issued.
|
||||
"""
|
||||
|
||||
current_schema_value = connection.exec_driver_sql("SELECT current_schema()").scalar_one_or_none()
|
||||
if not current_schema_value:
|
||||
raise RuntimeError("openGauss schema integrity check: current_schema() is empty")
|
||||
schema = _string(current_schema_value)
|
||||
expected_schema = connection.engine.url.database
|
||||
if expected_schema and expected_schema != schema:
|
||||
raise RuntimeError(f"openGauss schema integrity check: URL schema {expected_schema!r} does not match current_schema() {schema!r}")
|
||||
|
||||
catalog_rows = connection.exec_driver_sql(
|
||||
"SELECT c.relname, c.relkind, a.attname "
|
||||
"FROM pg_catalog.pg_class AS c "
|
||||
"JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace "
|
||||
"LEFT JOIN pg_catalog.pg_attribute AS a ON a.attrelid = c.oid "
|
||||
"AND a.attnum > 0 AND NOT a.attisdropped "
|
||||
"WHERE n.nspname = %s AND c.relkind IN ('r', 'p', 'v', 'm', 'f')",
|
||||
(schema,),
|
||||
).fetchall()
|
||||
columns_by_table: dict[str, set[str]] = {}
|
||||
kinds_by_table: dict[str, str] = {}
|
||||
for table_name, kind, column_name in catalog_rows:
|
||||
table_name = _string(table_name)
|
||||
kinds_by_table[table_name] = _string(kind)
|
||||
columns_by_table.setdefault(table_name, set())
|
||||
if column_name is not None:
|
||||
columns_by_table[table_name].add(_string(column_name))
|
||||
|
||||
plan = []
|
||||
failures: list[str] = []
|
||||
for table in metadata.sorted_tables:
|
||||
if table.schema is not None and table.schema != schema:
|
||||
failures.append(f"{table.fullname}: explicit schema differs from {schema!r}")
|
||||
continue
|
||||
existing = columns_by_table.get(table.name)
|
||||
if existing is None:
|
||||
failures.append(f"{schema}.{table.name}: table is still missing after create_all")
|
||||
continue
|
||||
if kinds_by_table[table.name] not in {"r", "p"}:
|
||||
failures.append(f"{schema}.{table.name}: existing relation is not an ordinary table")
|
||||
continue
|
||||
missing = [column for column in table.columns if column.name not in existing]
|
||||
if not missing:
|
||||
continue
|
||||
missing_names = {column.name for column in missing}
|
||||
for column in missing:
|
||||
label = f"{schema}.{table.name}.{column.name}"
|
||||
if not column.nullable and column.server_default is None:
|
||||
failures.append(f"{label}: NOT NULL column has no server_default; explicit migration required")
|
||||
elif column.primary_key or column.identity is not None or column.computed is not None:
|
||||
failures.append(f"{label}: primary-key/identity/computed column requires an explicit migration")
|
||||
for constraint in table.constraints:
|
||||
if isinstance(constraint, (ForeignKeyConstraint, UniqueConstraint)):
|
||||
constrained = {column.name for column in constraint.columns}
|
||||
if constrained & missing_names:
|
||||
failures.append(f"{schema}.{table.name}: missing columns participate in {type(constraint).__name__}; explicit constraint migration required")
|
||||
elif isinstance(constraint, CheckConstraint):
|
||||
constrained = {column.name for column in constraint.columns}
|
||||
if not constrained or constrained & missing_names:
|
||||
failures.append(f"{schema}.{table.name}: CHECK-constrained schema needs an explicit migration")
|
||||
indexes = [index for index in table.indexes if {column.name for column in index.columns} & missing_names]
|
||||
for index in indexes:
|
||||
if not index.name:
|
||||
failures.append(f"{schema}.{table.name}: index involving new columns has no stable name")
|
||||
plan.append((table, missing, indexes))
|
||||
|
||||
if failures:
|
||||
raise RuntimeError("openGauss schema is incomplete; startup repair stopped:\n- " + "\n- ".join(failures))
|
||||
if not plan:
|
||||
logger.info("openGauss schema integrity check passed: %s ORM tables", len(metadata.tables))
|
||||
return
|
||||
|
||||
existing_indexes = {
|
||||
(_string(table_name), _string(index_name))
|
||||
for table_name, index_name in connection.exec_driver_sql(
|
||||
"SELECT t.relname, i.relname FROM pg_catalog.pg_index AS x "
|
||||
"JOIN pg_catalog.pg_class AS t ON t.oid = x.indrelid "
|
||||
"JOIN pg_catalog.pg_class AS i ON i.oid = x.indexrelid "
|
||||
"JOIN pg_catalog.pg_namespace AS n ON n.oid = t.relnamespace "
|
||||
"WHERE n.nspname = %s",
|
||||
(schema,),
|
||||
).fetchall()
|
||||
}
|
||||
preparer = connection.dialect.identifier_preparer
|
||||
added_columns = 0
|
||||
for table, missing, indexes in plan:
|
||||
qualified = f"{preparer.quote_schema(schema)}.{preparer.quote(table.name)}"
|
||||
for column in missing:
|
||||
definition = str(CreateColumn(column).compile(dialect=connection.dialect))
|
||||
connection.exec_driver_sql(f"ALTER TABLE {qualified} ADD COLUMN {definition}")
|
||||
added_columns += 1
|
||||
logger.info("Added safe missing openGauss column: %s.%s", qualified, column.name)
|
||||
for index in sorted(indexes, key=lambda item: str(item.name)):
|
||||
if (table.name, str(index.name)) not in existing_indexes:
|
||||
connection.execute(CreateIndex(index))
|
||||
logger.info("Created openGauss index involving new columns: %s", index.name)
|
||||
logger.info("openGauss schema integrity repair completed: %s columns", added_columns)
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Durable, owner-scoped records for the public-document workbench."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.gongwen_documents.memory import MemoryGongwenDocumentStore
|
||||
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
|
||||
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore, SqlGongwenDocumentStore
|
||||
|
||||
__all__ = [
|
||||
"GongwenDocumentRow",
|
||||
"GongwenDocumentStore",
|
||||
"GongwenDocumentVersionRow",
|
||||
"MemoryGongwenDocumentStore",
|
||||
"SqlGongwenDocumentStore",
|
||||
"make_gongwen_document_store",
|
||||
]
|
||||
|
||||
|
||||
def make_gongwen_document_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryGongwenDocumentStore()
|
||||
return SqlGongwenDocumentStore(session_factory)
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
"""Small in-memory fallback used when persistence is disabled in tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.gongwen_documents.store import GongwenDocumentStore
|
||||
|
||||
|
||||
class MemoryGongwenDocumentStore(GongwenDocumentStore):
|
||||
def __init__(self) -> None:
|
||||
self._documents: dict[str, dict[str, Any]] = {}
|
||||
self._versions: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
@staticmethod
|
||||
def _owned(document: dict[str, Any] | None, user_id: str | None) -> bool:
|
||||
return bool(document and document.get("user_id") == user_id)
|
||||
|
||||
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row = {
|
||||
"id": str(data["id"]), "user_id": data.get("user_id"), "title": str(data.get("title") or ""),
|
||||
"document_type": str(data.get("document_type") or "notice"), "source": str(data.get("source") or "upload"),
|
||||
"status": str(data.get("status") or "checking"), "template_key": data.get("template_key"),
|
||||
"research_session_id": data.get("research_session_id"), "original_version_id": None, "latest_version_id": None,
|
||||
"issue_counts": deepcopy(data.get("issue_counts")), "error": data.get("error"), "created_at": now, "updated_at": now,
|
||||
}
|
||||
self._documents[row["id"]] = row
|
||||
return deepcopy(row)
|
||||
|
||||
async def list_documents(self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._documents.values() if r.get("user_id") == user_id]
|
||||
if source:
|
||||
rows = [r for r in rows if r.get("source") == source]
|
||||
if status:
|
||||
rows = [r for r in rows if r.get("status") == status]
|
||||
return deepcopy(sorted(rows, key=lambda item: item["updated_at"], reverse=True)[:limit])
|
||||
|
||||
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
row = self._documents.get(document_id)
|
||||
return deepcopy(row) if self._owned(row, user_id) else None
|
||||
|
||||
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
|
||||
if not self._owned(self._documents.get(document_id), user_id):
|
||||
return []
|
||||
return deepcopy(list(reversed(self._versions.get(document_id, []))))
|
||||
|
||||
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
if not self._owned(self._documents.get(document_id), user_id):
|
||||
return None
|
||||
return next((deepcopy(row) for row in self._versions.get(document_id, []) if row["id"] == version_id), None)
|
||||
|
||||
async def append_version(self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
document = self._documents.get(document_id)
|
||||
if not self._owned(document, user_id):
|
||||
return None
|
||||
rows = self._versions.setdefault(document_id, [])
|
||||
row = {**deepcopy(version), "document_id": document_id, "version_no": len(rows) + 1, "created_at": datetime.now(UTC).isoformat()}
|
||||
rows.append(row)
|
||||
document.update(deepcopy(document_patch))
|
||||
document["latest_version_id"] = row["id"]
|
||||
if row["kind"] == "original" and not document.get("original_version_id"):
|
||||
document["original_version_id"] = row["id"]
|
||||
document["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return deepcopy(document), deepcopy(row)
|
||||
|
||||
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
document = self._documents.get(document_id)
|
||||
if not self._owned(document, user_id):
|
||||
return None
|
||||
document.update(deepcopy(patch))
|
||||
document["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return deepcopy(document)
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
"""ORM rows for the public-document library.
|
||||
|
||||
The binary files deliberately stay outside the database. A document row is
|
||||
the durable library entry; every uploaded original, generated output and
|
||||
format-repaired output is an immutable version row beneath it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class GongwenDocumentRow(Base):
|
||||
__tablename__ = "gongwen_documents"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
|
||||
document_type: Mapped[str] = mapped_column(String(48), nullable=False, default="notice", index=True)
|
||||
# upload | generated. Keep source separate from status so the library can
|
||||
# filter both axes independently.
|
||||
source: Mapped[str] = mapped_column(String(24), nullable=False, default="upload", index=True)
|
||||
# checking | review_required | verified | repaired | failed
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="checking", index=True)
|
||||
template_key: Mapped[str | None] = mapped_column(String(96), nullable=True)
|
||||
research_session_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
original_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
latest_version_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
issue_counts: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
|
||||
class GongwenDocumentVersionRow(Base):
|
||||
__tablename__ = "gongwen_document_versions"
|
||||
__table_args__ = (
|
||||
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
|
||||
sa.Index("ix_gongwen_document_versions_document_created", "document_id", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
document_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
version_no: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
# original | generated | repaired | markdown
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
file_name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
storage_key: Mapped[str] = mapped_column(String(1024), nullable=False)
|
||||
content_type: Mapped[str] = mapped_column(String(128), nullable=False, default="application/octet-stream")
|
||||
size_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
# Exact review snapshot for this version. Later rule updates must not
|
||||
# rewrite history or make past repair decisions impossible to explain.
|
||||
review_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
|
||||
)
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
"""SQL and in-memory stores for public-document library metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
|
||||
|
||||
|
||||
def _json_load(value: Any) -> Any:
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _json_dump(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def _as_dict(row: GongwenDocumentRow | GongwenDocumentVersionRow) -> dict[str, Any]:
|
||||
result = row.to_dict()
|
||||
for key, value in tuple(result.items()):
|
||||
if isinstance(value, datetime):
|
||||
result[key] = value.isoformat()
|
||||
if "issue_counts" in result:
|
||||
result["issue_counts"] = _json_load(result["issue_counts"])
|
||||
if "review_snapshot" in result:
|
||||
result["review_snapshot"] = _json_load(result["review_snapshot"])
|
||||
return result
|
||||
|
||||
|
||||
class GongwenDocumentStore(ABC):
|
||||
@abstractmethod
|
||||
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_documents(
|
||||
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def append_version(
|
||||
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
|
||||
) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SqlGongwenDocumentStore(GongwenDocumentStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@staticmethod
|
||||
def _owned(stmt, user_id: str | None):
|
||||
return stmt.where(GongwenDocumentRow.user_id == user_id)
|
||||
|
||||
async def create_document(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = GongwenDocumentRow(
|
||||
id=str(data["id"]),
|
||||
user_id=data.get("user_id"),
|
||||
title=str(data.get("title") or ""),
|
||||
document_type=str(data.get("document_type") or "notice"),
|
||||
source=str(data.get("source") or "upload"),
|
||||
status=str(data.get("status") or "checking"),
|
||||
template_key=data.get("template_key"),
|
||||
research_session_id=data.get("research_session_id"),
|
||||
issue_counts=_json_dump(data.get("issue_counts")),
|
||||
error=data.get("error"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _as_dict(row)
|
||||
|
||||
async def list_documents(
|
||||
self, user_id: str | None, *, source: str | None = None, status: str | None = None, limit: int = 100
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = self._owned(select(GongwenDocumentRow), user_id)
|
||||
if source:
|
||||
stmt = stmt.where(GongwenDocumentRow.source == source)
|
||||
if status:
|
||||
stmt = stmt.where(GongwenDocumentRow.status == status)
|
||||
stmt = stmt.order_by(GongwenDocumentRow.updated_at.desc()).limit(max(1, min(limit, 200)))
|
||||
async with self._sf() as session:
|
||||
return [_as_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def get_document(self, document_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _as_dict(row) if row else None
|
||||
|
||||
async def list_versions(self, document_id: str, user_id: str | None) -> list[dict[str, Any]]:
|
||||
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
async with self._sf() as session:
|
||||
if (await session.execute(stmt)).scalar_one_or_none() is None:
|
||||
return []
|
||||
versions = await session.execute(
|
||||
select(GongwenDocumentVersionRow)
|
||||
.where(GongwenDocumentVersionRow.document_id == document_id)
|
||||
.order_by(GongwenDocumentVersionRow.version_no.desc())
|
||||
)
|
||||
return [_as_dict(row) for row in versions.scalars()]
|
||||
|
||||
async def get_version(self, document_id: str, version_id: str, user_id: str | None) -> dict[str, Any] | None:
|
||||
stmt = self._owned(select(GongwenDocumentRow.id).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
async with self._sf() as session:
|
||||
if (await session.execute(stmt)).scalar_one_or_none() is None:
|
||||
return None
|
||||
row = (
|
||||
await session.execute(
|
||||
select(GongwenDocumentVersionRow).where(
|
||||
GongwenDocumentVersionRow.document_id == document_id,
|
||||
GongwenDocumentVersionRow.id == version_id,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return _as_dict(row) if row else None
|
||||
|
||||
async def append_version(
|
||||
self, document_id: str, user_id: str | None, version: dict[str, Any], document_patch: dict[str, Any]
|
||||
) -> tuple[dict[str, Any], dict[str, Any]] | None:
|
||||
async with self._sf() as session:
|
||||
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
document = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if document is None:
|
||||
return None
|
||||
version_no = int(
|
||||
(await session.execute(
|
||||
select(func.coalesce(func.max(GongwenDocumentVersionRow.version_no), 0)).where(
|
||||
GongwenDocumentVersionRow.document_id == document_id
|
||||
)
|
||||
)).scalar_one()
|
||||
) + 1
|
||||
row = GongwenDocumentVersionRow(
|
||||
id=str(version["id"]),
|
||||
document_id=document_id,
|
||||
version_no=version_no,
|
||||
kind=str(version["kind"]),
|
||||
file_name=str(version["file_name"]),
|
||||
storage_key=str(version["storage_key"]),
|
||||
content_type=str(version.get("content_type") or "application/octet-stream"),
|
||||
size_bytes=int(version.get("size_bytes") or 0),
|
||||
sha256=str(version.get("sha256") or ""),
|
||||
review_snapshot=_json_dump(version.get("review_snapshot")),
|
||||
)
|
||||
for key, value in document_patch.items():
|
||||
if key == "issue_counts":
|
||||
value = _json_dump(value)
|
||||
if hasattr(document, key):
|
||||
setattr(document, key, value)
|
||||
document.latest_version_id = row.id
|
||||
if row.kind == "original" and not document.original_version_id:
|
||||
document.original_version_id = row.id
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(document)
|
||||
await session.refresh(row)
|
||||
return _as_dict(document), _as_dict(row)
|
||||
|
||||
async def update_document(self, document_id: str, user_id: str | None, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
stmt = self._owned(select(GongwenDocumentRow).where(GongwenDocumentRow.id == document_id), user_id)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
for key, value in patch.items():
|
||||
if key == "issue_counts":
|
||||
value = _json_dump(value)
|
||||
if hasattr(row, key):
|
||||
setattr(row, key, value)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _as_dict(row)
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""Database-backed global ordinary-knowledge retrieval settings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.knowledge_retrieval_settings.base import (
|
||||
KnowledgeRetrievalSettingsStore,
|
||||
)
|
||||
from deerflow.persistence.knowledge_retrieval_settings.memory import (
|
||||
MemoryKnowledgeRetrievalSettingsStore,
|
||||
)
|
||||
from deerflow.persistence.knowledge_retrieval_settings.model import (
|
||||
KnowledgeRetrievalSettingsRow,
|
||||
)
|
||||
from deerflow.persistence.knowledge_retrieval_settings.sql import (
|
||||
KnowledgeRetrievalSettingsRepository,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"KnowledgeRetrievalSettingsRepository",
|
||||
"KnowledgeRetrievalSettingsRow",
|
||||
"KnowledgeRetrievalSettingsStore",
|
||||
"MemoryKnowledgeRetrievalSettingsStore",
|
||||
"make_knowledge_retrieval_settings_store",
|
||||
]
|
||||
|
||||
|
||||
def make_knowledge_retrieval_settings_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryKnowledgeRetrievalSettingsStore()
|
||||
return KnowledgeRetrievalSettingsRepository(session_factory)
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
"""Persistence contract for the global ordinary-knowledge retrieval switch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class KnowledgeRetrievalSettingsStore(ABC):
|
||||
"""Store the deployment-wide smart-reasoning availability flag."""
|
||||
|
||||
@abstractmethod
|
||||
async def get(self) -> bool:
|
||||
"""Return whether smart reasoning is enabled; missing state is disabled."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
|
||||
"""Atomically replace the global flag and return the saved value."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_advanced_management_enabled(self) -> bool:
|
||||
"""Return whether advanced knowledge-management entries are exposed."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_advanced_management_enabled(
|
||||
self,
|
||||
enabled: bool,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
"""Atomically replace the advanced-management flag."""
|
||||
|
||||
raise NotImplementedError
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
"""In-memory ordinary-knowledge retrieval settings store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from deerflow.persistence.knowledge_retrieval_settings.base import (
|
||||
KnowledgeRetrievalSettingsStore,
|
||||
)
|
||||
|
||||
|
||||
class MemoryKnowledgeRetrievalSettingsStore(KnowledgeRetrievalSettingsStore):
|
||||
def __init__(self) -> None:
|
||||
self._enabled = False
|
||||
self._advanced_management_enabled = False
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def get(self) -> bool:
|
||||
async with self._lock:
|
||||
return self._enabled
|
||||
|
||||
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
|
||||
del updated_by
|
||||
async with self._lock:
|
||||
self._enabled = bool(enabled)
|
||||
return self._enabled
|
||||
|
||||
async def get_advanced_management_enabled(self) -> bool:
|
||||
async with self._lock:
|
||||
return self._advanced_management_enabled
|
||||
|
||||
async def set_advanced_management_enabled(
|
||||
self,
|
||||
enabled: bool,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
del updated_by
|
||||
async with self._lock:
|
||||
self._advanced_management_enabled = bool(enabled)
|
||||
return self._advanced_management_enabled
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
"""ORM row for the database-backed ordinary-knowledge retrieval switch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, false
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class KnowledgeRetrievalSettingsRow(Base):
|
||||
__tablename__ = "knowledge_retrieval_settings"
|
||||
|
||||
scope: Mapped[str] = mapped_column(String(64), primary_key=True, default="global")
|
||||
smart_reasoning_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
server_default=false(),
|
||||
)
|
||||
advanced_management_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
server_default=false(),
|
||||
)
|
||||
revision: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
"""SQLAlchemy-backed ordinary-knowledge retrieval settings store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.knowledge_retrieval_settings.base import (
|
||||
KnowledgeRetrievalSettingsStore,
|
||||
)
|
||||
from deerflow.persistence.knowledge_retrieval_settings.model import (
|
||||
KnowledgeRetrievalSettingsRow,
|
||||
)
|
||||
|
||||
_GLOBAL_SCOPE = "global"
|
||||
|
||||
|
||||
class KnowledgeRetrievalSettingsRepository(KnowledgeRetrievalSettingsStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def get(self) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeRetrievalSettingsRow, _GLOBAL_SCOPE)
|
||||
return bool(row.smart_reasoning_enabled) if row is not None else False
|
||||
|
||||
async def get_advanced_management_enabled(self) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(KnowledgeRetrievalSettingsRow, _GLOBAL_SCOPE)
|
||||
return bool(row.advanced_management_enabled) if row is not None else False
|
||||
|
||||
async def set(self, enabled: bool, *, updated_by: str | None = None) -> bool:
|
||||
saved = bool(enabled)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(KnowledgeRetrievalSettingsRow)
|
||||
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
|
||||
.with_for_update()
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
row = KnowledgeRetrievalSettingsRow(
|
||||
scope=_GLOBAL_SCOPE,
|
||||
smart_reasoning_enabled=saved,
|
||||
revision=1,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.smart_reasoning_enabled = saved
|
||||
row.updated_by = updated_by
|
||||
row.revision = int(row.revision or 0) + 1
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Two workers may both observe the initially absent singleton.
|
||||
# The loser retries as an update instead of surfacing a 500.
|
||||
await session.rollback()
|
||||
result = await session.execute(
|
||||
select(KnowledgeRetrievalSettingsRow)
|
||||
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
|
||||
.with_for_update()
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
raise
|
||||
row.smart_reasoning_enabled = saved
|
||||
row.updated_by = updated_by
|
||||
row.revision = int(row.revision or 0) + 1
|
||||
await session.commit()
|
||||
return saved
|
||||
|
||||
async def set_advanced_management_enabled(
|
||||
self,
|
||||
enabled: bool,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
saved = bool(enabled)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(KnowledgeRetrievalSettingsRow)
|
||||
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
|
||||
.with_for_update()
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
row = KnowledgeRetrievalSettingsRow(
|
||||
scope=_GLOBAL_SCOPE,
|
||||
smart_reasoning_enabled=False,
|
||||
advanced_management_enabled=saved,
|
||||
revision=1,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.advanced_management_enabled = saved
|
||||
row.updated_by = updated_by
|
||||
row.revision = int(row.revision or 0) + 1
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
result = await session.execute(
|
||||
select(KnowledgeRetrievalSettingsRow)
|
||||
.where(KnowledgeRetrievalSettingsRow.scope == _GLOBAL_SCOPE)
|
||||
.with_for_update()
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
raise
|
||||
row.advanced_management_enabled = saved
|
||||
row.updated_by = updated_by
|
||||
row.revision = int(row.revision or 0) + 1
|
||||
await session.commit()
|
||||
return saved
|
||||
+50
-1
@@ -6,11 +6,47 @@ from abc import ABC, abstractmethod
|
||||
from typing import Any, Literal
|
||||
|
||||
LlmWikiScope = Literal["personal", "public", "all"]
|
||||
UNSET = object()
|
||||
|
||||
|
||||
def serialize_wiki_parent_directories(
|
||||
value: list[str] | None,
|
||||
) -> list[str] | None:
|
||||
return None if value is None else [str(item) for item in value]
|
||||
|
||||
|
||||
def serialize_wiki_directory_paths(
|
||||
value: list[list[str]] | None,
|
||||
) -> list[list[str]] | None:
|
||||
return (
|
||||
None
|
||||
if value is None
|
||||
else [[str(segment) for segment in path] for path in value]
|
||||
)
|
||||
|
||||
|
||||
class LlmWikiStore(ABC):
|
||||
@abstractmethod
|
||||
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]: ...
|
||||
async def create_mapping(
|
||||
self,
|
||||
*,
|
||||
weknora_id: str,
|
||||
owner_user_id: str,
|
||||
name: str,
|
||||
description: str,
|
||||
kb_type: str,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_parent_directories: list[str] | None = None,
|
||||
wiki_directory_paths: list[list[str]] | None = None,
|
||||
wiki_focus_instructions: str | None = None,
|
||||
wiki_content_instructions_raw: str | None = None,
|
||||
wiki_extraction_instructions_raw: str | None = None,
|
||||
wiki_generation_managed: bool | None = None,
|
||||
wiki_config_schema_version: int | None = None,
|
||||
wiki_config_revision: int | None = None,
|
||||
wiki_config_apply_status: str | None = None,
|
||||
wiki_config_apply_error: str | None = None,
|
||||
) -> dict[str, Any]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]: ...
|
||||
@@ -28,8 +64,21 @@ class LlmWikiStore(ABC):
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
test_questions: list[str] | None = None,
|
||||
test_questions_set: bool = False,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
wiki_parent_directories: list[str] | None | object = UNSET,
|
||||
wiki_directory_paths: list[list[str]] | None | object = UNSET,
|
||||
wiki_focus_instructions: str | None | object = UNSET,
|
||||
wiki_content_instructions_raw: str | None | object = UNSET,
|
||||
wiki_extraction_instructions_raw: str | None | object = UNSET,
|
||||
wiki_generation_managed: bool | None | object = UNSET,
|
||||
wiki_config_schema_version: int | None | object = UNSET,
|
||||
wiki_config_revision: int | None | object = UNSET,
|
||||
wiki_config_apply_status: str | None | object = UNSET,
|
||||
wiki_config_apply_error: str | None | object = UNSET,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
|
||||
+136
-15
@@ -3,22 +3,58 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
|
||||
from deerflow.persistence.llmwiki.base import (
|
||||
UNSET,
|
||||
LlmWikiScope,
|
||||
LlmWikiStore,
|
||||
serialize_wiki_directory_paths,
|
||||
serialize_wiki_parent_directories,
|
||||
)
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _mapping_to_dict(row: dict[str, Any]) -> dict[str, Any]:
|
||||
data = dict(row)
|
||||
data["wiki_parent_directories"] = serialize_wiki_parent_directories(
|
||||
data.get("wiki_parent_directories")
|
||||
)
|
||||
data["wiki_directory_paths"] = serialize_wiki_directory_paths(
|
||||
data.get("wiki_directory_paths")
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
class MemoryLlmWikiStore(LlmWikiStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._conversation_deposits: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
|
||||
async def create_mapping(
|
||||
self,
|
||||
*,
|
||||
weknora_id: str,
|
||||
owner_user_id: str,
|
||||
name: str,
|
||||
description: str,
|
||||
kb_type: str,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_parent_directories: list[str] | None = None,
|
||||
wiki_directory_paths: list[list[str]] | None = None,
|
||||
wiki_focus_instructions: str | None = None,
|
||||
wiki_content_instructions_raw: str | None = None,
|
||||
wiki_extraction_instructions_raw: str | None = None,
|
||||
wiki_generation_managed: bool | None = None,
|
||||
wiki_config_schema_version: int | None = None,
|
||||
wiki_config_revision: int | None = None,
|
||||
wiki_config_apply_status: str | None = None,
|
||||
wiki_config_apply_error: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if any(row["weknora_id"] == weknora_id for row in self._rows.values()):
|
||||
raise ValueError("WeKnora knowledge base is already mapped")
|
||||
now = _now()
|
||||
@@ -28,6 +64,26 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
"owner_user_id": owner_user_id,
|
||||
"name": name,
|
||||
"description": description,
|
||||
"test_questions": None,
|
||||
"retrieval_settings": (
|
||||
dict(retrieval_settings)
|
||||
if retrieval_settings is not None
|
||||
else None
|
||||
),
|
||||
"wiki_parent_directories": serialize_wiki_parent_directories(
|
||||
wiki_parent_directories
|
||||
),
|
||||
"wiki_directory_paths": serialize_wiki_directory_paths(
|
||||
wiki_directory_paths
|
||||
),
|
||||
"wiki_focus_instructions": wiki_focus_instructions,
|
||||
"wiki_content_instructions_raw": wiki_content_instructions_raw,
|
||||
"wiki_extraction_instructions_raw": wiki_extraction_instructions_raw,
|
||||
"wiki_generation_managed": wiki_generation_managed,
|
||||
"wiki_config_schema_version": wiki_config_schema_version,
|
||||
"wiki_config_revision": wiki_config_revision,
|
||||
"wiki_config_apply_status": wiki_config_apply_status,
|
||||
"wiki_config_apply_error": wiki_config_apply_error,
|
||||
"kb_type": kb_type,
|
||||
"publication_status": "private",
|
||||
"published_by": None,
|
||||
@@ -40,7 +96,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
"updated_at": now,
|
||||
}
|
||||
self._rows[row["id"]] = row
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def list_visible(self, user_id: str, *, scope: LlmWikiScope = "all", is_admin: bool = False) -> list[dict[str, Any]]:
|
||||
rows = list(self._rows.values())
|
||||
@@ -50,22 +106,22 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
rows = [row for row in rows if row["publication_status"] == "published"]
|
||||
else:
|
||||
rows = rows if is_admin else [row for row in rows if row["owner_user_id"] == user_id or row["publication_status"] == "published"]
|
||||
return [dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
|
||||
return [_mapping_to_dict(row) for row in sorted(rows, key=lambda row: row["created_at"], reverse=True)]
|
||||
|
||||
async def get_authorized(self, mapping_id: str, user_id: str, *, write: bool, is_admin: bool) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None:
|
||||
return None
|
||||
if is_admin or row["owner_user_id"] == user_id:
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
if not write and row["publication_status"] == "published":
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
return None
|
||||
|
||||
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None:
|
||||
for row in self._rows.values():
|
||||
if row["weknora_id"] == weknora_id:
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
return None
|
||||
|
||||
async def update_mapping(
|
||||
@@ -74,8 +130,21 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
test_questions: list[str] | None = None,
|
||||
test_questions_set: bool = False,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
wiki_parent_directories: list[str] | None | object = UNSET,
|
||||
wiki_directory_paths: list[list[str]] | None | object = UNSET,
|
||||
wiki_focus_instructions: str | None | object = UNSET,
|
||||
wiki_content_instructions_raw: str | None | object = UNSET,
|
||||
wiki_extraction_instructions_raw: str | None | object = UNSET,
|
||||
wiki_generation_managed: bool | None | object = UNSET,
|
||||
wiki_config_schema_version: int | None | object = UNSET,
|
||||
wiki_config_revision: int | None | object = UNSET,
|
||||
wiki_config_apply_status: str | None | object = UNSET,
|
||||
wiki_config_apply_error: str | None | object = UNSET,
|
||||
) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
if row is None:
|
||||
@@ -84,13 +153,65 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
row["name"] = name
|
||||
if description is not None:
|
||||
row["description"] = description
|
||||
if test_questions_set:
|
||||
row["test_questions"] = list(test_questions or [])
|
||||
if retrieval_settings is not None:
|
||||
row["retrieval_settings"] = dict(retrieval_settings)
|
||||
if wiki_index_enabled is not None:
|
||||
row["wiki_index_enabled"] = wiki_index_enabled
|
||||
if external_search_enabled is not None:
|
||||
row["external_search_enabled"] = external_search_enabled
|
||||
row["external_search_updated_at"] = _now()
|
||||
if wiki_parent_directories is not UNSET:
|
||||
row["wiki_parent_directories"] = serialize_wiki_parent_directories(
|
||||
cast(list[str] | None, wiki_parent_directories)
|
||||
)
|
||||
if wiki_directory_paths is not UNSET:
|
||||
row["wiki_directory_paths"] = serialize_wiki_directory_paths(
|
||||
cast(list[list[str]] | None, wiki_directory_paths)
|
||||
)
|
||||
if wiki_focus_instructions is not UNSET:
|
||||
row["wiki_focus_instructions"] = cast(
|
||||
str | None,
|
||||
wiki_focus_instructions,
|
||||
)
|
||||
if wiki_content_instructions_raw is not UNSET:
|
||||
row["wiki_content_instructions_raw"] = cast(
|
||||
str | None,
|
||||
wiki_content_instructions_raw,
|
||||
)
|
||||
if wiki_extraction_instructions_raw is not UNSET:
|
||||
row["wiki_extraction_instructions_raw"] = cast(
|
||||
str | None,
|
||||
wiki_extraction_instructions_raw,
|
||||
)
|
||||
if wiki_generation_managed is not UNSET:
|
||||
row["wiki_generation_managed"] = cast(
|
||||
bool | None,
|
||||
wiki_generation_managed,
|
||||
)
|
||||
if wiki_config_schema_version is not UNSET:
|
||||
row["wiki_config_schema_version"] = cast(
|
||||
int | None,
|
||||
wiki_config_schema_version,
|
||||
)
|
||||
if wiki_config_revision is not UNSET:
|
||||
row["wiki_config_revision"] = cast(
|
||||
int | None,
|
||||
wiki_config_revision,
|
||||
)
|
||||
if wiki_config_apply_status is not UNSET:
|
||||
row["wiki_config_apply_status"] = cast(
|
||||
str | None,
|
||||
wiki_config_apply_status,
|
||||
)
|
||||
if wiki_config_apply_error is not UNSET:
|
||||
row["wiki_config_apply_error"] = cast(
|
||||
str | None,
|
||||
wiki_config_apply_error,
|
||||
)
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def adopt_conversation_deposit_mapping(
|
||||
self,
|
||||
@@ -115,7 +236,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def delete_mapping(self, mapping_id: str) -> bool:
|
||||
return self._rows.pop(mapping_id, None) is not None
|
||||
@@ -129,7 +250,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
row["reviewed_by"] = None
|
||||
row["published_at"] = _now()
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def review_publish(self, mapping_id: str, reviewer_user_id: str, *, approved: bool) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
@@ -139,7 +260,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
row["reviewed_by"] = reviewer_user_id
|
||||
row["published_at"] = _now() if approved else None
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def unpublish(self, mapping_id: str, actor_user_id: str, *, is_admin: bool) -> dict[str, Any] | None:
|
||||
row = self._rows.get(mapping_id)
|
||||
@@ -149,18 +270,18 @@ class MemoryLlmWikiStore(LlmWikiStore):
|
||||
row["reviewed_by"] = actor_user_id if is_admin else row.get("reviewed_by")
|
||||
row["published_at"] = None
|
||||
row["updated_at"] = _now()
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
|
||||
async def list_publications(self, *, status: str | None = None) -> list[dict[str, Any]]:
|
||||
rows = [row for row in self._rows.values() if row["publication_status"] != "private"]
|
||||
if status:
|
||||
rows = [row for row in rows if row["publication_status"] == status]
|
||||
return [dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
|
||||
return [_mapping_to_dict(row) for row in sorted(rows, key=lambda row: row["updated_at"], reverse=True)]
|
||||
|
||||
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
|
||||
for row in self._rows.values():
|
||||
if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
|
||||
return dict(row)
|
||||
return _mapping_to_dict(row)
|
||||
return None
|
||||
|
||||
async def get_conversation_deposit(
|
||||
|
||||
+17
-2
@@ -4,11 +4,11 @@ from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, String, UniqueConstraint, false, true
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint, false, true
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class LlmWikiKnowledgeBaseRow(Base):
|
||||
@@ -19,6 +19,21 @@ class LlmWikiKnowledgeBaseRow(Base):
|
||||
owner_user_id: Mapped[str] = mapped_column(String(64), index=True)
|
||||
name: Mapped[str] = mapped_column(String(255), default="")
|
||||
description: Mapped[str] = mapped_column(String(2048), default="")
|
||||
# ``None`` means "use the product defaults derived from the knowledge-base
|
||||
# name". An explicit empty list is meaningful: the owner disabled starter
|
||||
# questions for this library.
|
||||
test_questions: Mapped[list | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
retrieval_settings: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
wiki_parent_directories: Mapped[list[str] | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
wiki_directory_paths: Mapped[list[list[str]] | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
wiki_focus_instructions: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
wiki_content_instructions_raw: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
wiki_extraction_instructions_raw: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
wiki_generation_managed: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
|
||||
wiki_config_schema_version: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
wiki_config_revision: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
wiki_config_apply_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
wiki_config_apply_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
kb_type: Mapped[str] = mapped_column(String(32), default="document")
|
||||
publication_status: Mapped[str] = mapped_column(String(16), default="private", index=True)
|
||||
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
|
||||
+120
-3
@@ -3,13 +3,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.llmwiki.base import LlmWikiScope, LlmWikiStore
|
||||
from deerflow.persistence.llmwiki.base import (
|
||||
UNSET,
|
||||
LlmWikiScope,
|
||||
LlmWikiStore,
|
||||
serialize_wiki_directory_paths,
|
||||
serialize_wiki_parent_directories,
|
||||
)
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
|
||||
|
||||
@@ -19,6 +25,14 @@ def _now() -> datetime:
|
||||
|
||||
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
if "wiki_parent_directories" in data:
|
||||
data["wiki_parent_directories"] = serialize_wiki_parent_directories(
|
||||
data["wiki_parent_directories"]
|
||||
)
|
||||
if "wiki_directory_paths" in data:
|
||||
data["wiki_directory_paths"] = serialize_wiki_directory_paths(
|
||||
data["wiki_directory_paths"]
|
||||
)
|
||||
for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
|
||||
value = data.get(key)
|
||||
if isinstance(value, datetime):
|
||||
@@ -30,13 +44,51 @@ class LlmWikiRepository(LlmWikiStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def create_mapping(self, *, weknora_id: str, owner_user_id: str, name: str, description: str, kb_type: str) -> dict[str, Any]:
|
||||
async def create_mapping(
|
||||
self,
|
||||
*,
|
||||
weknora_id: str,
|
||||
owner_user_id: str,
|
||||
name: str,
|
||||
description: str,
|
||||
kb_type: str,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_parent_directories: list[str] | None = None,
|
||||
wiki_directory_paths: list[list[str]] | None = None,
|
||||
wiki_focus_instructions: str | None = None,
|
||||
wiki_content_instructions_raw: str | None = None,
|
||||
wiki_extraction_instructions_raw: str | None = None,
|
||||
wiki_generation_managed: bool | None = None,
|
||||
wiki_config_schema_version: int | None = None,
|
||||
wiki_config_revision: int | None = None,
|
||||
wiki_config_apply_status: str | None = None,
|
||||
wiki_config_apply_error: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
row = LlmWikiKnowledgeBaseRow(
|
||||
id=str(uuid4()),
|
||||
weknora_id=weknora_id,
|
||||
owner_user_id=owner_user_id,
|
||||
name=name,
|
||||
description=description,
|
||||
retrieval_settings=(
|
||||
dict(retrieval_settings)
|
||||
if retrieval_settings is not None
|
||||
else None
|
||||
),
|
||||
wiki_parent_directories=serialize_wiki_parent_directories(
|
||||
wiki_parent_directories
|
||||
),
|
||||
wiki_directory_paths=serialize_wiki_directory_paths(
|
||||
wiki_directory_paths
|
||||
),
|
||||
wiki_focus_instructions=wiki_focus_instructions,
|
||||
wiki_content_instructions_raw=wiki_content_instructions_raw,
|
||||
wiki_extraction_instructions_raw=wiki_extraction_instructions_raw,
|
||||
wiki_generation_managed=wiki_generation_managed,
|
||||
wiki_config_schema_version=wiki_config_schema_version,
|
||||
wiki_config_revision=wiki_config_revision,
|
||||
wiki_config_apply_status=wiki_config_apply_status,
|
||||
wiki_config_apply_error=wiki_config_apply_error,
|
||||
kb_type=kb_type,
|
||||
publication_status="private",
|
||||
created_at=_now(),
|
||||
@@ -88,8 +140,21 @@ class LlmWikiRepository(LlmWikiStore):
|
||||
*,
|
||||
name: str | None = None,
|
||||
description: str | None = None,
|
||||
test_questions: list[str] | None = None,
|
||||
test_questions_set: bool = False,
|
||||
retrieval_settings: dict[str, Any] | None = None,
|
||||
wiki_index_enabled: bool | None = None,
|
||||
external_search_enabled: bool | None = None,
|
||||
wiki_parent_directories: list[str] | None | object = UNSET,
|
||||
wiki_directory_paths: list[list[str]] | None | object = UNSET,
|
||||
wiki_focus_instructions: str | None | object = UNSET,
|
||||
wiki_content_instructions_raw: str | None | object = UNSET,
|
||||
wiki_extraction_instructions_raw: str | None | object = UNSET,
|
||||
wiki_generation_managed: bool | None | object = UNSET,
|
||||
wiki_config_schema_version: int | None | object = UNSET,
|
||||
wiki_config_revision: int | None | object = UNSET,
|
||||
wiki_config_apply_status: str | None | object = UNSET,
|
||||
wiki_config_apply_error: str | None | object = UNSET,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
|
||||
@@ -99,11 +164,63 @@ class LlmWikiRepository(LlmWikiStore):
|
||||
row.name = name
|
||||
if description is not None:
|
||||
row.description = description
|
||||
if test_questions_set:
|
||||
row.test_questions = list(test_questions or [])
|
||||
if retrieval_settings is not None:
|
||||
row.retrieval_settings = dict(retrieval_settings)
|
||||
if wiki_index_enabled is not None:
|
||||
row.wiki_index_enabled = wiki_index_enabled
|
||||
if external_search_enabled is not None:
|
||||
row.external_search_enabled = external_search_enabled
|
||||
row.external_search_updated_at = _now()
|
||||
if wiki_parent_directories is not UNSET:
|
||||
row.wiki_parent_directories = serialize_wiki_parent_directories(
|
||||
cast(list[str] | None, wiki_parent_directories)
|
||||
)
|
||||
if wiki_directory_paths is not UNSET:
|
||||
row.wiki_directory_paths = serialize_wiki_directory_paths(
|
||||
cast(list[list[str]] | None, wiki_directory_paths)
|
||||
)
|
||||
if wiki_focus_instructions is not UNSET:
|
||||
row.wiki_focus_instructions = cast(
|
||||
str | None,
|
||||
wiki_focus_instructions,
|
||||
)
|
||||
if wiki_content_instructions_raw is not UNSET:
|
||||
row.wiki_content_instructions_raw = cast(
|
||||
str | None,
|
||||
wiki_content_instructions_raw,
|
||||
)
|
||||
if wiki_extraction_instructions_raw is not UNSET:
|
||||
row.wiki_extraction_instructions_raw = cast(
|
||||
str | None,
|
||||
wiki_extraction_instructions_raw,
|
||||
)
|
||||
if wiki_generation_managed is not UNSET:
|
||||
row.wiki_generation_managed = cast(
|
||||
bool | None,
|
||||
wiki_generation_managed,
|
||||
)
|
||||
if wiki_config_schema_version is not UNSET:
|
||||
row.wiki_config_schema_version = cast(
|
||||
int | None,
|
||||
wiki_config_schema_version,
|
||||
)
|
||||
if wiki_config_revision is not UNSET:
|
||||
row.wiki_config_revision = cast(
|
||||
int | None,
|
||||
wiki_config_revision,
|
||||
)
|
||||
if wiki_config_apply_status is not UNSET:
|
||||
row.wiki_config_apply_status = cast(
|
||||
str | None,
|
||||
wiki_config_apply_status,
|
||||
)
|
||||
if wiki_config_apply_error is not UNSET:
|
||||
row.wiki_config_apply_error = cast(
|
||||
str | None,
|
||||
wiki_config_apply_error,
|
||||
)
|
||||
row.updated_at = _now()
|
||||
await session.commit()
|
||||
return _to_dict(row)
|
||||
|
||||
+8
-6
@@ -2,16 +2,17 @@
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.model import (
|
||||
LlmWikiWikiEdgeRow,
|
||||
LlmWikiWikiPageRow,
|
||||
LlmWikiWikiSyncStateRow,
|
||||
LlmWikiWikiVectorRow,
|
||||
)
|
||||
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
|
||||
|
||||
|
||||
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
|
||||
return (
|
||||
MemoryLlmWikiIndexStore()
|
||||
if session_factory is None
|
||||
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
|
||||
)
|
||||
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
|
||||
|
||||
|
||||
__all__ = [
|
||||
@@ -19,6 +20,7 @@ __all__ = [
|
||||
"MemoryLlmWikiIndexStore",
|
||||
"SqlLlmWikiIndexStore",
|
||||
"LlmWikiWikiPageRow",
|
||||
"LlmWikiWikiEdgeRow",
|
||||
"LlmWikiWikiVectorRow",
|
||||
"LlmWikiWikiSyncStateRow",
|
||||
"make_llmwiki_index_store",
|
||||
|
||||
+130
-5
@@ -14,8 +14,102 @@ class LlmWikiIndexStore(ABC):
|
||||
@abstractmethod
|
||||
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
|
||||
|
||||
async def search_pages_by_keywords(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
tokens: list[str],
|
||||
*,
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return local Wiki body pages containing any token.
|
||||
|
||||
The default keeps lightweight test doubles and third-party stores
|
||||
backward compatible. Database and in-memory stores override it.
|
||||
"""
|
||||
|
||||
return []
|
||||
|
||||
async def list_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
*,
|
||||
limit: int = 500,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return a bounded, newest-first Wiki page snapshot.
|
||||
|
||||
Smart-reasoning regex search applies its expression in Python so the
|
||||
behavior stays identical on SQLite, MySQL and GaussDB. Third-party
|
||||
stores may keep the empty default; the tool then uses the remote Wiki
|
||||
API fallback.
|
||||
"""
|
||||
|
||||
return []
|
||||
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
edges: list[dict[str, Any]],
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Replace one mapping's directed Wiki graph snapshot.
|
||||
|
||||
The default keeps lightweight third-party/test stores compatible. An
|
||||
implementation must resolve both endpoints inside ``mapping_id`` and
|
||||
discard dangling or cross-mapping edges. ``None`` means the caller no
|
||||
longer owns the supplied sync lease and no mutation was made.
|
||||
"""
|
||||
|
||||
return {"edge_count": 0, "changed": False}
|
||||
|
||||
async def get_adjacent_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
seed_slugs: dict[str, list[str]] | list[str],
|
||||
*,
|
||||
direction: str = "both",
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return one-hop neighbors within the explicitly allowed mappings.
|
||||
|
||||
Passing a ``{mapping_id: [slug, ...]}`` mapping is preferred because it
|
||||
retains the exact seed provenance. A flat slug list is applied only to
|
||||
the supplied ``mapping_ids`` and never broadens authorization.
|
||||
"""
|
||||
|
||||
return []
|
||||
|
||||
@abstractmethod
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None: ...
|
||||
async def touch_page(
|
||||
self,
|
||||
page_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
mapping_id: str | None = None,
|
||||
owner: str | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def refresh_page_metadata(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
owner: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Refresh a mirrored page without replacing its vectors.
|
||||
|
||||
The write is fenced by the mapping sync lease. ``None`` means the
|
||||
caller no longer owns that lease (or the expected page disappeared),
|
||||
while the returned ``changed`` flag reports whether durable page
|
||||
metadata changed. The page is always marked as seen by ``sync_id``.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def replace_page_vectors(
|
||||
@@ -25,13 +119,34 @@ class LlmWikiIndexStore(ABC):
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any]: ...
|
||||
owner: str | None = None,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
|
||||
async def record_page_failure(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
error: str,
|
||||
owner: str | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
|
||||
async def mark_missing_pages_deleted(
|
||||
self,
|
||||
mapping_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
) -> int | None:
|
||||
"""Retire pages absent from a fenced full scan.
|
||||
|
||||
``None`` means the supplied owner/sync pair no longer owns the mapping
|
||||
lease and no mutation was made.
|
||||
"""
|
||||
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
|
||||
@@ -73,7 +188,17 @@ class LlmWikiIndexStore(ABC):
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
|
||||
async def finish_sync(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
success: bool,
|
||||
remote_page_count: int,
|
||||
error: str | None = None,
|
||||
result: dict[str, Any] | None = None,
|
||||
) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
|
||||
|
||||
+297
-18
@@ -17,11 +17,28 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
def __init__(self) -> None:
|
||||
self.pages: dict[str, dict[str, Any]] = {}
|
||||
self.vectors: dict[str, list[dict[str, Any]]] = {}
|
||||
self.edges: dict[str, list[dict[str, Any]]] = {}
|
||||
self.states: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def _state(self, mapping_id: str) -> dict[str, Any]:
|
||||
return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
|
||||
|
||||
def _has_current_sync_lease(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> bool:
|
||||
state = self.states.get(mapping_id)
|
||||
return bool(
|
||||
state is not None
|
||||
and state.get("lease_owner") == owner
|
||||
and state.get("active_sync_id") == sync_id
|
||||
and isinstance(state.get("lease_expires_at"), datetime)
|
||||
and state["lease_expires_at"] >= _now()
|
||||
)
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
|
||||
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
|
||||
@@ -30,12 +47,241 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
if page_id in self.pages:
|
||||
self.pages[page_id]["last_seen_sync_id"] = sync_id
|
||||
self.pages[page_id]["last_error"] = None
|
||||
async def search_pages_by_keywords(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
tokens: list[str],
|
||||
*,
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
wanted = set(mapping_ids)
|
||||
needles = [str(item).strip().lower() for item in tokens if str(item).strip()]
|
||||
rows = []
|
||||
for page in self.pages.values():
|
||||
if page.get("knowledge_base_mapping_id") not in wanted or page.get("is_remote_deleted"):
|
||||
continue
|
||||
if page.get("status") == "draft" and not include_drafts:
|
||||
continue
|
||||
if page.get("status") not in {"published", "draft"}:
|
||||
continue
|
||||
haystack = " ".join(str(page.get(key) or "") for key in ("title", "summary", "content_md")).lower()
|
||||
if needles and any(needle in haystack for needle in needles):
|
||||
rows.append(dict(page))
|
||||
if len(rows) >= max(1, limit):
|
||||
break
|
||||
return rows
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
async def list_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
*,
|
||||
limit: int = 500,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
wanted = {str(item).strip() for item in mapping_ids if str(item).strip()}
|
||||
cap = max(1, min(int(limit), 2000))
|
||||
rows = [
|
||||
dict(page)
|
||||
for page in self.pages.values()
|
||||
if page.get("knowledge_base_mapping_id") in wanted
|
||||
and not page.get("is_remote_deleted")
|
||||
and page.get("status") in ({"published", "draft"} if include_drafts else {"published"})
|
||||
]
|
||||
rows.sort(
|
||||
key=lambda page: str(page.get("remote_updated_at") or page.get("updated_at") or ""),
|
||||
reverse=True,
|
||||
)
|
||||
return rows[:cap]
|
||||
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
edges: list[dict[str, Any]],
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return None
|
||||
pages_by_slug = {str(page.get("slug") or ""): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") == mapping_id and not page.get("is_remote_deleted") and page.get("status") in {"published", "draft"}}
|
||||
canonical = {
|
||||
(
|
||||
str(edge.get("source_slug") or "").strip(),
|
||||
str(edge.get("target_slug") or "").strip(),
|
||||
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
|
||||
)
|
||||
for edge in edges
|
||||
if isinstance(edge, dict)
|
||||
}
|
||||
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
|
||||
replacement = [
|
||||
{
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"source_page_id": pages_by_slug[source]["id"],
|
||||
"target_page_id": pages_by_slug[target]["id"],
|
||||
"source_slug": source,
|
||||
"target_slug": target,
|
||||
"edge_type": edge_type,
|
||||
}
|
||||
for source, target, edge_type in sorted(canonical)
|
||||
]
|
||||
current = self.edges.get(mapping_id, [])
|
||||
current_keys = {(row["source_slug"], row["target_slug"], row["edge_type"]) for row in current}
|
||||
changed = current_keys != canonical
|
||||
if changed:
|
||||
self.edges[mapping_id] = replacement
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return {"edge_count": len(replacement), "changed": changed}
|
||||
|
||||
async def get_adjacent_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
seed_slugs: dict[str, list[str]] | list[str],
|
||||
*,
|
||||
direction: str = "both",
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
allowed = set(str(item).strip() for item in mapping_ids if str(item).strip())
|
||||
if direction not in {"incoming", "outgoing", "both"}:
|
||||
raise ValueError("direction must be incoming, outgoing, or both")
|
||||
if isinstance(seed_slugs, dict):
|
||||
seeds = {mapping_id: {str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip()} for mapping_id in allowed}
|
||||
else:
|
||||
shared = {str(slug).strip() for slug in seed_slugs if str(slug).strip()}
|
||||
seeds = {mapping_id: shared for mapping_id in allowed}
|
||||
pages_by_id = {str(page["id"]): page for page in self.pages.values() if page.get("knowledge_base_mapping_id") in allowed}
|
||||
result: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str, str, str]] = set()
|
||||
for mapping_id in sorted(allowed):
|
||||
mapping_seeds = seeds.get(mapping_id) or set()
|
||||
for edge in self.edges.get(mapping_id, []):
|
||||
traversals: list[tuple[str, str, str, str]] = []
|
||||
if direction in {"outgoing", "both"} and edge["source_slug"] in mapping_seeds:
|
||||
traversals.append(
|
||||
(
|
||||
edge["source_slug"],
|
||||
edge["target_slug"],
|
||||
"outgoing",
|
||||
edge["target_page_id"],
|
||||
)
|
||||
)
|
||||
if direction in {"incoming", "both"} and edge["target_slug"] in mapping_seeds:
|
||||
traversals.append(
|
||||
(
|
||||
edge["target_slug"],
|
||||
edge["source_slug"],
|
||||
"incoming",
|
||||
edge["source_page_id"],
|
||||
)
|
||||
)
|
||||
for seed_slug, neighbor_slug, traversal_direction, page_id in traversals:
|
||||
page = pages_by_id.get(page_id)
|
||||
if not page or page.get("is_remote_deleted"):
|
||||
continue
|
||||
if page.get("status") == "draft" and not include_drafts:
|
||||
continue
|
||||
if page.get("status") not in {"published", "draft"}:
|
||||
continue
|
||||
key = (mapping_id, seed_slug, neighbor_slug, traversal_direction)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append(
|
||||
{
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"seed_slug": seed_slug,
|
||||
"neighbor_slug": neighbor_slug,
|
||||
"direction": traversal_direction,
|
||||
"edge_type": edge["edge_type"],
|
||||
"source_slug": edge["source_slug"],
|
||||
"target_slug": edge["target_slug"],
|
||||
"page": dict(page),
|
||||
}
|
||||
)
|
||||
result.sort(
|
||||
key=lambda row: (
|
||||
row["knowledge_base_mapping_id"],
|
||||
row["seed_slug"],
|
||||
row["neighbor_slug"],
|
||||
row["direction"],
|
||||
)
|
||||
)
|
||||
return result[: max(1, min(int(limit), 1000))]
|
||||
|
||||
async def touch_page(
|
||||
self,
|
||||
page_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
mapping_id: str | None = None,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
page = self.pages.get(page_id)
|
||||
if page is None:
|
||||
return False
|
||||
actual_mapping_id = str(page.get("knowledge_base_mapping_id") or "")
|
||||
if mapping_id is not None and mapping_id != actual_mapping_id:
|
||||
return False
|
||||
if owner is not None and not self._has_current_sync_lease(
|
||||
actual_mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
page["last_seen_sync_id"] = sync_id
|
||||
page["last_error"] = None
|
||||
return True
|
||||
|
||||
async def refresh_page_metadata(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
owner: str,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
if not self._has_current_sync_lease(
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
existing = await self.get_page_by_slug_hash(mapping_id, str(page["slug_hash"]))
|
||||
if existing is None:
|
||||
return None
|
||||
durable_changed = any(existing.get(key) != value for key, value in page.items())
|
||||
row = self.pages[str(existing["id"])]
|
||||
row.update(page)
|
||||
row.update(
|
||||
{
|
||||
"last_seen_sync_id": sync_id,
|
||||
"last_error": None,
|
||||
"is_remote_deleted": False,
|
||||
"updated_at": _now().isoformat(),
|
||||
}
|
||||
)
|
||||
if durable_changed:
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return {"changed": durable_changed, "page": dict(row)}
|
||||
|
||||
async def replace_page_vectors(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
owner: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
if owner is not None and not self._has_current_sync_lease(
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
page_id = existing["id"] if existing else str(uuid4())
|
||||
now = _now().isoformat()
|
||||
@@ -59,7 +305,21 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
state["embedding_fingerprint"] = fingerprint
|
||||
return dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
async def record_page_failure(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
error: str,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
if owner is not None and not self._has_current_sync_lease(
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
|
||||
if existing:
|
||||
self.pages[existing["id"]].update(
|
||||
@@ -70,11 +330,20 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
"index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
|
||||
}
|
||||
)
|
||||
return
|
||||
return True
|
||||
page_id = str(uuid4())
|
||||
self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
|
||||
return True
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
async def mark_missing_pages_deleted(
|
||||
self,
|
||||
mapping_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
) -> int | None:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return None
|
||||
changed = 0
|
||||
for page in self.pages.values():
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
|
||||
@@ -82,6 +351,8 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
self.vectors.pop(page["id"], None)
|
||||
changed += 1
|
||||
if changed:
|
||||
active_page_ids = {page["id"] for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")}
|
||||
self.edges[mapping_id] = [edge for edge in self.edges.get(mapping_id, []) if edge["source_page_id"] in active_page_ids and edge["target_page_id"] in active_page_ids]
|
||||
self._state(mapping_id)["index_revision"] += 1
|
||||
return changed
|
||||
|
||||
@@ -131,17 +402,27 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
sync_id: str,
|
||||
remote_page_count: int,
|
||||
) -> bool:
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return False
|
||||
state = self._state(mapping_id)
|
||||
state["remote_page_count"] = max(0, remote_page_count)
|
||||
state["updated_at"] = _now()
|
||||
return True
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
async def finish_sync(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
success: bool,
|
||||
remote_page_count: int,
|
||||
error: str | None = None,
|
||||
result: dict[str, Any] | None = None,
|
||||
) -> bool:
|
||||
if not self._has_current_sync_lease(mapping_id, owner=owner, sync_id=sync_id):
|
||||
return False
|
||||
state = self._state(mapping_id)
|
||||
if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
|
||||
return
|
||||
pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
|
||||
was_rebuilding = state.get("state") == "rebuilding"
|
||||
state.update(
|
||||
@@ -158,8 +439,10 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
"vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
|
||||
"last_completed_at": _now() if success and not error else state.get("last_completed_at"),
|
||||
"last_success_at": _now() if success else state.get("last_success_at"),
|
||||
"last_sync_result": dict(result) if result is not None else state.get("last_sync_result"),
|
||||
}
|
||||
)
|
||||
return True
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
wanted = set(mapping_ids or [])
|
||||
@@ -168,11 +451,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
async def list_page_index_status(self, mapping_id: str) -> list[dict[str, Any]]:
|
||||
result = []
|
||||
for page in sorted(
|
||||
(
|
||||
page
|
||||
for page in self.pages.values()
|
||||
if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")
|
||||
),
|
||||
(page for page in self.pages.values() if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted")),
|
||||
key=lambda page: (str(page.get("wiki_path") or ""), str(page.get("slug") or "")),
|
||||
):
|
||||
result.append(
|
||||
|
||||
+48
@@ -72,6 +72,53 @@ class LlmWikiWikiVectorRow(Base):
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
|
||||
|
||||
class LlmWikiWikiEdgeRow(Base):
|
||||
"""One directed, mapping-scoped link between two mirrored Wiki pages."""
|
||||
|
||||
__tablename__ = "llmwiki_wiki_edges"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"knowledge_base_mapping_id",
|
||||
"source_page_id",
|
||||
"target_page_id",
|
||||
"edge_type",
|
||||
name="uq_llmwiki_wiki_edge_mapping_pages_type",
|
||||
),
|
||||
Index(
|
||||
"ix_llmwiki_wiki_edge_mapping_source",
|
||||
"knowledge_base_mapping_id",
|
||||
"source_page_id",
|
||||
),
|
||||
Index(
|
||||
"ix_llmwiki_wiki_edge_mapping_target",
|
||||
"knowledge_base_mapping_id",
|
||||
"target_page_id",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(36), primary_key=True)
|
||||
knowledge_base_mapping_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
source_page_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
target_page_id: Mapped[str] = mapped_column(
|
||||
String(36),
|
||||
ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
edge_type: Mapped[str] = mapped_column(String(32), nullable=False, default="wiki_link")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
|
||||
|
||||
|
||||
class LlmWikiWikiSyncStateRow(Base):
|
||||
__tablename__ = "llmwiki_wiki_sync_states"
|
||||
|
||||
@@ -91,4 +138,5 @@ class LlmWikiWikiSyncStateRow(Base):
|
||||
failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
last_sync_result: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
|
||||
+425
-10
@@ -7,12 +7,18 @@ from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import case, delete, func, insert, or_, select, update
|
||||
from sqlalchemy import and_, case, delete, func, insert, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.model import (
|
||||
LlmWikiWikiEdgeRow,
|
||||
LlmWikiWikiPageRow,
|
||||
LlmWikiWikiSyncStateRow,
|
||||
LlmWikiWikiVectorRow,
|
||||
)
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
@@ -40,6 +46,32 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
await session.flush()
|
||||
return row
|
||||
|
||||
async def _fence_current_sync_lease(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> bool:
|
||||
"""Fence destructive snapshot writes behind the current live lease."""
|
||||
|
||||
# Use a conditional UPDATE rather than SELECT ... FOR UPDATE. SQLite
|
||||
# ignores FOR UPDATE, while this statement obtains its write lock on
|
||||
# every supported backend before the page/edge mutation starts.
|
||||
stmt = (
|
||||
update(LlmWikiWikiSyncStateRow)
|
||||
.where(
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiSyncStateRow.lease_owner == owner,
|
||||
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
|
||||
)
|
||||
.values(updated_at=_now())
|
||||
)
|
||||
return bool((await session.execute(stmt)).rowcount)
|
||||
|
||||
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
|
||||
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
|
||||
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
|
||||
@@ -53,18 +85,351 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return _dict(row) if row else None
|
||||
|
||||
async def touch_page(self, page_id: str, sync_id: str) -> None:
|
||||
async def search_pages_by_keywords(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
tokens: list[str],
|
||||
*,
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
normalized_ids = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
|
||||
normalized_tokens = list(dict.fromkeys(str(item).strip().lower() for item in tokens if str(item).strip()))
|
||||
if not normalized_ids or not normalized_tokens:
|
||||
return []
|
||||
conditions = []
|
||||
for token in normalized_tokens[:32]:
|
||||
needle = f"%{token}%"
|
||||
conditions.extend(
|
||||
(
|
||||
func.lower(LlmWikiWikiPageRow.title).like(needle),
|
||||
func.lower(LlmWikiWikiPageRow.summary).like(needle),
|
||||
func.lower(LlmWikiWikiPageRow.content_md).like(needle),
|
||||
)
|
||||
)
|
||||
statuses = ["published", "draft"] if include_drafts else ["published"]
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(statuses),
|
||||
func.length(func.trim(LlmWikiWikiPageRow.content_md)) > 0,
|
||||
or_(*conditions),
|
||||
)
|
||||
.order_by(LlmWikiWikiPageRow.updated_at.desc())
|
||||
.limit(max(1, min(int(limit), 2000)))
|
||||
)
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def list_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
*,
|
||||
limit: int = 500,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
normalized_ids = list(
|
||||
dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip())
|
||||
)
|
||||
if not normalized_ids:
|
||||
return []
|
||||
statuses = ["published", "draft"] if include_drafts else ["published"]
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(normalized_ids),
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(statuses),
|
||||
)
|
||||
.order_by(
|
||||
LlmWikiWikiPageRow.remote_updated_at.desc(),
|
||||
LlmWikiWikiPageRow.updated_at.desc(),
|
||||
)
|
||||
.limit(max(1, min(int(limit), 2000)))
|
||||
)
|
||||
async with self._sf() as session:
|
||||
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
|
||||
|
||||
async def replace_graph_edges(
|
||||
self,
|
||||
mapping_id: str,
|
||||
edges: list[dict[str, Any]],
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomically replace an authorized mapping's resolved edge snapshot."""
|
||||
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
page_stmt = select(LlmWikiWikiPageRow).where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
|
||||
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
|
||||
)
|
||||
pages = list((await session.execute(page_stmt)).scalars())
|
||||
pages_by_slug = {page.slug: page for page in pages}
|
||||
canonical = {
|
||||
(
|
||||
str(edge.get("source_slug") or "").strip(),
|
||||
str(edge.get("target_slug") or "").strip(),
|
||||
str(edge.get("edge_type") or "wiki_link").strip()[:32] or "wiki_link",
|
||||
)
|
||||
for edge in edges
|
||||
if isinstance(edge, dict)
|
||||
}
|
||||
canonical = {edge for edge in canonical if edge[0] in pages_by_slug and edge[1] in pages_by_slug and edge[0] != edge[1]}
|
||||
existing_stmt = select(
|
||||
LlmWikiWikiEdgeRow.source_page_id,
|
||||
LlmWikiWikiEdgeRow.target_page_id,
|
||||
LlmWikiWikiEdgeRow.edge_type,
|
||||
).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id)
|
||||
existing = {(str(source_id), str(target_id), str(edge_type)) for source_id, target_id, edge_type in (await session.execute(existing_stmt)).all()}
|
||||
replacement_ids = {
|
||||
(
|
||||
pages_by_slug[source].id,
|
||||
pages_by_slug[target].id,
|
||||
edge_type,
|
||||
)
|
||||
for source, target, edge_type in canonical
|
||||
}
|
||||
changed = existing != replacement_ids
|
||||
if changed:
|
||||
await session.execute(delete(LlmWikiWikiEdgeRow).where(LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id))
|
||||
now = _now()
|
||||
edge_rows = [
|
||||
{
|
||||
"id": str(uuid4()),
|
||||
"knowledge_base_mapping_id": mapping_id,
|
||||
"source_page_id": pages_by_slug[source].id,
|
||||
"target_page_id": pages_by_slug[target].id,
|
||||
"edge_type": edge_type,
|
||||
"created_at": now,
|
||||
}
|
||||
for source, target, edge_type in sorted(canonical)
|
||||
]
|
||||
for start in range(0, len(edge_rows), self.write_batch_size):
|
||||
await session.execute(
|
||||
insert(LlmWikiWikiEdgeRow),
|
||||
edge_rows[start : start + self.write_batch_size],
|
||||
)
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return {"edge_count": len(canonical), "changed": changed}
|
||||
|
||||
async def get_adjacent_pages(
|
||||
self,
|
||||
mapping_ids: list[str],
|
||||
seed_slugs: dict[str, list[str]] | list[str],
|
||||
*,
|
||||
direction: str = "both",
|
||||
limit: int = 100,
|
||||
include_drafts: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
allowed = list(dict.fromkeys(str(item).strip() for item in mapping_ids if str(item).strip()))
|
||||
if not allowed:
|
||||
return []
|
||||
if direction not in {"incoming", "outgoing", "both"}:
|
||||
raise ValueError("direction must be incoming, outgoing, or both")
|
||||
if isinstance(seed_slugs, dict):
|
||||
seeds = {mapping_id: list(dict.fromkeys(str(slug).strip() for slug in seed_slugs.get(mapping_id, []) if str(slug).strip())) for mapping_id in allowed}
|
||||
else:
|
||||
shared = list(dict.fromkeys(str(slug).strip() for slug in seed_slugs if str(slug).strip()))
|
||||
seeds = {mapping_id: shared for mapping_id in allowed}
|
||||
seeds = {mapping_id: slugs for mapping_id, slugs in seeds.items() if slugs}
|
||||
if not seeds:
|
||||
return []
|
||||
|
||||
source_page = aliased(LlmWikiWikiPageRow)
|
||||
target_page = aliased(LlmWikiWikiPageRow)
|
||||
statuses = ["published", "draft"] if include_drafts else ["published"]
|
||||
cap = max(1, min(int(limit), 1000))
|
||||
|
||||
async def _query(traversal_direction: str) -> list[dict[str, Any]]:
|
||||
seed_page = source_page if traversal_direction == "outgoing" else target_page
|
||||
neighbor_page = target_page if traversal_direction == "outgoing" else source_page
|
||||
seed_conditions = or_(
|
||||
*(
|
||||
and_(
|
||||
LlmWikiWikiEdgeRow.knowledge_base_mapping_id == mapping_id,
|
||||
seed_page.slug.in_(slugs),
|
||||
)
|
||||
for mapping_id, slugs in seeds.items()
|
||||
)
|
||||
)
|
||||
stmt = (
|
||||
select(LlmWikiWikiEdgeRow, source_page, target_page)
|
||||
.join(source_page, source_page.id == LlmWikiWikiEdgeRow.source_page_id)
|
||||
.join(target_page, target_page.id == LlmWikiWikiEdgeRow.target_page_id)
|
||||
.where(
|
||||
LlmWikiWikiEdgeRow.knowledge_base_mapping_id.in_(allowed),
|
||||
source_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
|
||||
target_page.knowledge_base_mapping_id == LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
|
||||
source_page.is_remote_deleted.is_(False),
|
||||
target_page.is_remote_deleted.is_(False),
|
||||
source_page.status.in_(statuses),
|
||||
target_page.status.in_(statuses),
|
||||
seed_conditions,
|
||||
)
|
||||
.order_by(
|
||||
LlmWikiWikiEdgeRow.knowledge_base_mapping_id,
|
||||
seed_page.slug,
|
||||
neighbor_page.slug,
|
||||
)
|
||||
.limit(cap)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
triples = list((await session.execute(stmt)).all())
|
||||
rows: list[dict[str, Any]] = []
|
||||
for edge, source, target in triples:
|
||||
seed = source if traversal_direction == "outgoing" else target
|
||||
neighbor = target if traversal_direction == "outgoing" else source
|
||||
rows.append(
|
||||
{
|
||||
"knowledge_base_mapping_id": edge.knowledge_base_mapping_id,
|
||||
"seed_slug": seed.slug,
|
||||
"neighbor_slug": neighbor.slug,
|
||||
"direction": traversal_direction,
|
||||
"edge_type": edge.edge_type,
|
||||
"source_slug": source.slug,
|
||||
"target_slug": target.slug,
|
||||
"page": _dict(neighbor),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
result: list[dict[str, Any]] = []
|
||||
if direction in {"outgoing", "both"}:
|
||||
result.extend(await _query("outgoing"))
|
||||
if direction in {"incoming", "both"}:
|
||||
result.extend(await _query("incoming"))
|
||||
result.sort(
|
||||
key=lambda row: (
|
||||
row["knowledge_base_mapping_id"],
|
||||
row["seed_slug"],
|
||||
row["neighbor_slug"],
|
||||
row["direction"],
|
||||
)
|
||||
)
|
||||
deduplicated: list[dict[str, Any]] = []
|
||||
seen: set[tuple[str, str, str, str]] = set()
|
||||
for row in result:
|
||||
key = (
|
||||
row["knowledge_base_mapping_id"],
|
||||
row["seed_slug"],
|
||||
row["neighbor_slug"],
|
||||
row["direction"],
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduplicated.append(row)
|
||||
if len(deduplicated) >= cap:
|
||||
break
|
||||
return deduplicated
|
||||
|
||||
async def touch_page(
|
||||
self,
|
||||
page_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
mapping_id: str | None = None,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
async with self._sf() as session:
|
||||
if owner is not None:
|
||||
if not mapping_id or not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
conditions = [LlmWikiWikiPageRow.id == page_id]
|
||||
if mapping_id is not None:
|
||||
conditions.append(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id)
|
||||
result = await session.execute(
|
||||
update(LlmWikiWikiPageRow)
|
||||
.where(LlmWikiWikiPageRow.id == page_id)
|
||||
.where(*conditions)
|
||||
.values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
|
||||
async def refresh_page_metadata(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
owner: str,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiPageRow.slug_hash == page["slug_hash"],
|
||||
)
|
||||
.with_for_update()
|
||||
)
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is None:
|
||||
return None
|
||||
if row.slug != page["slug"]:
|
||||
raise ValueError("Wiki slug hash collision detected")
|
||||
durable_changed = bool(row.is_remote_deleted or row.last_error) or any(
|
||||
getattr(row, key) != value for key, value in page.items()
|
||||
)
|
||||
for key, value in page.items():
|
||||
setattr(row, key, value)
|
||||
now = _now()
|
||||
row.last_seen_sync_id = sync_id
|
||||
row.last_error = None
|
||||
row.is_remote_deleted = False
|
||||
row.updated_at = now
|
||||
if durable_changed:
|
||||
state = await self._ensure_state(session, mapping_id)
|
||||
state.index_revision = int(state.index_revision or 0) + 1
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return {"changed": durable_changed, "page": _dict(row)}
|
||||
|
||||
async def replace_page_vectors(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
vectors: list[dict[str, Any]],
|
||||
fingerprint: str,
|
||||
sync_id: str,
|
||||
owner: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
async with self._sf() as session:
|
||||
if owner is not None and not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
if row is not None and row.slug != page["slug"]:
|
||||
@@ -115,10 +480,24 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
await session.commit()
|
||||
return _dict(row)
|
||||
|
||||
async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
|
||||
async def record_page_failure(
|
||||
self,
|
||||
*,
|
||||
page: dict[str, Any],
|
||||
sync_id: str,
|
||||
error: str,
|
||||
owner: str | None = None,
|
||||
) -> bool:
|
||||
mapping_id = str(page["knowledge_base_mapping_id"])
|
||||
safe_error = error[:4000]
|
||||
async with self._sf() as session:
|
||||
if owner is not None and not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
now = _now()
|
||||
@@ -136,9 +515,23 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
row.updated_at = now
|
||||
await self._ensure_state(session, mapping_id)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
|
||||
async def mark_missing_pages_deleted(
|
||||
self,
|
||||
mapping_id: str,
|
||||
sync_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
) -> int | None:
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return None
|
||||
stmt = (
|
||||
select(LlmWikiWikiPageRow)
|
||||
.where(
|
||||
@@ -287,17 +680,36 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
|
||||
LlmWikiWikiSyncStateRow.lease_owner == owner,
|
||||
LlmWikiWikiSyncStateRow.active_sync_id == sync_id,
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at.is_not(None),
|
||||
LlmWikiWikiSyncStateRow.lease_expires_at >= _now(),
|
||||
)
|
||||
.values(remote_page_count=max(0, remote_page_count), updated_at=_now())
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
|
||||
async def finish_sync(
|
||||
self,
|
||||
mapping_id: str,
|
||||
*,
|
||||
owner: str,
|
||||
sync_id: str,
|
||||
success: bool,
|
||||
remote_page_count: int,
|
||||
error: str | None = None,
|
||||
result: dict[str, Any] | None = None,
|
||||
) -> bool:
|
||||
async with self._sf() as session:
|
||||
if not await self._fence_current_sync_lease(
|
||||
session,
|
||||
mapping_id,
|
||||
owner=owner,
|
||||
sync_id=sync_id,
|
||||
):
|
||||
return False
|
||||
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||
if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
|
||||
return
|
||||
return False
|
||||
visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
|
||||
local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
|
||||
ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
|
||||
@@ -329,12 +741,15 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||
state.ready_page_count = ready_count
|
||||
state.failed_page_count = failed_count
|
||||
state.vector_count = vector_count
|
||||
if result is not None:
|
||||
state.last_sync_result = dict(result)
|
||||
if success and not error:
|
||||
state.last_completed_at = now
|
||||
if success:
|
||||
state.last_success_at = now
|
||||
state.updated_at = now
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
|
||||
stmt = select(LlmWikiWikiSyncStateRow)
|
||||
|
||||
+2
-1
@@ -4,13 +4,14 @@ from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.menu_overrides.base import MenuOverrideStore
|
||||
from deerflow.persistence.menu_overrides.memory import MemoryMenuOverrideStore
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
|
||||
from deerflow.persistence.menu_overrides.sql import MenuOverrideRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryMenuOverrideStore",
|
||||
"MenuOverrideRepository",
|
||||
"MenuOverrideRow",
|
||||
"UserMenuOverrideConfigRow",
|
||||
"MenuOverrideStore",
|
||||
"make_menu_override_store",
|
||||
]
|
||||
|
||||
+27
-6
@@ -1,11 +1,11 @@
|
||||
"""Abstract sidebar-menu override persistence API.
|
||||
|
||||
Global / shared store: no ``user_id`` scoping. Reads are open to any caller; the
|
||||
router gates writes behind admin. ``updated_by`` is audit-only.
|
||||
|
||||
The whole override set is treated as one document: the editor computes the full
|
||||
desired layout and replaces every row atomically via :meth:`replace_all`, so the
|
||||
stored set never drifts out of sync with what the admin sees.
|
||||
The original ``menu_overrides`` document remains the administrator's default
|
||||
layout. User layouts are stored as a separate, complete snapshot so that a
|
||||
user can customise the menu without changing the default for anybody else.
|
||||
The router overlays a user's snapshot on top of the default and lets default
|
||||
rows for node ids not present in that snapshot flow through (new administrator
|
||||
menus therefore appear for existing users).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -28,3 +28,24 @@ class MenuOverrideStore(ABC):
|
||||
Returns the persisted rows.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
|
||||
"""Return one user's snapshot, or ``None`` when no snapshot exists."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
"""Replace one user's snapshot.
|
||||
|
||||
An empty list removes the snapshot and returns ``None``. This is the
|
||||
reset-to-default operation and intentionally differs from the admin
|
||||
``replace_all`` document, where an empty list means an empty default.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
+27
@@ -23,6 +23,7 @@ def _normalize(data: dict[str, Any], updated_by: str | None) -> dict[str, Any]:
|
||||
class MemoryMenuOverrideStore(MenuOverrideStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._user_rows: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
async def list_overrides(self) -> list[dict[str, Any]]:
|
||||
return [dict(r) for r in sorted(self._rows.values(), key=lambda r: r.get("sort_order", 0))]
|
||||
@@ -35,3 +36,29 @@ class MemoryMenuOverrideStore(MenuOverrideStore):
|
||||
continue
|
||||
self._rows[str(node_id)] = _normalize(r, updated_by)
|
||||
return [dict(r) for r in self._rows.values()]
|
||||
|
||||
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
|
||||
rows = self._user_rows.get(str(user_id))
|
||||
return [dict(r) for r in rows] if rows is not None else None
|
||||
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
key = str(user_id)
|
||||
if not rows:
|
||||
self._user_rows.pop(key, None)
|
||||
return None
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
node_id = row.get("node_id")
|
||||
if node_id:
|
||||
by_id[str(node_id)] = _normalize(row, updated_by or key)
|
||||
if not by_id:
|
||||
self._user_rows.pop(key, None)
|
||||
return None
|
||||
self._user_rows[key] = [dict(row) for row in by_id.values()]
|
||||
return [dict(row) for row in self._user_rows[key]]
|
||||
+25
-5
@@ -1,9 +1,10 @@
|
||||
"""ORM model for sidebar-menu overrides (菜单管理).
|
||||
|
||||
Global / shared — there is no ``user_id`` scoping; every account sees the same
|
||||
menu layout. The static menu tree (icons / routes / adminOnly) lives in the
|
||||
frontend code; this table only stores per-node **overrides** keyed by the static
|
||||
node id (or an injected iframe light-app node id ``light-app-<appId>``):
|
||||
``menu_overrides`` stores the administrator's shared default. Personal layouts
|
||||
live separately in ``user_menu_override_configs`` as complete snapshots. The
|
||||
static menu tree (icons / routes / adminOnly) lives in the frontend code; the
|
||||
default table stores per-node **overrides** keyed by the static node id (or an
|
||||
injected iframe light-app node id ``light-app-<appId>``):
|
||||
|
||||
- ``parent_id`` — the effective parent node id (NULL = top level). Re-parenting
|
||||
a node moves its whole subtree.
|
||||
@@ -23,7 +24,7 @@ from sqlalchemy import Boolean, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON
|
||||
|
||||
|
||||
class MenuOverrideRow(Base):
|
||||
@@ -40,3 +41,22 @@ class MenuOverrideRow(Base):
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
|
||||
|
||||
class UserMenuOverrideConfigRow(Base):
|
||||
"""A complete personal sidebar-menu snapshot.
|
||||
|
||||
Keeping the snapshot in one row is deliberate: the presence of a row is
|
||||
the ``has_user_override`` bit, while the node ids in ``overrides`` are the
|
||||
user's known-node set. This lets us distinguish an administrator-added
|
||||
node from an old node the user intentionally left at its default value.
|
||||
"""
|
||||
|
||||
__tablename__ = "user_menu_override_configs"
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
overrides: Mapped[list[dict]] = mapped_column(PortableJSON(), nullable=False, default=list)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC)
|
||||
)
|
||||
+84
-2
@@ -2,14 +2,14 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.menu_overrides.base import MenuOverrideStore
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
|
||||
|
||||
|
||||
def _to_dict(row: MenuOverrideRow) -> dict[str, Any]:
|
||||
@@ -31,6 +31,42 @@ def _row_from(data: dict[str, Any], updated_by: str | None) -> MenuOverrideRow:
|
||||
)
|
||||
|
||||
|
||||
def _normalize_user_row(data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Keep only the public override fields in the personal JSON document."""
|
||||
return {
|
||||
"node_id": str(data["node_id"]),
|
||||
"parent_id": (str(data["parent_id"]) if data.get("parent_id") not in (None, "") else None),
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"label": (data.get("label") or None),
|
||||
"disabled": bool(data.get("disabled", False)),
|
||||
}
|
||||
|
||||
|
||||
def _user_rows(row: UserMenuOverrideConfigRow) -> list[dict[str, Any]]:
|
||||
value = row.overrides
|
||||
if isinstance(value, str):
|
||||
# PortableJSON handles MySQL automatically, but this keeps old rows
|
||||
# readable if a driver returned the raw TEXT value.
|
||||
import json
|
||||
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
value = []
|
||||
if not isinstance(value, list):
|
||||
value = []
|
||||
updated_at = row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at
|
||||
return [
|
||||
{
|
||||
**_normalize_user_row(item),
|
||||
"updated_by": row.updated_by,
|
||||
"updated_at": updated_at,
|
||||
}
|
||||
for item in value
|
||||
if isinstance(item, dict) and item.get("node_id")
|
||||
]
|
||||
|
||||
|
||||
class MenuOverrideRepository(MenuOverrideStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
@@ -56,3 +92,49 @@ class MenuOverrideRepository(MenuOverrideStore):
|
||||
for row in new_rows:
|
||||
await session.refresh(row)
|
||||
return [_to_dict(row) for row in new_rows]
|
||||
|
||||
async def list_user_overrides(self, user_id: str) -> list[dict[str, Any]] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserMenuOverrideConfigRow, str(user_id))
|
||||
return _user_rows(row) if row is not None else None
|
||||
|
||||
async def replace_user_overrides(
|
||||
self,
|
||||
user_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> list[dict[str, Any]] | None:
|
||||
key = str(user_id)
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for data in rows:
|
||||
node_id = data.get("node_id")
|
||||
if node_id:
|
||||
normalized = _normalize_user_row(data)
|
||||
by_id[normalized["node_id"]] = normalized
|
||||
async with self._sf() as session:
|
||||
row = await session.get(UserMenuOverrideConfigRow, key)
|
||||
if not by_id:
|
||||
if row is not None:
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
payload = list(by_id.values())
|
||||
if row is None:
|
||||
row = UserMenuOverrideConfigRow(
|
||||
user_id=key,
|
||||
overrides=payload,
|
||||
updated_by=updated_by or key,
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.overrides = payload
|
||||
row.updated_by = updated_by or key
|
||||
row.updated_at = now
|
||||
await session.commit()
|
||||
# The shared session factory uses ``expire_on_commit=False``. Build
|
||||
# the response from the writer-side instance instead of issuing a
|
||||
# post-commit refresh that a read/write-splitting endpoint could
|
||||
# route to a lagging replica.
|
||||
return _user_rows(row)
|
||||
+3
-1
@@ -54,7 +54,9 @@ def do_run_migrations(connection):
|
||||
# = 1 year; innodb_lock_wait_timeout = 50s). Without this, an idle
|
||||
# transaction on another connection can freeze startup indefinitely.
|
||||
# Tunable via DEER_FLOW_ALEMBIC_LOCK_WAIT (seconds, default 15).
|
||||
if connection.dialect.name == "mysql":
|
||||
from deerflow.persistence.gauss_mysql_compat import is_gauss_mysql_product
|
||||
|
||||
if connection.dialect.name == "mysql" and not is_gauss_mysql_product():
|
||||
try:
|
||||
lock_wait = int(os.getenv("DEER_FLOW_ALEMBIC_LOCK_WAIT", "15"))
|
||||
except ValueError:
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
"""Merge assistant-wiki metadata and report-collaboration heads.
|
||||
|
||||
Revision ID: 20260909_01
|
||||
Revises: 20260903_01, 20260907_01
|
||||
Create Date: 2026-09-09
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
revision = "20260909_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
"""Create durable public-document library tables.
|
||||
|
||||
Revision ID: 20260914_01
|
||||
Revises: 20260903_01, 20260907_01
|
||||
Create Date: 2026-09-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260914_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("gongwen_documents"):
|
||||
op.create_table(
|
||||
"gongwen_documents",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("user_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("title", sa.String(length=256), nullable=False, server_default=""),
|
||||
sa.Column("document_type", sa.String(length=48), nullable=False, server_default="notice"),
|
||||
sa.Column("source", sa.String(length=24), nullable=False, server_default="upload"),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="checking"),
|
||||
sa.Column("template_key", sa.String(length=96), nullable=True),
|
||||
sa.Column("research_session_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("original_version_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("latest_version_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("issue_counts", sa.Text(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_gongwen_documents_user_id", "gongwen_documents", ["user_id"])
|
||||
op.create_index("ix_gongwen_documents_document_type", "gongwen_documents", ["document_type"])
|
||||
op.create_index("ix_gongwen_documents_source", "gongwen_documents", ["source"])
|
||||
op.create_index("ix_gongwen_documents_status", "gongwen_documents", ["status"])
|
||||
op.create_index("ix_gongwen_documents_research_session_id", "gongwen_documents", ["research_session_id"])
|
||||
|
||||
if not inspector.has_table("gongwen_document_versions"):
|
||||
op.create_table(
|
||||
"gongwen_document_versions",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("document_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("version_no", sa.Integer(), nullable=False),
|
||||
sa.Column("kind", sa.String(length=32), nullable=False),
|
||||
sa.Column("file_name", sa.String(length=512), nullable=False),
|
||||
sa.Column("storage_key", sa.String(length=1024), nullable=False),
|
||||
sa.Column("content_type", sa.String(length=128), nullable=False, server_default="application/octet-stream"),
|
||||
sa.Column("size_bytes", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("sha256", sa.String(length=64), nullable=False, server_default=""),
|
||||
sa.Column("review_snapshot", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("document_id", "version_no", name="uq_gongwen_document_version_no"),
|
||||
)
|
||||
op.create_index("ix_gongwen_document_versions_document_id", "gongwen_document_versions", ["document_id"])
|
||||
op.create_index(
|
||||
"ix_gongwen_document_versions_document_created",
|
||||
"gongwen_document_versions",
|
||||
["document_id", "created_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if inspector.has_table("gongwen_document_versions"):
|
||||
op.drop_table("gongwen_document_versions")
|
||||
if inspector.has_table("gongwen_documents"):
|
||||
op.drop_table("gongwen_documents")
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
"""Persist mapping-scoped links for local LLMWiki graph expansion.
|
||||
|
||||
Revision ID: 20260916_01
|
||||
Revises: 20260903_01, 20260907_01
|
||||
Create Date: 2026-09-16
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260916_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if inspector.has_table("llmwiki_wiki_edges"):
|
||||
return
|
||||
op.create_table(
|
||||
"llmwiki_wiki_edges",
|
||||
sa.Column("id", sa.String(length=36), primary_key=True),
|
||||
sa.Column(
|
||||
"knowledge_base_mapping_id",
|
||||
sa.String(length=36),
|
||||
sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"source_page_id",
|
||||
sa.String(length=36),
|
||||
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"target_page_id",
|
||||
sa.String(length=36),
|
||||
sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"edge_type",
|
||||
sa.String(length=32),
|
||||
nullable=False,
|
||||
server_default="wiki_link",
|
||||
),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint(
|
||||
"knowledge_base_mapping_id",
|
||||
"source_page_id",
|
||||
"target_page_id",
|
||||
"edge_type",
|
||||
name="uq_llmwiki_wiki_edge_mapping_pages_type",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edges_knowledge_base_mapping_id",
|
||||
"llmwiki_wiki_edges",
|
||||
["knowledge_base_mapping_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edges_source_page_id",
|
||||
"llmwiki_wiki_edges",
|
||||
["source_page_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edges_target_page_id",
|
||||
"llmwiki_wiki_edges",
|
||||
["target_page_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edge_mapping_source",
|
||||
"llmwiki_wiki_edges",
|
||||
["knowledge_base_mapping_id", "source_page_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_llmwiki_wiki_edge_mapping_target",
|
||||
"llmwiki_wiki_edges",
|
||||
["knowledge_base_mapping_id", "target_page_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table("llmwiki_wiki_edges"):
|
||||
op.drop_table("llmwiki_wiki_edges")
|
||||
+240
@@ -0,0 +1,240 @@
|
||||
"""Add answer-level feedback reasons and agent score dimensions.
|
||||
|
||||
Revision ID: 20260916_02
|
||||
Revises: 20260916_01
|
||||
Create Date: 2026-09-16
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260916_02"
|
||||
down_revision = "20260916_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "feedback"
|
||||
_OLD_INDEX = "uq_feedback_thread_run_user"
|
||||
_TARGET_INDEX = "uq_feedback_target_key"
|
||||
_AGENT_INDEX = "ix_feedback_agent_created"
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
return {
|
||||
str(column["name"])
|
||||
for column in sa.inspect(op.get_bind()).get_columns(_TABLE)
|
||||
}
|
||||
|
||||
|
||||
def _indexes() -> set[str]:
|
||||
return {
|
||||
str(index["name"])
|
||||
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE)
|
||||
if index.get("name")
|
||||
}
|
||||
|
||||
|
||||
def _json_type() -> sa.types.TypeEngine:
|
||||
return sa.Text() if op.get_bind().dialect.name == "mysql" else sa.JSON()
|
||||
|
||||
|
||||
def _empty_reasons_value() -> str | list[str]:
|
||||
# PortableJSON is TEXT on MySQL, where binding a Python list directly to
|
||||
# a raw migration Text column is invalid. SQLite/PostgreSQL JSON columns
|
||||
# should receive the native list so their type processors serialize it.
|
||||
return "[]" if op.get_bind().dialect.name == "mysql" else []
|
||||
|
||||
|
||||
def _target_key(
|
||||
*,
|
||||
user_id: str | None,
|
||||
thread_id: str,
|
||||
run_id: str,
|
||||
message_id: str | None,
|
||||
agent_id: str,
|
||||
) -> str:
|
||||
message_key = message_id or f"run:{run_id}"
|
||||
raw = "\x1f".join(
|
||||
(user_id or "", thread_id, message_key, agent_id or "lead_agent")
|
||||
)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _backfill() -> None:
|
||||
bind = op.get_bind()
|
||||
feedback = sa.table(
|
||||
_TABLE,
|
||||
sa.column("feedback_id", sa.String(64)),
|
||||
sa.column("run_id", sa.String(64)),
|
||||
sa.column("thread_id", sa.String(64)),
|
||||
sa.column("user_id", sa.String(64)),
|
||||
sa.column("message_id", sa.String(64)),
|
||||
sa.column("agent_id", sa.String(128)),
|
||||
sa.column("target_key", sa.String(64)),
|
||||
sa.column("reasons", _json_type()),
|
||||
sa.column("created_at", sa.DateTime(timezone=True)),
|
||||
sa.column("updated_at", sa.DateTime(timezone=True)),
|
||||
)
|
||||
rows = list(
|
||||
bind.execute(
|
||||
sa.select(
|
||||
feedback.c.feedback_id,
|
||||
feedback.c.run_id,
|
||||
feedback.c.thread_id,
|
||||
feedback.c.user_id,
|
||||
feedback.c.message_id,
|
||||
feedback.c.agent_id,
|
||||
feedback.c.created_at,
|
||||
)
|
||||
).mappings()
|
||||
)
|
||||
|
||||
run_agents: dict[str, str] = {}
|
||||
inspector = sa.inspect(bind)
|
||||
if rows and inspector.has_table("runs"):
|
||||
run_columns = {str(column["name"]) for column in inspector.get_columns("runs")}
|
||||
if "assistant_id" in run_columns:
|
||||
runs = sa.table(
|
||||
"runs",
|
||||
sa.column("run_id", sa.String(64)),
|
||||
sa.column("assistant_id", sa.String(128)),
|
||||
)
|
||||
run_ids = sorted({str(row["run_id"]) for row in rows})
|
||||
run_agents = {
|
||||
str(row.run_id): str(row.assistant_id)
|
||||
for row in bind.execute(
|
||||
sa.select(runs.c.run_id, runs.c.assistant_id).where(
|
||||
runs.c.run_id.in_(run_ids)
|
||||
)
|
||||
)
|
||||
if row.assistant_id
|
||||
}
|
||||
|
||||
seen_target_keys: set[str] = set()
|
||||
for row in rows:
|
||||
agent_id = str(
|
||||
run_agents.get(str(row["run_id"]))
|
||||
or row.get("agent_id")
|
||||
or "lead_agent"
|
||||
)[:128]
|
||||
target_key = _target_key(
|
||||
user_id=row.get("user_id"),
|
||||
thread_id=str(row["thread_id"]),
|
||||
run_id=str(row["run_id"]),
|
||||
message_id=row.get("message_id"),
|
||||
agent_id=agent_id,
|
||||
)
|
||||
# SQL unique indexes historically allowed several NULL-user rows.
|
||||
# Keep those legacy records without blocking the new unique digest;
|
||||
# one canonical row retains the deterministic future-upsert key.
|
||||
if target_key in seen_target_keys:
|
||||
legacy_raw = f"{target_key}\x1flegacy:{row['feedback_id']}"
|
||||
target_key = hashlib.sha256(legacy_raw.encode("utf-8")).hexdigest()
|
||||
else:
|
||||
seen_target_keys.add(target_key)
|
||||
bind.execute(
|
||||
sa.update(feedback)
|
||||
.where(feedback.c.feedback_id == row["feedback_id"])
|
||||
.values(
|
||||
agent_id=agent_id,
|
||||
target_key=target_key,
|
||||
reasons=_empty_reasons_value(),
|
||||
updated_at=row.get("created_at"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TABLE):
|
||||
return
|
||||
|
||||
columns = _columns()
|
||||
if "agent_id" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(
|
||||
"agent_id",
|
||||
sa.String(length=128),
|
||||
nullable=True,
|
||||
server_default="lead_agent",
|
||||
),
|
||||
)
|
||||
if "target_key" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("target_key", sa.String(length=64), nullable=True),
|
||||
)
|
||||
if "reasons" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("reasons", _json_type(), nullable=True),
|
||||
)
|
||||
if "updated_at" not in columns:
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
|
||||
_backfill()
|
||||
|
||||
# SQLite needs a batch table rebuild for nullability changes; Alembic uses
|
||||
# native ALTER statements on MySQL/PostgreSQL where possible.
|
||||
with op.batch_alter_table(_TABLE) as batch:
|
||||
batch.alter_column(
|
||||
"agent_id",
|
||||
existing_type=sa.String(length=128),
|
||||
nullable=False,
|
||||
server_default=None,
|
||||
)
|
||||
batch.alter_column(
|
||||
"target_key",
|
||||
existing_type=sa.String(length=64),
|
||||
nullable=False,
|
||||
)
|
||||
batch.alter_column(
|
||||
"reasons",
|
||||
existing_type=_json_type(),
|
||||
nullable=False,
|
||||
)
|
||||
batch.alter_column(
|
||||
"updated_at",
|
||||
existing_type=sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
indexes = _indexes()
|
||||
if _OLD_INDEX in indexes:
|
||||
op.drop_index(_OLD_INDEX, table_name=_TABLE)
|
||||
indexes = _indexes()
|
||||
if _TARGET_INDEX not in indexes:
|
||||
op.create_index(_TARGET_INDEX, _TABLE, ["target_key"], unique=True)
|
||||
if _AGENT_INDEX not in indexes:
|
||||
op.create_index(_AGENT_INDEX, _TABLE, ["agent_id", "created_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
return
|
||||
indexes = _indexes()
|
||||
if _AGENT_INDEX in indexes:
|
||||
op.drop_index(_AGENT_INDEX, table_name=_TABLE)
|
||||
if _TARGET_INDEX in indexes:
|
||||
op.drop_index(_TARGET_INDEX, table_name=_TABLE)
|
||||
if _OLD_INDEX not in _indexes():
|
||||
op.create_index(
|
||||
_OLD_INDEX,
|
||||
_TABLE,
|
||||
["thread_id", "run_id", "user_id"],
|
||||
unique=True,
|
||||
mysql_length={"user_id": 63},
|
||||
)
|
||||
columns = _columns()
|
||||
with op.batch_alter_table(_TABLE) as batch:
|
||||
for name in ("updated_at", "reasons", "target_key", "agent_id"):
|
||||
if name in columns:
|
||||
batch.drop_column(name)
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
"""Merge feedback, public-document, and assistant-knowledge migration heads.
|
||||
|
||||
Revision ID: 20260916_03
|
||||
Revises: 20260916_02, 20260909_01, 20260914_01
|
||||
Create Date: 2026-09-16
|
||||
|
||||
This merge used to share revision id ``20260916_01`` with the LLMWiki graph
|
||||
edge migration. Giving the no-op merge its own id preserves the graph-edge
|
||||
revision for already stamped databases while restoring an unambiguous graph.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
revision = "20260916_03"
|
||||
down_revision = ("20260916_02", "20260909_01", "20260914_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Record the merge; all parent revisions already own their DDL."""
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Splitting the migration history requires no schema change."""
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""Add editable starter questions and retrieval settings to LLMWiki mappings.
|
||||
|
||||
Revision ID: 20260920_01
|
||||
Revises: 20260903_01, 20260907_01
|
||||
Create Date: 2026-09-20
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20260920_01"
|
||||
down_revision = ("20260903_01", "20260907_01")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table("llmwiki_knowledge_bases"):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspector.get_columns("llmwiki_knowledge_bases")}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
columns = _columns()
|
||||
if not columns:
|
||||
return
|
||||
if "test_questions" not in columns:
|
||||
op.add_column(
|
||||
"llmwiki_knowledge_bases",
|
||||
sa.Column("test_questions", PortableJSON(), nullable=True),
|
||||
)
|
||||
if "retrieval_settings" not in columns:
|
||||
op.add_column(
|
||||
"llmwiki_knowledge_bases",
|
||||
sa.Column("retrieval_settings", PortableJSON(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
columns = _columns()
|
||||
if "retrieval_settings" in columns:
|
||||
op.drop_column("llmwiki_knowledge_bases", "retrieval_settings")
|
||||
if "test_questions" in columns:
|
||||
op.drop_column("llmwiki_knowledge_bases", "test_questions")
|
||||
+194
@@ -0,0 +1,194 @@
|
||||
"""Reconcile LLMWiki schema after historical duplicate revision ids.
|
||||
|
||||
Revision ID: 20260920_02
|
||||
Revises: 20260920_01, 20260916_03
|
||||
Create Date: 2026-09-20
|
||||
|
||||
Older source snapshots used ``20260916_01`` and ``20260920_01`` twice. A
|
||||
database stamped with either value can therefore be missing the schema from
|
||||
the other migration that carried the same id. This merge is deliberately
|
||||
idempotent and restores every affected table, constraint, index, and JSON
|
||||
column before recording the new unambiguous head.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20260920_02"
|
||||
down_revision = ("20260920_01", "20260916_03")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_EDGE_TABLE = "llmwiki_wiki_edges"
|
||||
_EDGE_UNIQUE = "uq_llmwiki_wiki_edge_mapping_pages_type"
|
||||
_EDGE_COLUMNS = (
|
||||
"knowledge_base_mapping_id",
|
||||
"source_page_id",
|
||||
"target_page_id",
|
||||
"edge_type",
|
||||
)
|
||||
_EDGE_FOREIGN_KEYS = (
|
||||
(
|
||||
"fk_llmwiki_wiki_edges_mapping",
|
||||
"knowledge_base_mapping_id",
|
||||
"llmwiki_knowledge_bases",
|
||||
),
|
||||
(
|
||||
"fk_llmwiki_wiki_edges_source_page",
|
||||
"source_page_id",
|
||||
"llmwiki_wiki_pages",
|
||||
),
|
||||
(
|
||||
"fk_llmwiki_wiki_edges_target_page",
|
||||
"target_page_id",
|
||||
"llmwiki_wiki_pages",
|
||||
),
|
||||
)
|
||||
_EDGE_INDEXES = (
|
||||
(
|
||||
"ix_llmwiki_wiki_edges_knowledge_base_mapping_id",
|
||||
("knowledge_base_mapping_id",),
|
||||
),
|
||||
("ix_llmwiki_wiki_edges_source_page_id", ("source_page_id",)),
|
||||
("ix_llmwiki_wiki_edges_target_page_id", ("target_page_id",)),
|
||||
(
|
||||
"ix_llmwiki_wiki_edge_mapping_source",
|
||||
("knowledge_base_mapping_id", "source_page_id"),
|
||||
),
|
||||
(
|
||||
"ix_llmwiki_wiki_edge_mapping_target",
|
||||
("knowledge_base_mapping_id", "target_page_id"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _create_edges_table() -> None:
|
||||
op.create_table(
|
||||
_EDGE_TABLE,
|
||||
sa.Column("id", sa.String(length=36), primary_key=True),
|
||||
sa.Column("knowledge_base_mapping_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("source_page_id", sa.String(length=36), nullable=False),
|
||||
sa.Column("target_page_id", sa.String(length=36), nullable=False),
|
||||
sa.Column(
|
||||
"edge_type",
|
||||
sa.String(length=32),
|
||||
nullable=False,
|
||||
server_default="wiki_link",
|
||||
),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["knowledge_base_mapping_id"],
|
||||
["llmwiki_knowledge_bases.id"],
|
||||
name="fk_llmwiki_wiki_edges_mapping",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_page_id"],
|
||||
["llmwiki_wiki_pages.id"],
|
||||
name="fk_llmwiki_wiki_edges_source_page",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["target_page_id"],
|
||||
["llmwiki_wiki_pages.id"],
|
||||
name="fk_llmwiki_wiki_edges_target_page",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.UniqueConstraint(*_EDGE_COLUMNS, name=_EDGE_UNIQUE),
|
||||
)
|
||||
|
||||
|
||||
def _foreign_key_shape(item: dict) -> tuple[tuple[str, ...], str | None, tuple[str, ...]]:
|
||||
return (
|
||||
tuple(str(name) for name in item.get("constrained_columns") or ()),
|
||||
item.get("referred_table"),
|
||||
tuple(str(name) for name in item.get("referred_columns") or ()),
|
||||
)
|
||||
|
||||
|
||||
def _ensure_edge_constraints(inspector: sa.Inspector) -> None:
|
||||
unique_names = {
|
||||
str(item["name"])
|
||||
for item in inspector.get_unique_constraints(_EDGE_TABLE)
|
||||
if item.get("name")
|
||||
}
|
||||
foreign_key_shapes = {
|
||||
_foreign_key_shape(item) for item in inspector.get_foreign_keys(_EDGE_TABLE)
|
||||
}
|
||||
missing_foreign_keys = [
|
||||
(name, column, referred_table)
|
||||
for name, column, referred_table in _EDGE_FOREIGN_KEYS
|
||||
if ((column,), referred_table, ("id",)) not in foreign_key_shapes
|
||||
]
|
||||
if _EDGE_UNIQUE in unique_names and not missing_foreign_keys:
|
||||
return
|
||||
|
||||
with op.batch_alter_table(_EDGE_TABLE) as batch:
|
||||
if _EDGE_UNIQUE not in unique_names:
|
||||
batch.create_unique_constraint(_EDGE_UNIQUE, list(_EDGE_COLUMNS))
|
||||
for name, column, referred_table in missing_foreign_keys:
|
||||
batch.create_foreign_key(
|
||||
name,
|
||||
referred_table,
|
||||
[column],
|
||||
["id"],
|
||||
ondelete="CASCADE",
|
||||
)
|
||||
|
||||
|
||||
def _ensure_edge_indexes(inspector: sa.Inspector) -> None:
|
||||
index_names = {
|
||||
str(item["name"])
|
||||
for item in inspector.get_indexes(_EDGE_TABLE)
|
||||
if item.get("name")
|
||||
}
|
||||
for name, columns in _EDGE_INDEXES:
|
||||
if name not in index_names:
|
||||
op.create_index(name, _EDGE_TABLE, list(columns))
|
||||
|
||||
|
||||
def _ensure_edges() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table(_EDGE_TABLE):
|
||||
_create_edges_table()
|
||||
else:
|
||||
_ensure_edge_constraints(inspector)
|
||||
_ensure_edge_indexes(sa.inspect(bind))
|
||||
|
||||
|
||||
def _columns(table: str) -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(table):
|
||||
return set()
|
||||
return {str(item["name"]) for item in inspector.get_columns(table)}
|
||||
|
||||
|
||||
def _ensure_json_columns(table: str, columns: Iterable[str]) -> None:
|
||||
existing = _columns(table)
|
||||
if not existing:
|
||||
return
|
||||
for name in columns:
|
||||
if name not in existing:
|
||||
op.add_column(table, sa.Column(name, PortableJSON(), nullable=True))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
_ensure_edges()
|
||||
_ensure_json_columns(
|
||||
"llmwiki_knowledge_bases",
|
||||
("test_questions", "retrieval_settings"),
|
||||
)
|
||||
_ensure_json_columns("llmwiki_wiki_sync_states", ("last_sync_result",))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
columns = _columns("llmwiki_wiki_sync_states")
|
||||
if "last_sync_result" in columns:
|
||||
op.drop_column("llmwiki_wiki_sync_states", "last_sync_result")
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""Persist DeerFlow-owned fixed parent directories for LLMWiki mappings.
|
||||
|
||||
Revision ID: 20261006_01
|
||||
Revises: 20260920_02
|
||||
Create Date: 2026-10-06
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20261006_01"
|
||||
down_revision = "20260920_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "llmwiki_knowledge_bases"
|
||||
_COLUMN = "wiki_parent_directories"
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TABLE):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
columns = _columns()
|
||||
if columns and _COLUMN not in columns:
|
||||
op.add_column(_TABLE, sa.Column(_COLUMN, PortableJSON(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _COLUMN in _columns():
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
"""Persist administrator-managed per-model runtime capabilities.
|
||||
|
||||
Revision ID: 20261009_01
|
||||
Revises: 20261006_01
|
||||
Create Date: 2026-10-09
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20261009_01"
|
||||
down_revision = "20261006_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "model_capabilities"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("model_name", sa.String(length=160), primary_key=True),
|
||||
sa.Column(
|
||||
"supports_reasoning_effort",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
sa.Column("updated_by", sa.String(length=64), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
op.drop_table(_TABLE)
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
"""Persist the global ordinary-knowledge smart-reasoning switch.
|
||||
|
||||
Revision ID: 20261014_01
|
||||
Revises: 20261009_01
|
||||
Create Date: 2026-10-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20261014_01"
|
||||
down_revision = "20261009_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "knowledge_retrieval_settings"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("scope", sa.String(length=64), primary_key=True),
|
||||
sa.Column(
|
||||
"smart_reasoning_enabled",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
sa.Column("revision", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("updated_by", sa.String(length=64), nullable=True),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if sa.inspect(op.get_bind()).has_table(_TABLE):
|
||||
op.drop_table(_TABLE)
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""Persist hierarchical Wiki generation configuration.
|
||||
|
||||
Revision ID: 20261014_02
|
||||
Revises: 20261014_01
|
||||
Create Date: 2026-10-14
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from deerflow.persistence.types import PortableJSON
|
||||
|
||||
revision = "20261014_02"
|
||||
down_revision = "20261014_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "llmwiki_knowledge_bases"
|
||||
_COLUMNS: tuple[sa.Column, ...] = (
|
||||
sa.Column("wiki_directory_paths", PortableJSON(), nullable=True),
|
||||
sa.Column("wiki_focus_instructions", sa.Text(), nullable=True),
|
||||
sa.Column("wiki_content_instructions_raw", sa.Text(), nullable=True),
|
||||
sa.Column("wiki_extraction_instructions_raw", sa.Text(), nullable=True),
|
||||
sa.Column("wiki_generation_managed", sa.Boolean(), nullable=True),
|
||||
sa.Column("wiki_config_schema_version", sa.Integer(), nullable=True),
|
||||
sa.Column("wiki_config_revision", sa.Integer(), nullable=True),
|
||||
sa.Column("wiki_config_apply_status", sa.String(length=32), nullable=True),
|
||||
sa.Column("wiki_config_apply_error", sa.Text(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TABLE):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
existing = _columns()
|
||||
if not existing:
|
||||
return
|
||||
for column in _COLUMNS:
|
||||
if column.name not in existing:
|
||||
op.add_column(_TABLE, column)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
existing = _columns()
|
||||
for column in reversed(_COLUMNS):
|
||||
if column.name in existing:
|
||||
op.drop_column(_TABLE, column.name)
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
"""Index task deep-link thread lookups outside the metadata JSON blob.
|
||||
|
||||
Revision ID: 20261015_01
|
||||
Revises: 20260920_02
|
||||
Create Date: 2026-10-15
|
||||
|
||||
``threads_meta.metadata_json`` remains the public source of truth. The new
|
||||
columns are maintained projections for the high-frequency task workspace
|
||||
lookup, whose task rows are shared across users and therefore cannot use the
|
||||
ordinary ``user_id`` index.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect, text
|
||||
|
||||
revision = "20261015_01"
|
||||
down_revision = "20260920_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "threads_meta"
|
||||
_COLUMNS = (
|
||||
("task_id", 128),
|
||||
("agent_id", 128),
|
||||
("action_id", 128),
|
||||
)
|
||||
_INDEXES = (
|
||||
("ix_threads_meta_task_agent_updated", ("task_id", "agent_id", "updated_at", "thread_id")),
|
||||
("ix_threads_meta_task_action_agent_updated", ("task_id", "action_id", "agent_id", "updated_at", "thread_id")),
|
||||
("ix_threads_meta_user_updated", ("user_id", "updated_at", "thread_id")),
|
||||
)
|
||||
|
||||
|
||||
def _has_table(table_name: str) -> bool:
|
||||
return table_name in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _has_column(column_name: str) -> bool:
|
||||
return any(column["name"] == column_name for column in inspect(op.get_bind()).get_columns(_TABLE))
|
||||
|
||||
|
||||
def _has_index(index_name: str) -> bool:
|
||||
return index_name in {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
|
||||
|
||||
|
||||
def _json_scalar_expression(key: str) -> str | None:
|
||||
"""Return a portable-enough JSON scalar extractor for supported engines."""
|
||||
dialect = op.get_bind().dialect.name
|
||||
if dialect == "sqlite":
|
||||
return f"json_extract(metadata_json, '$.{key}')"
|
||||
if dialect == "mysql":
|
||||
# PortableJSON is LONGTEXT on MySQL, but JSON_EXTRACT accepts valid
|
||||
# JSON text and returns the quoted JSON scalar until unquoted here.
|
||||
return f"JSON_UNQUOTE(JSON_EXTRACT(metadata_json, '$.{key}'))"
|
||||
if dialect in {"postgresql", "opengauss"}:
|
||||
return f"metadata_json ->> '{key}'"
|
||||
return None
|
||||
|
||||
|
||||
def _backfill_projections() -> None:
|
||||
expressions = {
|
||||
"task_id": _json_scalar_expression("taskId"),
|
||||
"agent_id": _json_scalar_expression("agent_id"),
|
||||
"action_id": _json_scalar_expression("actionId"),
|
||||
}
|
||||
if any(expression is None for expression in expressions.values()):
|
||||
return
|
||||
assignments = ", ".join(
|
||||
f"{column} = {expression}"
|
||||
for column, expression in expressions.items()
|
||||
)
|
||||
# Only fresh projection columns are NULL. This makes a retry safe and
|
||||
# never overwrites values maintained by a newer running application.
|
||||
op.get_bind().execute(
|
||||
text(
|
||||
f"UPDATE {_TABLE} SET {assignments} "
|
||||
"WHERE task_id IS NULL AND agent_id IS NULL AND action_id IS NULL"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _has_table(_TABLE):
|
||||
return
|
||||
for column_name, length in _COLUMNS:
|
||||
if not _has_column(column_name):
|
||||
op.add_column(_TABLE, sa.Column(column_name, sa.String(length), nullable=True))
|
||||
_backfill_projections()
|
||||
for index_name, columns in _INDEXES:
|
||||
if not _has_index(index_name):
|
||||
op.create_index(index_name, _TABLE, list(columns))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table(_TABLE):
|
||||
return
|
||||
for index_name, _columns in reversed(_INDEXES):
|
||||
if _has_index(index_name):
|
||||
op.drop_index(index_name, table_name=_TABLE)
|
||||
for column_name, _length in reversed(_COLUMNS):
|
||||
if _has_column(column_name):
|
||||
op.drop_column(_TABLE, column_name)
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
"""Add dedicated single-agent task deep-link conversation index.
|
||||
|
||||
Revision ID: 20261019_01
|
||||
Revises: 20261015_01
|
||||
Create Date: 2026-10-19
|
||||
|
||||
This is intentionally schema-only. Existing ``threads_meta`` task rows are not
|
||||
backfilled; only conversations created/updated with the new ``taskGoPath``
|
||||
metadata marker enter the dedicated history.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import inspect
|
||||
|
||||
revision = "20261019_01"
|
||||
down_revision = "20261015_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "task_deeplink_threads"
|
||||
_INDEX = "ix_task_deeplink_threads_scope"
|
||||
|
||||
|
||||
def _has_table() -> bool:
|
||||
return _TABLE in inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if _has_table():
|
||||
return
|
||||
op.create_table(
|
||||
_TABLE,
|
||||
sa.Column("thread_id", sa.String(64), primary_key=True),
|
||||
sa.Column("task_id", sa.String(128), nullable=False),
|
||||
sa.Column("source_task_id", sa.String(128), nullable=False),
|
||||
sa.Column("go_path", sa.String(16), nullable=False),
|
||||
sa.Column("agent_id", sa.String(128), nullable=True),
|
||||
sa.Column("created_by", sa.String(64), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
_INDEX,
|
||||
_TABLE,
|
||||
["task_id", "go_path", "updated_at", "thread_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if not _has_table():
|
||||
return
|
||||
indexes = {index["name"] for index in inspect(op.get_bind()).get_indexes(_TABLE)}
|
||||
if _INDEX in indexes:
|
||||
op.drop_index(_INDEX, table_name=_TABLE)
|
||||
op.drop_table(_TABLE)
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
"""Persist the knowledge-management advanced-entry switch.
|
||||
|
||||
Revision ID: 20261020_01
|
||||
Revises: 20261019_01, 20261014_02
|
||||
Create Date: 2026-10-20
|
||||
|
||||
An earlier release accidentally reused revision ``20261019_01`` for this
|
||||
column and for the task deep-link table. A database that ran the column
|
||||
migration can therefore report the task migration as applied even though its
|
||||
table is absent. The upgrade reconciles that table and its scope index before
|
||||
ensuring the advanced-management column, so both fresh and affected databases
|
||||
converge on the same schema. The task objects belong to the parent revision and
|
||||
are intentionally left intact by this revision's downgrade.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20261020_01"
|
||||
down_revision = ("20261019_01", "20261014_02")
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE = "knowledge_retrieval_settings"
|
||||
_COLUMN = "advanced_management_enabled"
|
||||
_TASK_TABLE = "task_deeplink_threads"
|
||||
_TASK_INDEX = "ix_task_deeplink_threads_scope"
|
||||
|
||||
|
||||
def _columns() -> set[str]:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TABLE):
|
||||
return set()
|
||||
return {str(column["name"]) for column in inspector.get_columns(_TABLE)}
|
||||
|
||||
|
||||
def _ensure_task_deeplink_schema() -> None:
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if not inspector.has_table(_TASK_TABLE):
|
||||
op.create_table(
|
||||
_TASK_TABLE,
|
||||
sa.Column("thread_id", sa.String(64), primary_key=True),
|
||||
sa.Column("task_id", sa.String(128), nullable=False),
|
||||
sa.Column("source_task_id", sa.String(128), nullable=False),
|
||||
sa.Column("go_path", sa.String(16), nullable=False),
|
||||
sa.Column("agent_id", sa.String(128), nullable=True),
|
||||
sa.Column("created_by", sa.String(64), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
_TASK_INDEX,
|
||||
_TASK_TABLE,
|
||||
["task_id", "go_path", "updated_at", "thread_id"],
|
||||
)
|
||||
return
|
||||
|
||||
indexes = {
|
||||
str(index["name"])
|
||||
for index in inspector.get_indexes(_TASK_TABLE)
|
||||
if index.get("name")
|
||||
}
|
||||
if _TASK_INDEX not in indexes:
|
||||
op.create_index(
|
||||
_TASK_INDEX,
|
||||
_TASK_TABLE,
|
||||
["task_id", "go_path", "updated_at", "thread_id"],
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
_ensure_task_deeplink_schema()
|
||||
if _COLUMN in _columns():
|
||||
return
|
||||
op.add_column(
|
||||
_TABLE,
|
||||
sa.Column(
|
||||
_COLUMN,
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _COLUMN in _columns():
|
||||
op.drop_column(_TABLE, _COLUMN)
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""Administrator-managed per-model runtime capabilities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
|
||||
from deerflow.persistence.model_capabilities.memory import MemoryModelCapabilityStore
|
||||
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
|
||||
from deerflow.persistence.model_capabilities.sql import ModelCapabilityRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryModelCapabilityStore",
|
||||
"ModelCapabilityRepository",
|
||||
"ModelCapabilityRow",
|
||||
"ModelCapabilityStore",
|
||||
"import_legacy_reasoning_effort_capabilities",
|
||||
"make_model_capability_store",
|
||||
]
|
||||
|
||||
|
||||
async def import_legacy_reasoning_effort_capabilities(
|
||||
store: ModelCapabilityStore,
|
||||
models: Iterable[Any],
|
||||
) -> list[str]:
|
||||
"""One-time compatibility import for legacy config-backed ``true`` flags.
|
||||
|
||||
A configured ``false`` never creates a row. Once any database row exists,
|
||||
its value is authoritative and cannot be overwritten by legacy YAML.
|
||||
"""
|
||||
|
||||
imported: list[str] = []
|
||||
for model in models:
|
||||
if not bool(getattr(model, "supports_reasoning_effort", False)):
|
||||
continue
|
||||
model_name = str(getattr(model, "name", "") or "").strip()
|
||||
if model_name and await store.seed_reasoning_effort_support(
|
||||
model_name,
|
||||
updated_by="legacy-config-import",
|
||||
):
|
||||
imported.append(model_name)
|
||||
return imported
|
||||
|
||||
|
||||
def make_model_capability_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryModelCapabilityStore()
|
||||
return ModelCapabilityRepository(session_factory)
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""Persistence contract for administrator-managed model capabilities.
|
||||
|
||||
The provider connection details still live in ``config.yaml``. Runtime
|
||||
capabilities that administrators change from the web UI live here instead so
|
||||
they are shared by every Gateway worker and survive application restarts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ModelCapabilityStore(ABC):
|
||||
"""Global capability settings keyed by the stable configured model name."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_capabilities(self) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
supports_reasoning_effort: bool,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def seed_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
"""Insert legacy ``true`` only when no authoritative row exists."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_capability(self, model_name: str) -> None:
|
||||
raise NotImplementedError
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""In-memory model-capability store used by memory-mode deployments/tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
|
||||
|
||||
|
||||
class MemoryModelCapabilityStore(ModelCapabilityStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def list_capabilities(self) -> list[dict[str, Any]]:
|
||||
return [dict(self._rows[name]) for name in sorted(self._rows)]
|
||||
|
||||
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(model_name)
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
async def set_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
supports_reasoning_effort: bool,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
row = {
|
||||
"model_name": model_name,
|
||||
"supports_reasoning_effort": bool(supports_reasoning_effort),
|
||||
"updated_by": updated_by,
|
||||
"updated_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
self._rows[model_name] = row
|
||||
return dict(row)
|
||||
|
||||
async def seed_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
if model_name in self._rows:
|
||||
return False
|
||||
await self.set_reasoning_effort_support(
|
||||
model_name,
|
||||
supports_reasoning_effort=True,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
return True
|
||||
|
||||
async def delete_capability(self, model_name: str) -> None:
|
||||
self._rows.pop(model_name, None)
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""ORM row for administrator-managed model capabilities."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, String, false
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ModelCapabilityRow(Base):
|
||||
__tablename__ = "model_capabilities"
|
||||
|
||||
model_name: Mapped[str] = mapped_column(String(160), primary_key=True)
|
||||
supports_reasoning_effort: Mapped[bool] = mapped_column(
|
||||
Boolean,
|
||||
nullable=False,
|
||||
default=False,
|
||||
server_default=false(),
|
||||
)
|
||||
updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
"""SQLAlchemy-backed model-capability store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.model_capabilities.base import ModelCapabilityStore
|
||||
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
|
||||
|
||||
|
||||
def _to_dict(row: ModelCapabilityRow) -> dict[str, Any]:
|
||||
return {
|
||||
"model_name": row.model_name,
|
||||
"supports_reasoning_effort": bool(row.supports_reasoning_effort),
|
||||
"updated_by": row.updated_by,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
}
|
||||
|
||||
|
||||
class ModelCapabilityRepository(ModelCapabilityStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_capabilities(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(select(ModelCapabilityRow).order_by(ModelCapabilityRow.model_name.asc()))
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_capability(self, model_name: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ModelCapabilityRow, model_name)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def set_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
supports_reasoning_effort: bool,
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ModelCapabilityRow, model_name)
|
||||
if row is None:
|
||||
row = ModelCapabilityRow(model_name=model_name)
|
||||
session.add(row)
|
||||
row.supports_reasoning_effort = bool(supports_reasoning_effort)
|
||||
row.updated_by = updated_by
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def seed_reasoning_effort_support(
|
||||
self,
|
||||
model_name: str,
|
||||
*,
|
||||
updated_by: str | None = None,
|
||||
) -> bool:
|
||||
"""Insert a legacy ``true`` without ever overwriting an existing row.
|
||||
|
||||
The primary-key conflict handles concurrent Gateway startup safely: if
|
||||
another worker or an administrator creates the row after our read, the
|
||||
losing seed transaction rolls back and preserves that authoritative
|
||||
value, including ``false``.
|
||||
"""
|
||||
|
||||
async with self._sf() as session:
|
||||
if await session.get(ModelCapabilityRow, model_name) is not None:
|
||||
return False
|
||||
session.add(
|
||||
ModelCapabilityRow(
|
||||
model_name=model_name,
|
||||
supports_reasoning_effort=True,
|
||||
updated_by=updated_by,
|
||||
)
|
||||
)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
return False
|
||||
return True
|
||||
|
||||
async def delete_capability(self, model_name: str) -> None:
|
||||
async with self._sf() as session:
|
||||
await session.execute(delete(ModelCapabilityRow).where(ModelCapabilityRow.model_name == model_name))
|
||||
await session.commit()
|
||||
+78
-3
@@ -35,6 +35,18 @@ from deerflow.persistence.agents.model import (
|
||||
AgentRow,
|
||||
AgentSkillRow,
|
||||
)
|
||||
from deerflow.persistence.ai_writing_sessions.model import AIWritingSessionRow
|
||||
from deerflow.persistence.analysis_project_memory.model import (
|
||||
AnalysisProjectMemoryItemRow,
|
||||
AnalysisProjectMemoryJobRow,
|
||||
AnalysisProjectMemoryStateRow,
|
||||
)
|
||||
from deerflow.persistence.analysis_projects import AnalysisProjectConfigRow, AnalysisProjectRow, AnalysisProjectThreadRow
|
||||
from deerflow.persistence.article_types.model import ArticleTypeRow
|
||||
from deerflow.persistence.artifact_library.model import (
|
||||
ArtifactLibraryItemRow,
|
||||
ArtifactLibrarySyncStateRow,
|
||||
)
|
||||
from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeBaseRow,
|
||||
AssistantKnowledgeEntityContributionRow,
|
||||
@@ -48,7 +60,7 @@ from deerflow.persistence.assistant_knowledge.model import (
|
||||
AssistantKnowledgeWikiRevisionRow,
|
||||
)
|
||||
from deerflow.persistence.business_mapping.model import BusinessMappingRow
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
|
||||
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow, UserCompactMenuOverrideConfigRow
|
||||
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
|
||||
from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow
|
||||
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
|
||||
@@ -60,13 +72,26 @@ from deerflow.persistence.document_rewrite_versions.model import DocumentRewrite
|
||||
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
|
||||
from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
|
||||
from deerflow.persistence.force_situation.model import (
|
||||
ForceSituationDetailRow,
|
||||
ForceSituationDetailVersionRow,
|
||||
)
|
||||
from deerflow.persistence.gongwen_documents.model import GongwenDocumentRow, GongwenDocumentVersionRow
|
||||
from deerflow.persistence.html_pages.model import HtmlPageFavoriteRow, ScheduledHtmlPageRow
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
from deerflow.persistence.knowledge_retrieval_settings.model import KnowledgeRetrievalSettingsRow
|
||||
from deerflow.persistence.light_apps.model import LightAppRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
from deerflow.persistence.llmwiki_index.model import (
|
||||
LlmWikiWikiEdgeRow,
|
||||
LlmWikiWikiPageRow,
|
||||
LlmWikiWikiSyncStateRow,
|
||||
LlmWikiWikiVectorRow,
|
||||
)
|
||||
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
|
||||
from deerflow.persistence.menu_overrides.model import MenuOverrideRow, UserMenuOverrideConfigRow
|
||||
from deerflow.persistence.model_capabilities.model import ModelCapabilityRow
|
||||
from deerflow.persistence.models.run_event import RunEventRow
|
||||
from deerflow.persistence.notifications.model import NotificationRow
|
||||
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
|
||||
@@ -125,13 +150,28 @@ from deerflow.persistence.skill_knowledge.model import (
|
||||
SkillKnowledgeSyncJobRow,
|
||||
)
|
||||
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
|
||||
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
|
||||
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
|
||||
from deerflow.persistence.task_analysis.model import (
|
||||
TaskAnalysisBindingRow,
|
||||
TaskAnalysisContentVersionRow,
|
||||
TaskAnalysisRunMessageRow,
|
||||
TaskAnalysisRunRow,
|
||||
TaskAnalysisSettingRow,
|
||||
TaskAnalysisTaskRow,
|
||||
TaskDataBlockDefRow,
|
||||
TaskDataBlockRow,
|
||||
TaskDataPeriodRow,
|
||||
TaskDataSubmissionRow,
|
||||
)
|
||||
from deerflow.persistence.task_buttons.model import TaskButtonRow
|
||||
from deerflow.persistence.task_deeplink_threads.model import TaskDeeplinkThreadRow
|
||||
from deerflow.persistence.task_reports.model import TaskReportRow
|
||||
from deerflow.persistence.taskcop_tasks.model import TaskCopTaskRow
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.persistence.thread_shares.model import ThreadShareRow
|
||||
from deerflow.persistence.tool_metrics.model import ToolCallMetricRow
|
||||
from deerflow.persistence.upload_policy.model import UploadPolicyRow
|
||||
from deerflow.persistence.user.model import UserRow
|
||||
from deerflow.persistence.user_preferences.model import UserPreferencesRow
|
||||
from deerflow.persistence.user_prompts.model import UserCustomPromptRow
|
||||
@@ -149,8 +189,21 @@ from deerflow.persistence.workflow_runs.model import (
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
|
||||
__all__ = [
|
||||
"AIWritingSessionRow",
|
||||
"AnalysisProjectConfigRow",
|
||||
"AnalysisProjectMemoryItemRow",
|
||||
"AnalysisProjectMemoryJobRow",
|
||||
"AnalysisProjectMemoryStateRow",
|
||||
"ArtifactLibraryItemRow",
|
||||
"ArtifactLibrarySyncStateRow",
|
||||
"ArticleTypeRow",
|
||||
"FeedbackRow",
|
||||
"FixedQuestionRow",
|
||||
"ForceSituationDetailRow",
|
||||
"ForceSituationDetailVersionRow",
|
||||
"GongwenDocumentRow",
|
||||
"GongwenDocumentVersionRow",
|
||||
"HtmlPageFavoriteRow",
|
||||
"AgentExtrasRow",
|
||||
"AgentFavoriteRow",
|
||||
"AgentRow",
|
||||
@@ -159,6 +212,7 @@ __all__ = [
|
||||
"BusinessMappingRow",
|
||||
"AssistantKnowledgeBaseRow",
|
||||
"AssistantKnowledgeSourceRow",
|
||||
"AssistantKnowledgeVectorRow",
|
||||
"AssistantKnowledgeImportJobRow",
|
||||
"AssistantKnowledgeWikiPageRow",
|
||||
"AssistantKnowledgeWikiRevisionRow",
|
||||
@@ -167,6 +221,7 @@ __all__ = [
|
||||
"AssistantKnowledgeRelationRow",
|
||||
"AssistantKnowledgeRelationContributionRow",
|
||||
"CompactMenuOverrideRow",
|
||||
"UserCompactMenuOverrideConfigRow",
|
||||
"ConcurrencySampleRow",
|
||||
"DashboardSessionRow",
|
||||
"DocumentRewriteVersionRow",
|
||||
@@ -186,6 +241,7 @@ __all__ = [
|
||||
"KnowledgeRelationRow",
|
||||
"KnowledgeSourceRow",
|
||||
"KnowledgeTagRow",
|
||||
"KnowledgeRetrievalSettingsRow",
|
||||
"LightAppRow",
|
||||
"WorkflowDataSourceRow",
|
||||
"WorkflowDefinitionRow",
|
||||
@@ -200,9 +256,12 @@ __all__ = [
|
||||
"LlmWikiKnowledgeBaseRow",
|
||||
"LlmWikiConversationDepositRow",
|
||||
"LlmWikiWikiPageRow",
|
||||
"LlmWikiWikiEdgeRow",
|
||||
"LlmWikiWikiVectorRow",
|
||||
"LlmWikiWikiSyncStateRow",
|
||||
"MenuOverrideRow",
|
||||
"ModelCapabilityRow",
|
||||
"UserMenuOverrideConfigRow",
|
||||
"NotificationRow",
|
||||
"ParallelAgentTaskRow",
|
||||
"PositionRoundtableNodeRow",
|
||||
@@ -236,6 +295,7 @@ __all__ = [
|
||||
"RunEventRow",
|
||||
"RunRow",
|
||||
"ScheduledTaskDeliveryProfileRow",
|
||||
"ScheduledHtmlPageRow",
|
||||
"ScheduledTaskRow",
|
||||
"ScheduledTaskRunRow",
|
||||
"ScheduledTaskSubscriptionRow",
|
||||
@@ -254,9 +314,23 @@ __all__ = [
|
||||
"SkillKnowledgeSnapshotRow",
|
||||
"SkillKnowledgeSyncItemRow",
|
||||
"SkillKnowledgeSyncJobRow",
|
||||
"StrategyAgentRoleRow",
|
||||
"TagAssignmentRow",
|
||||
"TagRow",
|
||||
"TaskButtonRow",
|
||||
"TaskAnalysisBindingRow",
|
||||
"TaskAnalysisContentVersionRow",
|
||||
"TaskAnalysisRunMessageRow",
|
||||
"TaskAnalysisRunRow",
|
||||
"TaskAnalysisSettingRow",
|
||||
"TaskAnalysisTaskRow",
|
||||
"TaskDataBlockDefRow",
|
||||
"TaskDataBlockRow",
|
||||
"TaskDataPeriodRow",
|
||||
"TaskDataSubmissionRow",
|
||||
"TaskDeeplinkThreadRow",
|
||||
"AnalysisProjectRow",
|
||||
"AnalysisProjectThreadRow",
|
||||
"TaskCopTaskRow",
|
||||
"TaskReportRow",
|
||||
"EmbedSessionRow",
|
||||
@@ -264,6 +338,7 @@ __all__ = [
|
||||
"ThreadMetaRow",
|
||||
"ThreadShareRow",
|
||||
"ToolCallMetricRow",
|
||||
"UploadPolicyRow",
|
||||
"UserCustomPromptRow",
|
||||
"UserMemoryConfigRow",
|
||||
"UserPreferencesRow",
|
||||
|
||||
+61
-20
@@ -14,7 +14,7 @@ import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.parallel_agent_tasks.model import ParallelAgentTaskRow
|
||||
@@ -57,17 +57,40 @@ class ParallelAgentTaskRepository:
|
||||
data["messages"] = _loads(row.messages) or []
|
||||
return data
|
||||
|
||||
async def list_tasks(self, user_id: str, *, limit: int = 200) -> list[dict[str, Any]]:
|
||||
"""列出当前用户的任务(轻量元信息,不含 messages,供历史列表)。"""
|
||||
async def list_tasks(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
limit: int = 200,
|
||||
offset: int = 0,
|
||||
id_prefix: str | None = None,
|
||||
include_messages: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List the current user's tasks, optionally including saved snapshots.
|
||||
|
||||
History-list callers keep the lightweight default. Server-side
|
||||
projections such as the artifact library can opt into ``messages`` in
|
||||
one query instead of issuing an N+1 ``get_task`` sequence. ``id_prefix``
|
||||
and ``offset`` allow those projections to page only their own task
|
||||
namespace without loading unrelated, potentially large transcripts.
|
||||
"""
|
||||
stmt = select(ParallelAgentTaskRow).where(ParallelAgentTaskRow.user_id == user_id)
|
||||
if id_prefix:
|
||||
stmt = stmt.where(ParallelAgentTaskRow.id.startswith(id_prefix))
|
||||
stmt = (
|
||||
select(ParallelAgentTaskRow)
|
||||
.where(ParallelAgentTaskRow.user_id == user_id)
|
||||
.order_by(ParallelAgentTaskRow.updated_at.desc())
|
||||
stmt.order_by(
|
||||
ParallelAgentTaskRow.updated_at.desc(),
|
||||
ParallelAgentTaskRow.id.asc(),
|
||||
)
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._to_dict(row, with_messages=False) for row in result.scalars()]
|
||||
return [
|
||||
self._to_dict(row, with_messages=include_messages)
|
||||
for row in result.scalars()
|
||||
]
|
||||
|
||||
async def get_task(self, task_id: str, user_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
@@ -103,22 +126,40 @@ class ParallelAgentTaskRepository:
|
||||
brief=_UNSET,
|
||||
agents=_UNSET,
|
||||
messages=_UNSET,
|
||||
expected_updated_at: datetime | str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ParallelAgentTaskRow, task_id)
|
||||
if row is None or row.user_id != user_id:
|
||||
expected_revision = expected_updated_at
|
||||
if isinstance(expected_revision, str):
|
||||
try:
|
||||
expected_revision = datetime.fromisoformat(expected_revision.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
values: dict[str, Any] = {"updated_at": datetime.now(UTC)}
|
||||
if title is not _UNSET and title is not None:
|
||||
values["title"] = str(title).strip()
|
||||
if brief is not _UNSET:
|
||||
values["brief"] = brief or None
|
||||
if agents is not _UNSET:
|
||||
values["agents"] = _dumps(agents)
|
||||
if messages is not _UNSET:
|
||||
values["messages"] = _dumps(messages)
|
||||
|
||||
async with self._sf() as session:
|
||||
statement = update(ParallelAgentTaskRow).where(
|
||||
ParallelAgentTaskRow.id == task_id,
|
||||
ParallelAgentTaskRow.user_id == user_id,
|
||||
)
|
||||
if expected_revision is not None:
|
||||
statement = statement.where(ParallelAgentTaskRow.updated_at == expected_revision)
|
||||
result = await session.execute(statement.values(**values))
|
||||
if result.rowcount != 1:
|
||||
await session.rollback()
|
||||
return None
|
||||
if title is not _UNSET and title is not None:
|
||||
row.title = str(title).strip()
|
||||
if brief is not _UNSET:
|
||||
row.brief = brief or None
|
||||
if agents is not _UNSET:
|
||||
row.agents = _dumps(agents)
|
||||
if messages is not _UNSET:
|
||||
row.messages = _dumps(messages)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
row = await session.get(ParallelAgentTaskRow, task_id)
|
||||
if row is None:
|
||||
return None
|
||||
return self._to_dict(row)
|
||||
|
||||
async def delete_task(self, task_id: str, user_id: str) -> bool:
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
"""Global strategy-selection agent role registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.strategy_agent_roles.base import StrategyAgentRoleStore
|
||||
from deerflow.persistence.strategy_agent_roles.memory import (
|
||||
MemoryStrategyAgentRoleStore,
|
||||
)
|
||||
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
|
||||
from deerflow.persistence.strategy_agent_roles.sql import StrategyAgentRoleRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryStrategyAgentRoleStore",
|
||||
"StrategyAgentRoleRepository",
|
||||
"StrategyAgentRoleRow",
|
||||
"StrategyAgentRoleStore",
|
||||
"make_strategy_agent_role_store",
|
||||
]
|
||||
|
||||
|
||||
def make_strategy_agent_role_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryStrategyAgentRoleStore()
|
||||
return StrategyAgentRoleRepository(session_factory)
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""Abstract store for the global strategy-selection agent registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class StrategyAgentRoleStore(ABC):
|
||||
@abstractmethod
|
||||
async def ensure_defaults(self, rows: list[dict[str, Any]]) -> int:
|
||||
"""Insert missing default rows without re-enabling disabled angles."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_roles(self, *, include_disabled: bool) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_role(self, agent_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_role(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_role(
|
||||
self,
|
||||
agent_id: str,
|
||||
*,
|
||||
enabled: bool | None = None,
|
||||
sort_order: int | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_role(self, agent_id: str) -> bool:
|
||||
"""Hard-delete a registry row for create rollback only."""
|
||||
raise NotImplementedError
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
"""In-memory strategy-agent registry used by memory persistence and tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.strategy_agent_roles.base import StrategyAgentRoleStore
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
def _copy(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return dict(row)
|
||||
|
||||
|
||||
class MemoryStrategyAgentRoleStore(StrategyAgentRoleStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def ensure_defaults(self, rows: list[dict[str, Any]]) -> int:
|
||||
inserted = 0
|
||||
async with self._lock:
|
||||
for source in rows:
|
||||
agent_id = str(source["agent_id"])
|
||||
existing = self._rows.get(agent_id)
|
||||
if existing is not None:
|
||||
# Canonical fixed roles are executable infrastructure, not
|
||||
# administrator-managed lifecycle rows. Repair stale or
|
||||
# manually-corrupted metadata on every bootstrap. Angle
|
||||
# defaults deliberately retain their enabled/order state.
|
||||
if bool(source.get("fixed", False)):
|
||||
canonical = {
|
||||
"category": str(source["category"]),
|
||||
"fixed": True,
|
||||
"enabled": bool(source.get("enabled", True)),
|
||||
"sort_order": int(source.get("sort_order", 0)),
|
||||
}
|
||||
if any(existing.get(key) != value for key, value in canonical.items()):
|
||||
existing.update(canonical)
|
||||
existing["updated_at"] = _now()
|
||||
continue
|
||||
now = _now()
|
||||
self._rows[agent_id] = {
|
||||
"agent_id": agent_id,
|
||||
"category": str(source["category"]),
|
||||
"fixed": bool(source.get("fixed", False)),
|
||||
"enabled": bool(source.get("enabled", True)),
|
||||
"sort_order": int(source.get("sort_order", 0)),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
inserted += 1
|
||||
return inserted
|
||||
|
||||
async def list_roles(self, *, include_disabled: bool) -> list[dict[str, Any]]:
|
||||
async with self._lock:
|
||||
rows = [_copy(row) for row in self._rows.values() if include_disabled or bool(row.get("enabled"))]
|
||||
rows.sort(
|
||||
key=lambda row: (
|
||||
int(row.get("sort_order", 0)),
|
||||
str(row.get("agent_id", "")),
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
async def get_role(self, agent_id: str) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
row = self._rows.get(agent_id)
|
||||
return _copy(row) if row is not None else None
|
||||
|
||||
async def create_role(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
agent_id = str(data["agent_id"])
|
||||
async with self._lock:
|
||||
if agent_id in self._rows:
|
||||
raise ValueError(f"Strategy agent role '{agent_id}' already exists")
|
||||
now = _now()
|
||||
row = {
|
||||
"agent_id": agent_id,
|
||||
"category": str(data["category"]),
|
||||
"fixed": bool(data.get("fixed", False)),
|
||||
"enabled": bool(data.get("enabled", True)),
|
||||
"sort_order": int(data.get("sort_order", 0)),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._rows[agent_id] = row
|
||||
return _copy(row)
|
||||
|
||||
async def update_role(
|
||||
self,
|
||||
agent_id: str,
|
||||
*,
|
||||
enabled: bool | None = None,
|
||||
sort_order: int | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
row = self._rows.get(agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
if enabled is not None:
|
||||
row["enabled"] = bool(enabled)
|
||||
if sort_order is not None:
|
||||
row["sort_order"] = int(sort_order)
|
||||
row["updated_at"] = _now()
|
||||
return _copy(row)
|
||||
|
||||
async def delete_role(self, agent_id: str) -> bool:
|
||||
async with self._lock:
|
||||
return self._rows.pop(agent_id, None) is not None
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
"""Persistence model for the global strategy-selection agent registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class StrategyAgentRoleRow(Base):
|
||||
"""Classify folder-backed agents used by the strategy-selection workflow.
|
||||
|
||||
The agent's model, skills and SOUL remain authoritative in its normal
|
||||
``config.yaml`` / ``SOUL.md`` files. This small global table owns only the
|
||||
workflow classification and lifecycle flags. Keeping a disabled row (and
|
||||
the underlying agent) protects saved projects that still reference it.
|
||||
"""
|
||||
|
||||
__tablename__ = "strategy_agent_roles"
|
||||
|
||||
agent_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
category: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
|
||||
fixed: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="1", index=True)
|
||||
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+141
@@ -0,0 +1,141 @@
|
||||
"""SQLAlchemy-backed global strategy-selection agent registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.strategy_agent_roles.base import StrategyAgentRoleStore
|
||||
from deerflow.persistence.strategy_agent_roles.model import StrategyAgentRoleRow
|
||||
|
||||
|
||||
def _to_dict(row: StrategyAgentRoleRow) -> dict[str, Any]:
|
||||
return {
|
||||
"agent_id": row.agent_id,
|
||||
"category": row.category,
|
||||
"fixed": row.fixed,
|
||||
"enabled": row.enabled,
|
||||
"sort_order": row.sort_order,
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at,
|
||||
"updated_at": row.updated_at.isoformat() if isinstance(row.updated_at, datetime) else row.updated_at,
|
||||
}
|
||||
|
||||
|
||||
class StrategyAgentRoleRepository(StrategyAgentRoleStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def ensure_defaults(self, rows: list[dict[str, Any]]) -> int:
|
||||
inserted = 0
|
||||
for source in rows:
|
||||
# Commit each of the small number of defaults independently. If
|
||||
# two workers race on one primary key, rolling that save back must
|
||||
# not also discard unrelated defaults inserted by this worker.
|
||||
async with self._sf() as session:
|
||||
agent_id = str(source["agent_id"])
|
||||
existing = await session.get(StrategyAgentRoleRow, agent_id)
|
||||
if existing is not None:
|
||||
if bool(source.get("fixed", False)):
|
||||
changed = False
|
||||
canonical = {
|
||||
"category": str(source["category"]),
|
||||
"fixed": True,
|
||||
"enabled": bool(source.get("enabled", True)),
|
||||
"sort_order": int(source.get("sort_order", 0)),
|
||||
}
|
||||
for key, value in canonical.items():
|
||||
if getattr(existing, key) != value:
|
||||
setattr(existing, key, value)
|
||||
changed = True
|
||||
if changed:
|
||||
existing.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
continue
|
||||
now = datetime.now(UTC)
|
||||
session.add(
|
||||
StrategyAgentRoleRow(
|
||||
agent_id=agent_id,
|
||||
category=str(source["category"]),
|
||||
fixed=bool(source.get("fixed", False)),
|
||||
enabled=bool(source.get("enabled", True)),
|
||||
sort_order=int(source.get("sort_order", 0)),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# Another startup worker won this primary-key race.
|
||||
await session.rollback()
|
||||
continue
|
||||
inserted += 1
|
||||
return inserted
|
||||
|
||||
async def list_roles(self, *, include_disabled: bool) -> list[dict[str, Any]]:
|
||||
stmt = select(StrategyAgentRoleRow)
|
||||
if not include_disabled:
|
||||
stmt = stmt.where(StrategyAgentRoleRow.enabled.is_(True))
|
||||
stmt = stmt.order_by(
|
||||
StrategyAgentRoleRow.sort_order.asc(),
|
||||
StrategyAgentRoleRow.agent_id.asc(),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_to_dict(row) for row in result.scalars()]
|
||||
|
||||
async def get_role(self, agent_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(StrategyAgentRoleRow, agent_id)
|
||||
return _to_dict(row) if row is not None else None
|
||||
|
||||
async def create_role(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
now = datetime.now(UTC)
|
||||
row = StrategyAgentRoleRow(
|
||||
agent_id=str(data["agent_id"]),
|
||||
category=str(data["category"]),
|
||||
fixed=bool(data.get("fixed", False)),
|
||||
enabled=bool(data.get("enabled", True)),
|
||||
sort_order=int(data.get("sort_order", 0)),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError as exc:
|
||||
await session.rollback()
|
||||
raise ValueError(f"Strategy agent role '{row.agent_id}' already exists") from exc
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def update_role(
|
||||
self,
|
||||
agent_id: str,
|
||||
*,
|
||||
enabled: bool | None = None,
|
||||
sort_order: int | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(StrategyAgentRoleRow, agent_id)
|
||||
if row is None:
|
||||
return None
|
||||
if enabled is not None:
|
||||
row.enabled = bool(enabled)
|
||||
if sort_order is not None:
|
||||
row.sort_order = int(sort_order)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _to_dict(row)
|
||||
|
||||
async def delete_role(self, agent_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(StrategyAgentRoleRow).where(StrategyAgentRoleRow.agent_id == agent_id))
|
||||
await session.commit()
|
||||
return (result.rowcount or 0) > 0
|
||||
+3
-1
@@ -23,7 +23,9 @@ from deerflow.persistence.types import BeijingDateTime
|
||||
# Allowed tag/target types. Kept here so persistence + API share one source.
|
||||
# ``recommended_question`` was added for the 岗位 (position) subscription system
|
||||
# so admin-configured recommended questions can be tag-targeted like agents.
|
||||
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task", "recommended_question")
|
||||
# ``light_app`` labels 轻应用中心 apps (应用管理 / 应用列表); it is *not* a
|
||||
# 岗位 resource type — light apps stay globally visible.
|
||||
TAG_TYPES: tuple[str, ...] = ("agent", "skill", "scheduled_task", "recommended_question", "light_app")
|
||||
|
||||
|
||||
class TagRow(Base):
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""External-task multi-agent scheduled analysis persistence (任务分析定时)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
|
||||
from deerflow.persistence.task_analysis.data_blocks import (
|
||||
DEFAULT_DATA_BLOCK_KEYS,
|
||||
DEFAULT_DATA_BLOCKS,
|
||||
merge_payload,
|
||||
)
|
||||
from deerflow.persistence.task_analysis.memory import MemoryTaskAnalysisStore
|
||||
from deerflow.persistence.task_analysis.model import (
|
||||
TaskAnalysisBindingRow,
|
||||
TaskAnalysisContentVersionRow,
|
||||
TaskAnalysisRunMessageRow,
|
||||
TaskAnalysisRunRow,
|
||||
TaskAnalysisTaskRow,
|
||||
TaskDataBlockDefRow,
|
||||
TaskDataBlockRow,
|
||||
TaskDataPeriodRow,
|
||||
TaskDataSubmissionRow,
|
||||
)
|
||||
from deerflow.persistence.task_analysis.sql import TaskAnalysisRepository
|
||||
|
||||
__all__ = [
|
||||
"DEFAULT_DATA_BLOCK_KEYS",
|
||||
"DEFAULT_DATA_BLOCKS",
|
||||
"MemoryTaskAnalysisStore",
|
||||
"TaskAnalysisBindingRow",
|
||||
"TaskAnalysisContentVersionRow",
|
||||
"TaskAnalysisRepository",
|
||||
"TaskAnalysisRunMessageRow",
|
||||
"TaskAnalysisRunRow",
|
||||
"TaskAnalysisStore",
|
||||
"TaskAnalysisTaskRow",
|
||||
"TaskDataBlockDefRow",
|
||||
"TaskDataBlockRow",
|
||||
"TaskDataPeriodRow",
|
||||
"TaskDataSubmissionRow",
|
||||
"make_task_analysis_store",
|
||||
"merge_payload",
|
||||
]
|
||||
|
||||
|
||||
def make_task_analysis_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryTaskAnalysisStore()
|
||||
return TaskAnalysisRepository(session_factory)
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
"""Abstract store API for the external-task scheduled analysis feature."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
class TaskAnalysisStore(ABC):
|
||||
"""Tasks / bindings / runs / run-messages / content versions."""
|
||||
|
||||
# ── external tasks ──────────────────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create a task. Raises ValueError when source_task_id already exists."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_task(self, task_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_tasks(self) -> list[dict[str, Any]]:
|
||||
"""All tasks newest first, each augmented with its binding (if any)."""
|
||||
|
||||
@abstractmethod
|
||||
async def delete_task_cascade(self, task_id: str) -> bool:
|
||||
"""Delete task + binding + runs + messages + versions. True if existed."""
|
||||
|
||||
# ── bindings ────────────────────────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create-or-update the 1:1 binding for a task (refresh chain snapshot)."""
|
||||
|
||||
@abstractmethod
|
||||
async def delete_binding(self, external_task_id: str) -> bool: ...
|
||||
|
||||
@abstractmethod
|
||||
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
"""Enabled bindings with next_run_at <= now (execution candidates)."""
|
||||
|
||||
@abstractmethod
|
||||
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""Patch mutable fields (next_run_at / last_status / direction_notes...)."""
|
||||
|
||||
@abstractmethod
|
||||
async def last_bound_chain_id(self) -> str | None:
|
||||
"""Most recently updated binding's chain id (default option for create API)."""
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Idempotent create: same binding_id + scheduled_for returns existing row."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]: ...
|
||||
|
||||
@abstractmethod
|
||||
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
"""Runs still pending/running past ``older_than`` (watchdog recovery)."""
|
||||
|
||||
# ── run messages ────────────────────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
|
||||
"""Append batch of {seat_agent_id, stage_index, event_type, content}."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_run_messages(
|
||||
self, run_id: str, *, seat: str | None = None, limit: int = 500
|
||||
) -> list[dict[str, Any]]: ...
|
||||
|
||||
# ── content versions (timeline) ─────────────────────────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""One version per run (idempotent on run_id: replaces existing)."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_versions(self, external_task_id: str, *, limit: int = 50, offset: int = 0) -> list[dict[str, Any]]:
|
||||
"""Timeline versions newest first (对方详情接口数据源)."""
|
||||
|
||||
# ── data delivery: block defs / periods / submissions ────────────────────
|
||||
|
||||
@abstractmethod
|
||||
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
"""Seed the 5 default block defs when absent (create/legacy tasks); return ordered."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
"""Block defs ordered by sort_order (empty list when never seeded)."""
|
||||
|
||||
@abstractmethod
|
||||
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""Patch title/sort_order per block_key. ValueError on unknown block_key."""
|
||||
|
||||
@abstractmethod
|
||||
async def open_period(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
source: str,
|
||||
run_id: str | None = None,
|
||||
opened_at: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Append a new timeline period (period_index = max+1). Idempotent on run_id.
|
||||
|
||||
``opened_at`` 默认取当前时间;演示/回填场景可显式指定刻度时刻。
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None: ...
|
||||
|
||||
@abstractmethod
|
||||
async def current_period(self, task_id: str) -> dict[str, Any] | None:
|
||||
"""Latest period (submissions always land here)."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
|
||||
"""Periods ascending by period_index (timeline ticks)."""
|
||||
|
||||
@abstractmethod
|
||||
async def append_submission(
|
||||
self,
|
||||
task_id: str,
|
||||
period_index: int,
|
||||
block_key: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
submitted_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Append to the submission log (seq per task+period+block)."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_submissions(
|
||||
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Submission log for one block of one period ordered by seq (merge source)."""
|
||||
|
||||
@abstractmethod
|
||||
async def upsert_period_block(
|
||||
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
"""Write the merged payload for (task, period, block); bumps updated_at."""
|
||||
|
||||
@abstractmethod
|
||||
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
|
||||
"""All merged block rows of a task ({block_key, period_index, payload, updated_at})."""
|
||||
|
||||
@abstractmethod
|
||||
async def get_setting(self, key: str) -> str | None:
|
||||
"""Read one module setting value (``None`` when absent)."""
|
||||
|
||||
@abstractmethod
|
||||
async def set_setting(self, key: str, value: str) -> None:
|
||||
"""Upsert one module setting value."""
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
"""Constants and pure helpers for the scheduled data-delivery blocks.
|
||||
|
||||
块 key 定死 ``block_1``~``block_6``(已定稿口径,不做席位↔块映射);payload 不做
|
||||
schema 校验(schema 会变),只要求是 JSON 对象。合并规则:顶层 key 合并、同 key
|
||||
后写覆盖 —— 多个智能体可各写一个块的不同部分,一个智能体也可写多个块。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
# (block_key, 默认标题);建任务时自动种 6 条,标题/排序后续可经管理端修改。
|
||||
DEFAULT_DATA_BLOCKS: tuple[tuple[str, str], ...] = (
|
||||
("block_1", "数据块一"),
|
||||
("block_2", "数据块二"),
|
||||
("block_3", "数据块三"),
|
||||
("block_4", "数据块四"),
|
||||
("block_5", "数据块五"),
|
||||
("block_6", "数据块六"),
|
||||
)
|
||||
|
||||
DEFAULT_DATA_BLOCK_KEYS: tuple[str, ...] = tuple(key for key, _title in DEFAULT_DATA_BLOCKS)
|
||||
|
||||
# 期(时间轴刻度)触发来源。
|
||||
PERIOD_TRIGGER_CRON = "cron"
|
||||
PERIOD_TRIGGER_MANUAL = "manual"
|
||||
PERIOD_TRIGGER_EXTERNAL = "external"
|
||||
|
||||
|
||||
def default_block_defs() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"block_key": key, "title": title, "sort_order": index}
|
||||
for index, (key, title) in enumerate(DEFAULT_DATA_BLOCKS, start=1)
|
||||
]
|
||||
|
||||
|
||||
def is_default_block_key(block_key: str) -> bool:
|
||||
return block_key in DEFAULT_DATA_BLOCK_KEYS
|
||||
|
||||
|
||||
def merge_payload(base: dict[str, Any], incoming: dict[str, Any]) -> dict[str, Any]:
|
||||
"""顶层 key 合并、同 key 后写覆盖(纯函数,返回新 dict)。"""
|
||||
merged = dict(base)
|
||||
merged.update(incoming)
|
||||
return merged
|
||||
+467
@@ -0,0 +1,467 @@
|
||||
"""In-memory task-analysis store (no DB configured / tests).
|
||||
|
||||
Also serves as the behavioural reference for the SQL repository: field
|
||||
normalisation (JSON encode/decode, ISO datetime) lives in ``_normalize_*``
|
||||
helpers that the SQL layer mirrors.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
|
||||
|
||||
|
||||
def _iso(value: datetime | str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
return str(value)
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if value is None or isinstance(value, datetime):
|
||||
return value
|
||||
try:
|
||||
return datetime.fromisoformat(str(value))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _dump(value: Any) -> str | None:
|
||||
return None if value is None else json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _load(value: str | None, default: Any) -> Any:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
class MemoryTaskAnalysisStore(TaskAnalysisStore):
|
||||
def __init__(self) -> None:
|
||||
self._tasks: dict[str, dict[str, Any]] = {}
|
||||
self._settings: dict[str, str] = {}
|
||||
self._bindings: dict[str, dict[str, Any]] = {} # keyed by external_task_id
|
||||
self._runs: dict[str, dict[str, Any]] = {}
|
||||
self._messages: list[dict[str, Any]] = []
|
||||
self._versions: dict[str, dict[str, Any]] = {} # keyed by run_id
|
||||
# 数据交付:块定义 / 期刻度 / 合并块 / 提交流水。
|
||||
self._block_defs: dict[str, dict[str, dict[str, Any]]] = {} # task_id -> block_key -> def
|
||||
self._periods: dict[str, list[dict[str, Any]]] = {} # task_id -> ordered by period_index
|
||||
self._task_blocks: dict[tuple[str, int, str], dict[str, Any]] = {} # (task, period, block)
|
||||
self._submissions: dict[tuple[str, int, str], list[dict[str, Any]]] = {}
|
||||
self._seq = 0
|
||||
# 绑定最近写入序号(时钟精度不足时仍能稳定判定“最近一次绑定”)。
|
||||
self._binding_rev = 0
|
||||
|
||||
def _next_id(self, prefix: str) -> str:
|
||||
self._seq += 1
|
||||
return f"{prefix}-{self._seq:06d}"
|
||||
|
||||
# ── data delivery helpers ───────────────────────────────────────────────
|
||||
|
||||
def _seed_block_defs(self, task_id: str) -> None:
|
||||
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
|
||||
|
||||
existing = self._block_defs.setdefault(task_id, {})
|
||||
now = datetime.now(UTC).isoformat()
|
||||
for definition in default_block_defs():
|
||||
if definition["block_key"] in existing:
|
||||
continue
|
||||
existing[definition["block_key"]] = {
|
||||
"id": self._next_id("tdbd"),
|
||||
"task_id": task_id,
|
||||
**definition,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
|
||||
def _ordered_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
rows = list(self._block_defs.get(task_id, {}).values())
|
||||
rows.sort(key=lambda d: (d.get("sort_order", 0), d.get("block_key", "")))
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def _period_rows(self, task_id: str) -> list[dict[str, Any]]:
|
||||
rows = list(self._periods.get(task_id, []))
|
||||
rows.sort(key=lambda p: p["period_index"])
|
||||
return rows
|
||||
|
||||
# ── tasks ───────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
source = str(data["source_task_id"])
|
||||
if await self.get_task_by_source(source) is not None:
|
||||
raise ValueError(f"source_task_id already exists: {source}")
|
||||
now = datetime.now(UTC).isoformat()
|
||||
task = {
|
||||
"id": self._next_id("ta"),
|
||||
"source_task_id": source,
|
||||
"name": str(data.get("name") or ""),
|
||||
"description": data.get("description"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._tasks[task["id"]] = dict(task)
|
||||
# 建任务即种 6 条数据块定义(块 key 定死,标题后续可改)。
|
||||
self._seed_block_defs(task["id"])
|
||||
return dict(task)
|
||||
|
||||
async def get_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
task = self._tasks.get(task_id)
|
||||
return dict(task) if task else None
|
||||
|
||||
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None:
|
||||
for task in self._tasks.values():
|
||||
if task["source_task_id"] == source_task_id:
|
||||
return dict(task)
|
||||
return None
|
||||
|
||||
async def list_tasks(self) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
for task in sorted(self._tasks.values(), key=lambda t: t["created_at"], reverse=True):
|
||||
row = dict(task)
|
||||
row["binding"] = dict(self._bindings[task["id"]]) if task["id"] in self._bindings else None
|
||||
out.append(row)
|
||||
return out
|
||||
|
||||
async def delete_task_cascade(self, task_id: str) -> bool:
|
||||
if task_id not in self._tasks:
|
||||
return False
|
||||
del self._tasks[task_id]
|
||||
self._bindings.pop(task_id, None)
|
||||
run_ids = [run_id for run_id, run in self._runs.items() if run["external_task_id"] == task_id]
|
||||
for run_id in run_ids:
|
||||
del self._runs[run_id]
|
||||
self._versions.pop(run_id, None)
|
||||
self._messages = [m for m in self._messages if m["run_id"] not in set(run_ids)]
|
||||
self._block_defs.pop(task_id, None)
|
||||
self._periods.pop(task_id, None)
|
||||
self._task_blocks = {
|
||||
key: row for key, row in self._task_blocks.items() if key[0] != task_id
|
||||
}
|
||||
self._submissions = {
|
||||
key: rows for key, rows in self._submissions.items() if key[0] != task_id
|
||||
}
|
||||
return True
|
||||
|
||||
# ── bindings ────────────────────────────────────────────────────────────
|
||||
|
||||
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None:
|
||||
binding = self._bindings.get(external_task_id)
|
||||
return dict(binding) if binding else None
|
||||
|
||||
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None:
|
||||
for binding in self._bindings.values():
|
||||
if binding["id"] == binding_id:
|
||||
return dict(binding)
|
||||
return None
|
||||
|
||||
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
existing = self._bindings.get(external_task_id)
|
||||
now = datetime.now(UTC).isoformat()
|
||||
self._binding_rev += 1
|
||||
if existing is None:
|
||||
row = {
|
||||
"id": self._next_id("tab"),
|
||||
"external_task_id": external_task_id,
|
||||
"chain_id": str(data["chain_id"]),
|
||||
"chain_snapshot": data.get("chain_snapshot") or {},
|
||||
"cron_expr": str(data.get("cron_expr") or ""),
|
||||
"timezone": str(data.get("timezone") or "Asia/Shanghai"),
|
||||
"schedule_type": str(data.get("schedule_type") or "cron"),
|
||||
"enabled": bool(data.get("enabled", True)),
|
||||
"next_run_at": _iso(data.get("next_run_at")),
|
||||
"coordinator_prompt": data.get("coordinator_prompt"),
|
||||
"direction_notes": list(data.get("direction_notes") or []),
|
||||
"last_run_at": None,
|
||||
"last_status": None,
|
||||
"model": data.get("model"),
|
||||
"created_by": data.get("created_by"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"_rev": self._binding_rev,
|
||||
}
|
||||
else:
|
||||
row = dict(existing)
|
||||
row["_rev"] = self._binding_rev
|
||||
row.update({k: v for k, v in data.items() if k in {
|
||||
"chain_id", "chain_snapshot", "cron_expr", "timezone", "schedule_type", "enabled",
|
||||
"next_run_at", "coordinator_prompt", "model",
|
||||
}})
|
||||
if "direction_notes" in data:
|
||||
row["direction_notes"] = list(data["direction_notes"])
|
||||
row["updated_at"] = now
|
||||
self._bindings[external_task_id] = row
|
||||
return dict(row)
|
||||
|
||||
async def delete_binding(self, external_task_id: str) -> bool:
|
||||
return self._bindings.pop(external_task_id, None) is not None
|
||||
|
||||
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
due = []
|
||||
for binding in self._bindings.values():
|
||||
if not binding.get("enabled"):
|
||||
continue
|
||||
next_at = _parse_dt(binding.get("next_run_at"))
|
||||
if next_at is not None and next_at <= now:
|
||||
due.append(binding)
|
||||
due.sort(key=lambda b: b.get("next_run_at") or "")
|
||||
return [dict(b) for b in due[:limit]]
|
||||
|
||||
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
for key, binding in self._bindings.items():
|
||||
if binding["id"] == binding_id:
|
||||
binding.update({k: v for k, v in patch.items() if k != "id"})
|
||||
binding["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return dict(binding)
|
||||
return None
|
||||
|
||||
async def last_bound_chain_id(self) -> str | None:
|
||||
if not self._bindings:
|
||||
return None
|
||||
latest = max(self._bindings.values(), key=lambda b: b.get("_rev", 0))
|
||||
return latest.get("chain_id")
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
binding_id = data["binding_id"]
|
||||
scheduled_for = _parse_dt(data["scheduled_for"]) or datetime.now(UTC)
|
||||
for run in self._runs.values():
|
||||
if run["binding_id"] == binding_id and run["scheduled_for"] == scheduled_for.isoformat():
|
||||
return dict(run)
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row = {
|
||||
"id": self._next_id("tar"),
|
||||
"binding_id": binding_id,
|
||||
"external_task_id": str(data["external_task_id"]),
|
||||
"trigger": str(data.get("trigger") or "schedule"),
|
||||
"scheduled_for": scheduled_for.isoformat(),
|
||||
"started_at": now,
|
||||
"finished_at": None,
|
||||
"status": str(data.get("status") or "pending"),
|
||||
"stages_snapshot": data.get("stages_snapshot") or [],
|
||||
"stage_goals_snapshot": data.get("stage_goals_snapshot") or [],
|
||||
"seat_names_snapshot": data.get("seat_names_snapshot") or {},
|
||||
"seat_results": data.get("seat_results") or {},
|
||||
"error": data.get("error"),
|
||||
"retry_count": 0,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._runs[row["id"]] = row
|
||||
return dict(row)
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
return dict(run) if run else None
|
||||
|
||||
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._runs.values() if r["external_task_id"] == external_task_id]
|
||||
rows.sort(key=lambda r: r["started_at"], reverse=True)
|
||||
return [dict(r) for r in rows[:limit]]
|
||||
|
||||
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
run.update({k: v for k, v in patch.items() if k not in {"id", "binding_id"}})
|
||||
run["updated_at"] = datetime.now(UTC).isoformat()
|
||||
return dict(run)
|
||||
|
||||
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
for run in self._runs.values():
|
||||
if run["status"] in {"pending", "running"}:
|
||||
started = _parse_dt(run["started_at"])
|
||||
if started is not None and started < older_than:
|
||||
out.append(run)
|
||||
out.sort(key=lambda r: r["started_at"])
|
||||
return [dict(r) for r in out[:limit]]
|
||||
|
||||
# ── messages ────────────────────────────────────────────────────────────
|
||||
|
||||
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
|
||||
base_seq = sum(1 for m in self._messages if m["run_id"] == run_id)
|
||||
added = 0
|
||||
for index, message in enumerate(messages):
|
||||
self._messages.append({
|
||||
"id": self._next_id("tam"),
|
||||
"run_id": run_id,
|
||||
"seq": base_seq + index,
|
||||
"seat_agent_id": str(message.get("seat_agent_id") or ""),
|
||||
"stage_index": int(message.get("stage_index") or 0),
|
||||
"event_type": str(message.get("event_type") or "llm.ai"),
|
||||
"content": message.get("content"),
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
})
|
||||
added += 1
|
||||
return added
|
||||
|
||||
async def list_run_messages(
|
||||
self, run_id: str, *, seat: str | None = None, limit: int = 500
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = [m for m in self._messages if m["run_id"] == run_id]
|
||||
if seat is not None:
|
||||
rows = [m for m in rows if m["seat_agent_id"] == seat]
|
||||
rows.sort(key=lambda m: m["seq"])
|
||||
return [dict(m) for m in rows[:limit]]
|
||||
|
||||
# ── versions ────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(data["run_id"])
|
||||
row = {
|
||||
"id": self._next_id("tav"),
|
||||
"external_task_id": str(data["external_task_id"]),
|
||||
"run_id": run_id,
|
||||
"blocks": data.get("blocks") or {},
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
self._versions[run_id] = row
|
||||
return dict(row)
|
||||
|
||||
async def list_versions(
|
||||
self, external_task_id: str, *, limit: int = 50, offset: int = 0
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = [v for v in self._versions.values() if v["external_task_id"] == external_task_id]
|
||||
rows.sort(key=lambda v: v["created_at"], reverse=True)
|
||||
return [dict(v) for v in rows[offset : offset + limit]]
|
||||
|
||||
# ── data delivery: block defs / periods / submissions ────────────────────
|
||||
|
||||
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
self._seed_block_defs(task_id)
|
||||
return self._ordered_defs(task_id)
|
||||
|
||||
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
return self._ordered_defs(task_id)
|
||||
|
||||
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
definitions = self._block_defs.setdefault(task_id, {})
|
||||
now = datetime.now(UTC).isoformat()
|
||||
for update in updates:
|
||||
block_key = str(update.get("block_key") or "")
|
||||
row = definitions.get(block_key)
|
||||
if row is None:
|
||||
raise ValueError(f"未知数据块:{block_key}")
|
||||
if update.get("title") is not None:
|
||||
row["title"] = str(update["title"])
|
||||
if update.get("sort_order") is not None:
|
||||
row["sort_order"] = int(update["sort_order"])
|
||||
row["updated_at"] = now
|
||||
return self._ordered_defs(task_id)
|
||||
|
||||
async def open_period(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
source: str,
|
||||
run_id: str | None = None,
|
||||
opened_at: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if run_id:
|
||||
for period in self._periods.get(task_id, []):
|
||||
if period.get("run_id") == run_id:
|
||||
return dict(period)
|
||||
rows = self._periods.setdefault(task_id, [])
|
||||
index = (rows[-1]["period_index"] if rows else 0) + 1
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row = {
|
||||
"id": self._next_id("tdp"),
|
||||
"task_id": task_id,
|
||||
"period_index": index,
|
||||
"opened_at": _iso(opened_at) or now,
|
||||
"trigger": str(source or "cron"),
|
||||
"run_id": run_id,
|
||||
"created_at": now,
|
||||
}
|
||||
rows.append(row)
|
||||
return dict(row)
|
||||
|
||||
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None:
|
||||
for period in self._period_rows(task_id):
|
||||
if period["period_index"] == period_index:
|
||||
return dict(period)
|
||||
return None
|
||||
|
||||
async def current_period(self, task_id: str) -> dict[str, Any] | None:
|
||||
rows = self._period_rows(task_id)
|
||||
return dict(rows[-1]) if rows else None
|
||||
|
||||
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
|
||||
rows = self._period_rows(task_id)
|
||||
return [dict(p) for p in rows[-limit:]]
|
||||
|
||||
async def append_submission(
|
||||
self,
|
||||
task_id: str,
|
||||
period_index: int,
|
||||
block_key: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
submitted_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
key = (task_id, period_index, block_key)
|
||||
rows = self._submissions.setdefault(key, [])
|
||||
row = {
|
||||
"id": self._next_id("tds"),
|
||||
"task_id": task_id,
|
||||
"period_index": period_index,
|
||||
"block_key": block_key,
|
||||
"payload": dict(payload),
|
||||
"submitted_by": submitted_by,
|
||||
"seq": len(rows),
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
rows.append(row)
|
||||
return dict(row)
|
||||
|
||||
async def list_submissions(
|
||||
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = list(self._submissions.get((task_id, period_index, block_key), []))
|
||||
rows.sort(key=lambda s: s["seq"])
|
||||
return [dict(s) for s in rows[:limit]]
|
||||
|
||||
async def upsert_period_block(
|
||||
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
key = (task_id, period_index, block_key)
|
||||
now = datetime.now(UTC).isoformat()
|
||||
row = self._task_blocks.get(key)
|
||||
if row is None:
|
||||
row = {
|
||||
"id": self._next_id("tdb"),
|
||||
"task_id": task_id,
|
||||
"period_index": period_index,
|
||||
"block_key": block_key,
|
||||
"payload": dict(payload),
|
||||
"updated_at": now,
|
||||
"created_at": now,
|
||||
}
|
||||
self._task_blocks[key] = row
|
||||
else:
|
||||
row["payload"] = dict(payload)
|
||||
row["updated_at"] = now
|
||||
return dict(row)
|
||||
|
||||
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
|
||||
rows = [dict(row) for key, row in self._task_blocks.items() if key[0] == task_id]
|
||||
rows.sort(key=lambda r: (r["period_index"], r["block_key"]))
|
||||
return rows[:limit]
|
||||
|
||||
# ── settings ────────────────────────────────────────────────────────────
|
||||
|
||||
async def get_setting(self, key: str) -> str | None:
|
||||
return self._settings.get(key)
|
||||
|
||||
async def set_setting(self, key: str, value: str) -> None:
|
||||
self._settings[key] = value
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
"""ORM models for the external-task multi-agent scheduled analysis (任务分析定时).
|
||||
|
||||
Feature summary (design doc in the frontend ``official-task-analysis`` module):
|
||||
an external system pushes tasks via our create API; each task binds 1:1 to a
|
||||
business-chain orchestration plus a cron schedule; the scheduler runs the chain
|
||||
stage-by-stage (seats within a stage in parallel) and persists per-seat process
|
||||
messages plus a content "timeline" version per run. Manual runs wrap the
|
||||
roundtable Step-2 coordinator Q&A and are stored with the same message table
|
||||
(``seat_agent_id`` distinguishes seats; the coordinator uses ``__coordinator__``).
|
||||
|
||||
Tables:
|
||||
- ``task_analysis_tasks`` — external tasks (source_task_id unique)
|
||||
- ``task_analysis_bindings`` — 1:1 binding (chain snapshot + cron + notes)
|
||||
- ``task_analysis_runs`` — executions (trigger: schedule | manual)
|
||||
- ``task_analysis_run_messages`` — per-seat process messages for a run
|
||||
- ``task_analysis_content_versions`` — one timeline version per run
|
||||
- ``task_data_block_defs`` — the 5 fixed data blocks per task (title/sort editable)
|
||||
- ``task_data_periods`` — data-delivery timeline ticks (period_index per task)
|
||||
- ``task_data_blocks`` — merged payload per (task, period, block)
|
||||
- ``task_data_submissions`` — append-only submission log feeding the merge
|
||||
|
||||
JSON payloads are stored as LONGTEXT-backed JSON strings (PortableLongText),
|
||||
mirroring the sibling roundtable persistence modules for headroom.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
class TaskAnalysisTaskRow(Base):
|
||||
__tablename__ = "task_analysis_tasks"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# 对方系统任务号(创建接口传入;唯一)。
|
||||
source_task_id: Mapped[str] = mapped_column(String(128), nullable=False, unique=True, index=True)
|
||||
name: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
# 任务简介:创建时落库的存量即执行时用的版本(不回拉对方接口)。
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
|
||||
class TaskAnalysisBindingRow(Base):
|
||||
__tablename__ = "task_analysis_bindings"
|
||||
__table_args__ = (UniqueConstraint("external_task_id", name="uq_task_analysis_binding_task"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# 引用 roundtable_chains.id;执行前校验,链缺失/停用 → 跳过并标记 chain_missing。
|
||||
chain_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# 执行编排快照 JSON:{seats, stages, stage_goals, coordinator_prompt}。
|
||||
# 每次执行前刷新(跟随链最新版),历史 run 用自己的 stages_snapshot 回放。
|
||||
chain_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
cron_expr: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
timezone: Mapped[str] = mapped_column(String(64), nullable=False, default="Asia/Shanghai")
|
||||
# 执行方式:cron = 按频率定时;once = 只执行一次(创建即执行,next_run_at 恒空)。
|
||||
schedule_type: Mapped[str] = mapped_column(String(16), nullable=False, default="cron", server_default="cron")
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="1")
|
||||
next_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
|
||||
# 绑定级总控提示(与链内 coordinator_prompt 拼接使用)。
|
||||
coordinator_prompt: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# 问答沉淀的「执行方向备注」JSON 数组(逐条追加,注入后续每次定时执行)。
|
||||
direction_notes: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
last_run_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
# 执行模型(席位 run 的 model_name;NULL = 系统/席位默认)。
|
||||
model: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
|
||||
class TaskAnalysisRunRow(Base):
|
||||
__tablename__ = "task_analysis_runs"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
binding_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# schedule = cron 定时执行;manual = 总控问答(对方详情页发起)。
|
||||
trigger: Mapped[str] = mapped_column(String(16), nullable=False, default="schedule")
|
||||
# 幂等键:同 binding + scheduled_for 只建一条(对齐 scheduled_tasks.create_run_once)。
|
||||
scheduled_for: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False)
|
||||
started_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
# pending | running | succeeded | failed | partial
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
|
||||
# 本次编排快照 JSON:[[seat_id,...],...](回放不受链后续修改影响)。
|
||||
stages_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
stage_goals_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
seat_names_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# 各席位结果 JSON:{agent_id: {state, duration_ms, thread_id, error}}。
|
||||
seat_results: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
|
||||
class TaskAnalysisRunMessageRow(Base):
|
||||
__tablename__ = "task_analysis_run_messages"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# 消息归属席位(agent_id);总控消息用固定标识 __coordinator__。
|
||||
seat_agent_id: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
stage_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# input | llm.ai | tool.result | artifact(对齐现有 run messages 事件分类)。
|
||||
event_type: Mapped[str] = mapped_column(String(32), nullable=False, default="llm.ai")
|
||||
content: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
|
||||
|
||||
class TaskAnalysisContentVersionRow(Base):
|
||||
__tablename__ = "task_analysis_content_versions"
|
||||
__table_args__ = (UniqueConstraint("run_id", name="uq_task_analysis_version_run"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
external_task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# 一次执行 = 一期时间线版本(1:1)。
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# JSON:{key: {name, seat_agent_id, content, format}} —— 对方详情的各块内容。
|
||||
blocks: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
|
||||
|
||||
class TaskDataBlockDefRow(Base):
|
||||
"""数据块定义:块 key 定死 block_1~block_6,标题默认「数据块一~六」可改。"""
|
||||
|
||||
__tablename__ = "task_data_block_defs"
|
||||
__table_args__ = (UniqueConstraint("task_id", "block_key", name="uq_task_data_block_def_key"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# 关联 task_analysis_tasks.id(内部任务 id)。
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
title: Mapped[str] = mapped_column(String(128), nullable=False, default="")
|
||||
sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
|
||||
|
||||
class TaskDataPeriodRow(Base):
|
||||
"""数据交付时间轴刻度:每个任务 period_index 从 1 递增;run_id 作幂等锚点。"""
|
||||
|
||||
__tablename__ = "task_data_periods"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("task_id", "period_index", name="uq_task_data_period_index"),
|
||||
UniqueConstraint("run_id", name="uq_task_data_period_run"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
opened_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
# cron = 定时执行开期;manual = 手动触发/总控问答开期;external = 首次提交兜底开期。
|
||||
trigger: Mapped[str] = mapped_column(String(16), nullable=False, default="cron")
|
||||
# 触发本期的执行记录(问答/定时执行共用 run 表);提交兜底开的期没有 run。
|
||||
run_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
|
||||
|
||||
class TaskDataBlockRow(Base):
|
||||
"""每期每块的合并结果(由 task_data_submissions 流水按序合并得出)。"""
|
||||
|
||||
__tablename__ = "task_data_blocks"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("task_id", "period_index", "block_key", name="uq_task_data_block_tpb"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# JSON 对象(顶层 key 合并、同 key 后写覆盖)。
|
||||
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
|
||||
|
||||
class TaskDataSubmissionRow(Base):
|
||||
"""入库流水(追加只读):块 payload 的一致性来源。"""
|
||||
|
||||
__tablename__ = "task_data_submissions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
period_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
block_key: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# 本次提交的 JSON 对象片段(合并语义:顶层 key 合并、同 key 后写覆盖)。
|
||||
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
|
||||
# 提交方(席位 agent id / 用户 id,仅审计)。
|
||||
submitted_by: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
# 同 (task, period, block) 内的提交序号,从 0 递增 —— 合并顺序依据。
|
||||
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow)
|
||||
|
||||
|
||||
class TaskAnalysisSettingRow(Base):
|
||||
"""模块级 KV 设置(当前仅默认业务链)。"""
|
||||
|
||||
__tablename__ = "task_analysis_settings"
|
||||
|
||||
key: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
value: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
|
||||
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_utcnow, onupdate=_utcnow)
|
||||
+802
@@ -0,0 +1,802 @@
|
||||
"""SQLAlchemy-backed task-analysis store.
|
||||
|
||||
Mirrors the memory implementation's normalisation: JSON payloads
|
||||
(chain_snapshot / stages / seat_results / blocks / direction_notes) are stored
|
||||
as LONGTEXT JSON strings; datetimes are ISO strings in dict form.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.task_analysis.base import TaskAnalysisStore
|
||||
from deerflow.persistence.task_analysis.model import (
|
||||
TaskAnalysisBindingRow,
|
||||
TaskAnalysisContentVersionRow,
|
||||
TaskAnalysisRunMessageRow,
|
||||
TaskAnalysisRunRow,
|
||||
TaskAnalysisSettingRow,
|
||||
TaskAnalysisTaskRow,
|
||||
TaskDataBlockDefRow,
|
||||
TaskDataBlockRow,
|
||||
TaskDataPeriodRow,
|
||||
TaskDataSubmissionRow,
|
||||
)
|
||||
|
||||
|
||||
def _iso(value: datetime | str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value.isoformat()
|
||||
return str(value)
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(UTC)
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if value is None or isinstance(value, datetime):
|
||||
return value
|
||||
try:
|
||||
return datetime.fromisoformat(str(value))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _dump(value: Any) -> str | None:
|
||||
return None if value is None else json.dumps(value, ensure_ascii=False)
|
||||
|
||||
|
||||
def _load(value: str | None, default: Any) -> Any:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
try:
|
||||
return json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _task_dict(row: TaskAnalysisTaskRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
data["updated_at"] = _iso(row.updated_at)
|
||||
return data
|
||||
|
||||
|
||||
def _binding_dict(row: TaskAnalysisBindingRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["chain_snapshot"] = _load(row.chain_snapshot, {})
|
||||
data["direction_notes"] = _load(row.direction_notes, [])
|
||||
for key in ("next_run_at", "last_run_at", "created_at", "updated_at"):
|
||||
data[key] = _iso(getattr(row, key))
|
||||
return data
|
||||
|
||||
|
||||
def _run_dict(row: TaskAnalysisRunRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["scheduled_for"] = _iso(row.scheduled_for)
|
||||
data["started_at"] = _iso(row.started_at)
|
||||
data["finished_at"] = _iso(row.finished_at)
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
data["updated_at"] = _iso(row.updated_at)
|
||||
for key, default in (
|
||||
("stages_snapshot", []),
|
||||
("stage_goals_snapshot", []),
|
||||
("seat_names_snapshot", {}),
|
||||
("seat_results", {}),
|
||||
):
|
||||
data[key] = _load(getattr(row, key), default)
|
||||
return data
|
||||
|
||||
|
||||
def _message_dict(row: TaskAnalysisRunMessageRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
return data
|
||||
|
||||
|
||||
def _version_dict(row: TaskAnalysisContentVersionRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["blocks"] = _load(row.blocks, {})
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
return data
|
||||
|
||||
|
||||
def _block_def_dict(row: TaskDataBlockDefRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
data["updated_at"] = _iso(row.updated_at)
|
||||
return data
|
||||
|
||||
|
||||
def _period_dict(row: TaskDataPeriodRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["opened_at"] = _iso(row.opened_at)
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
return data
|
||||
|
||||
|
||||
def _task_block_dict(row: TaskDataBlockRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["payload"] = _load(row.payload, {})
|
||||
data["updated_at"] = _iso(row.updated_at)
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
return data
|
||||
|
||||
|
||||
def _submission_dict(row: TaskDataSubmissionRow) -> dict[str, Any]:
|
||||
data = row.to_dict()
|
||||
data["payload"] = _load(row.payload, {})
|
||||
data["created_at"] = _iso(row.created_at)
|
||||
return data
|
||||
|
||||
|
||||
class TaskAnalysisRepository(TaskAnalysisStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# ── tasks ───────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_task(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
source = str(data["source_task_id"])
|
||||
async with self._sf() as session:
|
||||
existing = await session.execute(
|
||||
select(TaskAnalysisTaskRow).where(TaskAnalysisTaskRow.source_task_id == source)
|
||||
)
|
||||
if existing.scalar_one_or_none() is not None:
|
||||
raise ValueError(f"source_task_id already exists: {source}")
|
||||
row = TaskAnalysisTaskRow(
|
||||
id=str(data.get("id") or f"ta-{source}"),
|
||||
source_task_id=source,
|
||||
name=str(data.get("name") or ""),
|
||||
description=data.get("description"),
|
||||
)
|
||||
session.add(row)
|
||||
# 建任务即种 6 条数据块定义(块 key 定死,标题后续可改)。
|
||||
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
|
||||
|
||||
for definition in default_block_defs():
|
||||
session.add(
|
||||
TaskDataBlockDefRow(
|
||||
id=f"tdbd-{row.id}-{definition['block_key']}",
|
||||
task_id=row.id,
|
||||
block_key=definition["block_key"],
|
||||
title=definition["title"],
|
||||
sort_order=definition["sort_order"],
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _task_dict(row)
|
||||
|
||||
async def get_task(self, task_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TaskAnalysisTaskRow, task_id)
|
||||
return _task_dict(row) if row else None
|
||||
|
||||
async def get_task_by_source(self, source_task_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisTaskRow).where(TaskAnalysisTaskRow.source_task_id == source_task_id)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _task_dict(row) if row else None
|
||||
|
||||
async def list_tasks(self) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisTaskRow).order_by(TaskAnalysisTaskRow.created_at.desc())
|
||||
)
|
||||
tasks = [_task_dict(row) for row in result.scalars()]
|
||||
bindings = await session.execute(select(TaskAnalysisBindingRow))
|
||||
by_task: dict[str, dict[str, Any]] = {}
|
||||
for row in bindings.scalars():
|
||||
by_task[row.external_task_id] = _binding_dict(row)
|
||||
for task in tasks:
|
||||
task["binding"] = by_task.get(task["id"])
|
||||
return tasks
|
||||
|
||||
async def delete_task_cascade(self, task_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TaskAnalysisTaskRow, task_id)
|
||||
if row is None:
|
||||
return False
|
||||
run_ids = (
|
||||
(
|
||||
await session.execute(
|
||||
select(TaskAnalysisRunRow.id).where(TaskAnalysisRunRow.external_task_id == task_id)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskAnalysisBindingRow).where(TaskAnalysisBindingRow.external_task_id == task_id)
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskDataPeriodRow).where(TaskDataPeriodRow.task_id == task_id)
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskDataBlockRow).where(TaskDataBlockRow.task_id == task_id)
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskDataSubmissionRow).where(TaskDataSubmissionRow.task_id == task_id)
|
||||
)
|
||||
if run_ids:
|
||||
await session.execute(
|
||||
delete(TaskAnalysisRunMessageRow).where(TaskAnalysisRunMessageRow.run_id.in_(run_ids))
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskAnalysisContentVersionRow).where(
|
||||
TaskAnalysisContentVersionRow.run_id.in_(run_ids)
|
||||
)
|
||||
)
|
||||
await session.execute(
|
||||
delete(TaskAnalysisRunRow).where(TaskAnalysisRunRow.external_task_id == task_id)
|
||||
)
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
# ── bindings ────────────────────────────────────────────────────────────
|
||||
|
||||
async def get_binding(self, external_task_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisBindingRow).where(
|
||||
TaskAnalysisBindingRow.external_task_id == external_task_id
|
||||
)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _binding_dict(row) if row else None
|
||||
|
||||
async def get_binding_by_id(self, binding_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TaskAnalysisBindingRow, binding_id)
|
||||
return _binding_dict(row) if row else None
|
||||
|
||||
async def upsert_binding(self, external_task_id: str, data: dict[str, Any]) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisBindingRow).where(
|
||||
TaskAnalysisBindingRow.external_task_id == external_task_id
|
||||
)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
row = TaskAnalysisBindingRow(
|
||||
id=str(data.get("id") or f"tab-{external_task_id}"),
|
||||
external_task_id=external_task_id,
|
||||
chain_id=str(data["chain_id"]),
|
||||
created_by=data.get("created_by"),
|
||||
)
|
||||
session.add(row)
|
||||
row.chain_id = str(data.get("chain_id") or row.chain_id)
|
||||
row.chain_snapshot = _dump(data.get("chain_snapshot") or {})
|
||||
row.cron_expr = str(data.get("cron_expr") or row.cron_expr or "")
|
||||
row.timezone = str(data.get("timezone") or row.timezone or "Asia/Shanghai")
|
||||
if "schedule_type" in data:
|
||||
row.schedule_type = str(data.get("schedule_type") or "cron")
|
||||
row.model = data.get("model")
|
||||
if "enabled" in data:
|
||||
row.enabled = bool(data["enabled"])
|
||||
if "next_run_at" in data:
|
||||
row.next_run_at = _parse_dt(data.get("next_run_at"))
|
||||
row.coordinator_prompt = data.get("coordinator_prompt")
|
||||
if "direction_notes" in data:
|
||||
row.direction_notes = _dump(list(data["direction_notes"] or []))
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _binding_dict(row)
|
||||
|
||||
async def delete_binding(self, external_task_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
delete(TaskAnalysisBindingRow).where(
|
||||
TaskAnalysisBindingRow.external_task_id == external_task_id
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return bool(result.rowcount)
|
||||
|
||||
async def due_bindings(self, now: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisBindingRow)
|
||||
.where(
|
||||
TaskAnalysisBindingRow.enabled.is_(True),
|
||||
TaskAnalysisBindingRow.next_run_at.is_not(None),
|
||||
TaskAnalysisBindingRow.next_run_at <= now,
|
||||
)
|
||||
.order_by(TaskAnalysisBindingRow.next_run_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_binding_dict(row) for row in result.scalars()]
|
||||
|
||||
async def update_binding(self, binding_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
allowed = {
|
||||
"chain_id", "cron_expr", "timezone", "schedule_type", "enabled", "next_run_at",
|
||||
"last_run_at", "last_status", "coordinator_prompt", "model",
|
||||
}
|
||||
values: dict[str, Any] = {k: v for k, v in patch.items() if k in allowed}
|
||||
if "next_run_at" in values:
|
||||
values["next_run_at"] = _parse_dt(values["next_run_at"])
|
||||
if "last_run_at" in values:
|
||||
values["last_run_at"] = _parse_dt(values["last_run_at"])
|
||||
if "direction_notes" in patch:
|
||||
values["direction_notes"] = _dump(list(patch["direction_notes"] or []))
|
||||
if not values:
|
||||
row = await self._get_binding_row(binding_id)
|
||||
return _binding_dict(row) if row else None
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(TaskAnalysisBindingRow).where(TaskAnalysisBindingRow.id == binding_id).values(**values)
|
||||
)
|
||||
await session.commit()
|
||||
row = await session.get(TaskAnalysisBindingRow, binding_id)
|
||||
return _binding_dict(row) if row else None
|
||||
|
||||
async def _get_binding_row(self, binding_id: str) -> TaskAnalysisBindingRow | None:
|
||||
async with self._sf() as session:
|
||||
return await session.get(TaskAnalysisBindingRow, binding_id)
|
||||
|
||||
async def last_bound_chain_id(self) -> str | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisBindingRow.chain_id)
|
||||
.order_by(TaskAnalysisBindingRow.updated_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none()
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_run_once(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
binding_id = str(data["binding_id"])
|
||||
scheduled_for = _parse_dt(data["scheduled_for"]) or datetime.now(UTC)
|
||||
import uuid as _uuid
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisRunRow).where(
|
||||
TaskAnalysisRunRow.binding_id == binding_id,
|
||||
TaskAnalysisRunRow.scheduled_for == scheduled_for,
|
||||
)
|
||||
)
|
||||
existing = result.scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return _run_dict(existing)
|
||||
row = TaskAnalysisRunRow(
|
||||
id=str(data.get("id") or f"tar-{_uuid.uuid4()}"),
|
||||
binding_id=binding_id,
|
||||
external_task_id=str(data["external_task_id"]),
|
||||
trigger=str(data.get("trigger") or "schedule"),
|
||||
scheduled_for=scheduled_for,
|
||||
status=str(data.get("status") or "pending"),
|
||||
stages_snapshot=_dump(data.get("stages_snapshot") or []),
|
||||
stage_goals_snapshot=_dump(data.get("stage_goals_snapshot") or []),
|
||||
seat_names_snapshot=_dump(data.get("seat_names_snapshot") or {}),
|
||||
seat_results=_dump(data.get("seat_results") or {}),
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _run_dict(row)
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TaskAnalysisRunRow, run_id)
|
||||
return _run_dict(row) if row else None
|
||||
|
||||
async def list_runs(self, external_task_id: str, *, limit: int = 50) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisRunRow)
|
||||
.where(TaskAnalysisRunRow.external_task_id == external_task_id)
|
||||
.order_by(TaskAnalysisRunRow.started_at.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_run_dict(row) for row in result.scalars()]
|
||||
|
||||
async def update_run(self, run_id: str, patch: dict[str, Any]) -> dict[str, Any] | None:
|
||||
allowed = {
|
||||
"trigger", "status", "finished_at", "error", "retry_count",
|
||||
"stages_snapshot", "stage_goals_snapshot", "seat_names_snapshot", "seat_results",
|
||||
}
|
||||
values: dict[str, Any] = {}
|
||||
for key, value in patch.items():
|
||||
if key not in allowed:
|
||||
continue
|
||||
if key == "finished_at":
|
||||
values[key] = _parse_dt(value)
|
||||
elif key in {"stages_snapshot", "stage_goals_snapshot", "seat_names_snapshot", "seat_results"}:
|
||||
values[key] = _dump(value)
|
||||
else:
|
||||
values[key] = value
|
||||
if not values:
|
||||
row = await self._get_run_row(run_id)
|
||||
return _run_dict(row) if row else None
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
update(TaskAnalysisRunRow).where(TaskAnalysisRunRow.id == run_id).values(**values)
|
||||
)
|
||||
await session.commit()
|
||||
row = await session.get(TaskAnalysisRunRow, run_id)
|
||||
return _run_dict(row) if row else None
|
||||
|
||||
async def _get_run_row(self, run_id: str) -> TaskAnalysisRunRow | None:
|
||||
async with self._sf() as session:
|
||||
return await session.get(TaskAnalysisRunRow, run_id)
|
||||
|
||||
async def stuck_runs(self, older_than: datetime, *, limit: int = 20) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisRunRow)
|
||||
.where(
|
||||
TaskAnalysisRunRow.status.in_(("pending", "running")),
|
||||
TaskAnalysisRunRow.started_at < older_than,
|
||||
)
|
||||
.order_by(TaskAnalysisRunRow.started_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_run_dict(row) for row in result.scalars()]
|
||||
|
||||
# ── messages ────────────────────────────────────────────────────────────
|
||||
|
||||
async def append_run_messages(self, run_id: str, messages: list[dict[str, Any]]) -> int:
|
||||
import uuid as _uuid
|
||||
|
||||
async with self._sf() as session:
|
||||
existing = (
|
||||
(
|
||||
await session.execute(
|
||||
select(TaskAnalysisRunMessageRow.id).where(
|
||||
TaskAnalysisRunMessageRow.run_id == run_id
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
base_seq = len(existing)
|
||||
rows = [
|
||||
TaskAnalysisRunMessageRow(
|
||||
id=str(_uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
seq=base_seq + index,
|
||||
seat_agent_id=str(message.get("seat_agent_id") or ""),
|
||||
stage_index=int(message.get("stage_index") or 0),
|
||||
event_type=str(message.get("event_type") or "llm.ai"),
|
||||
content=message.get("content"),
|
||||
)
|
||||
for index, message in enumerate(messages)
|
||||
]
|
||||
session.add_all(rows)
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
|
||||
async def list_run_messages(
|
||||
self, run_id: str, *, seat: str | None = None, limit: int = 500
|
||||
) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
stmt = (
|
||||
select(TaskAnalysisRunMessageRow)
|
||||
.where(TaskAnalysisRunMessageRow.run_id == run_id)
|
||||
.order_by(TaskAnalysisRunMessageRow.seq.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
if seat is not None:
|
||||
stmt = stmt.where(TaskAnalysisRunMessageRow.seat_agent_id == seat)
|
||||
result = await session.execute(stmt)
|
||||
return [_message_dict(row) for row in result.scalars()]
|
||||
|
||||
# ── versions ────────────────────────────────────────────────────────────
|
||||
|
||||
async def create_version(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
import uuid as _uuid
|
||||
|
||||
run_id = str(data["run_id"])
|
||||
async with self._sf() as session:
|
||||
await session.execute(
|
||||
delete(TaskAnalysisContentVersionRow).where(
|
||||
TaskAnalysisContentVersionRow.run_id == run_id
|
||||
)
|
||||
)
|
||||
row = TaskAnalysisContentVersionRow(
|
||||
id=str(_uuid.uuid4()),
|
||||
external_task_id=str(data["external_task_id"]),
|
||||
run_id=run_id,
|
||||
blocks=_dump(data.get("blocks") or {}),
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _version_dict(row)
|
||||
|
||||
async def list_versions(
|
||||
self, external_task_id: str, *, limit: int = 50, offset: int = 0
|
||||
) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskAnalysisContentVersionRow)
|
||||
.where(TaskAnalysisContentVersionRow.external_task_id == external_task_id)
|
||||
.order_by(TaskAnalysisContentVersionRow.created_at.desc())
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
return [_version_dict(row) for row in result.scalars()]
|
||||
|
||||
# ── data delivery: block defs / periods / submissions ────────────────────
|
||||
|
||||
async def ensure_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
from deerflow.persistence.task_analysis.data_blocks import default_block_defs
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
|
||||
)
|
||||
existing_keys = {row.block_key for row in result.scalars()}
|
||||
missing = [
|
||||
definition
|
||||
for definition in default_block_defs()
|
||||
if definition["block_key"] not in existing_keys
|
||||
]
|
||||
if not missing:
|
||||
return await self._ordered_defs(session, task_id)
|
||||
for definition in missing:
|
||||
session.add(
|
||||
TaskDataBlockDefRow(
|
||||
id=f"tdbd-{task_id}-{definition['block_key']}",
|
||||
task_id=task_id,
|
||||
block_key=definition["block_key"],
|
||||
title=definition["title"],
|
||||
sort_order=definition["sort_order"],
|
||||
)
|
||||
)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# 并发补种:另一个请求先种好了,沿用其结果。
|
||||
await session.rollback()
|
||||
return await self._ordered_defs(session, task_id)
|
||||
|
||||
@staticmethod
|
||||
async def _ordered_defs(session: AsyncSession, task_id: str) -> list[dict[str, Any]]:
|
||||
result = await session.execute(
|
||||
select(TaskDataBlockDefRow)
|
||||
.where(TaskDataBlockDefRow.task_id == task_id)
|
||||
.order_by(TaskDataBlockDefRow.sort_order.asc(), TaskDataBlockDefRow.block_key.asc())
|
||||
)
|
||||
return [_block_def_dict(row) for row in result.scalars()]
|
||||
|
||||
async def list_block_defs(self, task_id: str) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
return await self._ordered_defs(session, task_id)
|
||||
|
||||
async def update_block_defs(self, task_id: str, updates: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataBlockDefRow).where(TaskDataBlockDefRow.task_id == task_id)
|
||||
)
|
||||
rows = {row.block_key: row for row in result.scalars()}
|
||||
touched = False
|
||||
for item in updates:
|
||||
block_key = str(item.get("block_key") or "")
|
||||
row = rows.get(block_key)
|
||||
if row is None:
|
||||
raise ValueError(f"未知数据块:{block_key}")
|
||||
patch: dict[str, Any] = {}
|
||||
if item.get("title") is not None:
|
||||
patch["title"] = str(item["title"])
|
||||
if item.get("sort_order") is not None:
|
||||
patch["sort_order"] = int(item["sort_order"])
|
||||
if patch:
|
||||
await session.execute(
|
||||
update(TaskDataBlockDefRow)
|
||||
.where(TaskDataBlockDefRow.id == row.id)
|
||||
.values(**patch)
|
||||
)
|
||||
touched = True
|
||||
if touched:
|
||||
await session.commit()
|
||||
return await self._ordered_defs(session, task_id)
|
||||
|
||||
async def open_period(
|
||||
self,
|
||||
task_id: str,
|
||||
*,
|
||||
source: str,
|
||||
run_id: str | None = None,
|
||||
opened_at: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
import uuid as _uuid
|
||||
|
||||
for _attempt in range(3):
|
||||
async with self._sf() as session:
|
||||
if run_id:
|
||||
existing = await session.execute(
|
||||
select(TaskDataPeriodRow).where(TaskDataPeriodRow.run_id == run_id)
|
||||
)
|
||||
row = existing.scalar_one_or_none()
|
||||
if row is not None:
|
||||
return _period_dict(row)
|
||||
max_index = (
|
||||
await session.execute(
|
||||
select(func.max(TaskDataPeriodRow.period_index)).where(
|
||||
TaskDataPeriodRow.task_id == task_id
|
||||
)
|
||||
)
|
||||
).scalar()
|
||||
row = TaskDataPeriodRow(
|
||||
id=f"tdp-{_uuid.uuid4()}",
|
||||
task_id=task_id,
|
||||
period_index=(max_index or 0) + 1,
|
||||
opened_at=opened_at or _utcnow(),
|
||||
trigger=str(source or "cron"),
|
||||
run_id=run_id,
|
||||
)
|
||||
session.add(row)
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
# 并发开期撞了期号唯一键(或 run_id 幂等锚点)——重查/重试。
|
||||
await session.rollback()
|
||||
continue
|
||||
await session.refresh(row)
|
||||
return _period_dict(row)
|
||||
raise RuntimeError(f"open_period 连续冲突,task={task_id} source={source}")
|
||||
|
||||
async def get_period(self, task_id: str, period_index: int) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataPeriodRow).where(
|
||||
TaskDataPeriodRow.task_id == task_id,
|
||||
TaskDataPeriodRow.period_index == period_index,
|
||||
)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _period_dict(row) if row else None
|
||||
|
||||
async def current_period(self, task_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataPeriodRow)
|
||||
.where(TaskDataPeriodRow.task_id == task_id)
|
||||
.order_by(TaskDataPeriodRow.period_index.desc())
|
||||
.limit(1)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
return _period_dict(row) if row else None
|
||||
|
||||
async def list_periods(self, task_id: str, *, limit: int = 500) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataPeriodRow)
|
||||
.where(TaskDataPeriodRow.task_id == task_id)
|
||||
.order_by(TaskDataPeriodRow.period_index.desc())
|
||||
.limit(limit)
|
||||
)
|
||||
rows = [_period_dict(row) for row in result.scalars()]
|
||||
rows.reverse() # 时间轴语义:按期号升序返回
|
||||
return rows
|
||||
|
||||
async def append_submission(
|
||||
self,
|
||||
task_id: str,
|
||||
period_index: int,
|
||||
block_key: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
submitted_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
import uuid as _uuid
|
||||
|
||||
async with self._sf() as session:
|
||||
base_seq = (
|
||||
(
|
||||
await session.execute(
|
||||
select(func.count(TaskDataSubmissionRow.id)).where(
|
||||
TaskDataSubmissionRow.task_id == task_id,
|
||||
TaskDataSubmissionRow.period_index == period_index,
|
||||
TaskDataSubmissionRow.block_key == block_key,
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalar_one()
|
||||
)
|
||||
row = TaskDataSubmissionRow(
|
||||
id=f"tds-{_uuid.uuid4()}",
|
||||
task_id=task_id,
|
||||
period_index=period_index,
|
||||
block_key=block_key,
|
||||
payload=_dump(payload),
|
||||
submitted_by=submitted_by,
|
||||
seq=int(base_seq),
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _submission_dict(row)
|
||||
|
||||
async def list_submissions(
|
||||
self, task_id: str, period_index: int, block_key: str, *, limit: int = 500
|
||||
) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataSubmissionRow)
|
||||
.where(
|
||||
TaskDataSubmissionRow.task_id == task_id,
|
||||
TaskDataSubmissionRow.period_index == period_index,
|
||||
TaskDataSubmissionRow.block_key == block_key,
|
||||
)
|
||||
.order_by(TaskDataSubmissionRow.seq.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_submission_dict(row) for row in result.scalars()]
|
||||
|
||||
async def upsert_period_block(
|
||||
self, task_id: str, period_index: int, block_key: str, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
import uuid as _uuid
|
||||
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataBlockRow).where(
|
||||
TaskDataBlockRow.task_id == task_id,
|
||||
TaskDataBlockRow.period_index == period_index,
|
||||
TaskDataBlockRow.block_key == block_key,
|
||||
)
|
||||
)
|
||||
row = result.scalar_one_or_none()
|
||||
if row is None:
|
||||
row = TaskDataBlockRow(
|
||||
id=f"tdb-{_uuid.uuid4()}",
|
||||
task_id=task_id,
|
||||
period_index=period_index,
|
||||
block_key=block_key,
|
||||
payload=_dump(payload),
|
||||
)
|
||||
session.add(row)
|
||||
else:
|
||||
row.payload = _dump(payload)
|
||||
row.updated_at = _utcnow()
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _task_block_dict(row)
|
||||
|
||||
async def list_task_blocks(self, task_id: str, *, limit: int = 2000) -> list[dict[str, Any]]:
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
select(TaskDataBlockRow)
|
||||
.where(TaskDataBlockRow.task_id == task_id)
|
||||
.order_by(TaskDataBlockRow.period_index.asc(), TaskDataBlockRow.block_key.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
return [_task_block_dict(row) for row in result.scalars()]
|
||||
|
||||
# ── settings ────────────────────────────────────────────────────────────
|
||||
|
||||
async def get_setting(self, key: str) -> str | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TaskAnalysisSettingRow, key)
|
||||
return row.value if row else None
|
||||
|
||||
async def set_setting(self, key: str, value: str) -> None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(TaskAnalysisSettingRow, key)
|
||||
if row is None:
|
||||
session.add(TaskAnalysisSettingRow(key=key, value=value))
|
||||
else:
|
||||
row.value = value
|
||||
await session.commit()
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"""Dedicated taskId + goPath conversation-history index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langgraph.store.base import BaseStore
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore
|
||||
|
||||
__all__ = ["make_task_deeplink_thread_store"]
|
||||
|
||||
|
||||
def make_task_deeplink_thread_store(
|
||||
session_factory: async_sessionmaker[AsyncSession] | None,
|
||||
langgraph_store: BaseStore,
|
||||
thread_store: ThreadMetaStore,
|
||||
):
|
||||
# Keep imports local: ThreadMetaStore's in-memory implementation imports
|
||||
# the sync helpers from this package, so eager imports here would form a
|
||||
# package-initialization cycle.
|
||||
if session_factory is not None:
|
||||
from deerflow.persistence.task_deeplink_threads.sql import (
|
||||
TaskDeeplinkThreadRepository,
|
||||
)
|
||||
|
||||
return TaskDeeplinkThreadRepository(session_factory)
|
||||
from deerflow.persistence.task_deeplink_threads.memory import (
|
||||
MemoryTaskDeeplinkThreadStore,
|
||||
)
|
||||
|
||||
return MemoryTaskDeeplinkThreadStore(langgraph_store, thread_store)
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
"""Shared helpers for task deep-link conversation indexing.
|
||||
|
||||
The LangGraph thread/checkpoint remains the source of truth for messages and
|
||||
execution state. This package only owns a small, dedicated lookup index so the
|
||||
task workspaces never have to search/filter the global ``threads_meta`` table.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, TypedDict
|
||||
|
||||
TASK_DEEPLINK_GO_PATHS = frozenset({"3qfx", "rwfx", "xdfx"})
|
||||
|
||||
|
||||
class TaskDeeplinkBinding(TypedDict):
|
||||
task_id: str
|
||||
source_task_id: str
|
||||
go_path: str
|
||||
agent_id: str | None
|
||||
|
||||
|
||||
def task_deeplink_binding(metadata: dict[str, Any] | None) -> TaskDeeplinkBinding | None:
|
||||
"""Extract a valid dedicated-history binding from thread metadata.
|
||||
|
||||
``taskRecordId`` is the original deep-link ``?taskId=``. It equals the
|
||||
actual task id for 3qfx/rwfx and the action id for xdfx. Older callers may
|
||||
omit it, in which case ``taskId`` is a safe fallback. A ``taskGoPath`` is
|
||||
mandatory, deliberately preventing legacy task rows from being migrated or
|
||||
accidentally mixed into the new store.
|
||||
"""
|
||||
|
||||
md = metadata or {}
|
||||
go_path = str(md.get("taskGoPath") or "").strip().lower()
|
||||
if go_path not in TASK_DEEPLINK_GO_PATHS:
|
||||
return None
|
||||
source_task_id = str(md.get("taskId") or "").strip()
|
||||
task_id = str(md.get("taskRecordId") or source_task_id).strip()
|
||||
if not task_id or not source_task_id:
|
||||
return None
|
||||
agent_id = str(md.get("agent_id") or "").strip() or None
|
||||
return {
|
||||
"task_id": task_id,
|
||||
"source_task_id": source_task_id,
|
||||
"go_path": go_path,
|
||||
"agent_id": agent_id,
|
||||
}
|
||||
+86
@@ -0,0 +1,86 @@
|
||||
"""In-memory task deep-link index backed by LangGraph BaseStore."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from langgraph.store.base import BaseStore
|
||||
|
||||
from deerflow.persistence.task_deeplink_threads.base import task_deeplink_binding
|
||||
from deerflow.utils.time import now_iso
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore
|
||||
|
||||
|
||||
TASK_DEEPLINK_THREADS_NS: tuple[str, ...] = ("task_deeplink_threads",)
|
||||
|
||||
|
||||
async def sync_task_deeplink_index(store: BaseStore, thread: dict[str, Any]) -> None:
|
||||
metadata = thread.get("metadata") or {}
|
||||
binding = task_deeplink_binding(metadata)
|
||||
thread_id = str(thread.get("thread_id") or "")
|
||||
if not thread_id:
|
||||
return
|
||||
if binding is None:
|
||||
if "taskGoPath" in metadata or "taskRecordId" in metadata:
|
||||
await store.adelete(TASK_DEEPLINK_THREADS_NS, thread_id)
|
||||
return
|
||||
await store.aput(
|
||||
TASK_DEEPLINK_THREADS_NS,
|
||||
thread_id,
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
**binding,
|
||||
"created_by": thread.get("user_id"),
|
||||
"created_at": thread.get("created_at") or now_iso(),
|
||||
"updated_at": thread.get("updated_at") or now_iso(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def delete_task_deeplink_index(store: BaseStore, thread_id: str) -> None:
|
||||
await store.adelete(TASK_DEEPLINK_THREADS_NS, thread_id)
|
||||
|
||||
|
||||
class MemoryTaskDeeplinkThreadStore:
|
||||
def __init__(self, store: BaseStore, thread_store: ThreadMetaStore) -> None:
|
||||
self._store = store
|
||||
self._thread_store = thread_store
|
||||
|
||||
async def list_threads(
|
||||
self,
|
||||
task_id: str,
|
||||
go_path: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
) -> list[dict[str, Any]]:
|
||||
items = await self._store.asearch(
|
||||
TASK_DEEPLINK_THREADS_NS,
|
||||
filter={"task_id": task_id, "go_path": go_path},
|
||||
limit=limit,
|
||||
)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
thread_id = str(item.value.get("thread_id") or "")
|
||||
if not thread_id:
|
||||
continue
|
||||
thread = await self._thread_store.get(thread_id, user_id=None)
|
||||
if thread is None:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"thread_id": thread_id,
|
||||
"status": thread.get("status", "idle"),
|
||||
"created_at": thread.get("created_at", ""),
|
||||
"updated_at": thread.get("updated_at", ""),
|
||||
"metadata": thread.get("metadata") or {},
|
||||
"values": ({"title": thread.get("display_name")} if thread.get("display_name") else {}),
|
||||
"interrupts": {},
|
||||
}
|
||||
)
|
||||
rows.sort(
|
||||
key=lambda row: (str(row.get("updated_at") or ""), row["thread_id"]),
|
||||
reverse=True,
|
||||
)
|
||||
return rows[:limit]
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
"""Dedicated lookup rows for single-agent task deep-link conversations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Index, String
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class TaskDeeplinkThreadRow(Base):
|
||||
"""One task-workspace history entry pointing at one LangGraph thread.
|
||||
|
||||
The row intentionally stores no messages or title. Those remain in the
|
||||
canonical checkpoint / ``threads_meta`` record and are joined by primary
|
||||
key when listing a task's history.
|
||||
"""
|
||||
|
||||
__tablename__ = "task_deeplink_threads"
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_task_deeplink_threads_scope",
|
||||
"task_id",
|
||||
"go_path",
|
||||
"updated_at",
|
||||
"thread_id",
|
||||
),
|
||||
)
|
||||
|
||||
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
# Original URL ?taskId=. For xdfx this is the action id by product contract.
|
||||
task_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
# Resolved real task id used by model/run context (equals task_id except xdfx).
|
||||
source_task_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
go_path: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
# Audit only. Task deep-link histories remain shared across authenticated users.
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
nullable=False,
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
"""SQL implementation of the dedicated task deep-link conversation index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.task_deeplink_threads.base import task_deeplink_binding
|
||||
from deerflow.persistence.task_deeplink_threads.model import TaskDeeplinkThreadRow
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
|
||||
|
||||
async def sync_task_deeplink_index(session: AsyncSession, thread: ThreadMetaRow) -> None:
|
||||
"""Synchronize one index row inside the caller's thread-meta transaction."""
|
||||
|
||||
metadata = thread.metadata_json or {}
|
||||
binding = task_deeplink_binding(metadata)
|
||||
if binding is None:
|
||||
# Ordinary chats take no extra query. Only an explicitly cleared/invalid
|
||||
# deep-link marker can require removal of a previously registered row.
|
||||
if "taskGoPath" in metadata or "taskRecordId" in metadata:
|
||||
await session.execute(delete(TaskDeeplinkThreadRow).where(TaskDeeplinkThreadRow.thread_id == thread.thread_id))
|
||||
return
|
||||
|
||||
existing = await session.get(TaskDeeplinkThreadRow, thread.thread_id)
|
||||
now = thread.updated_at if isinstance(thread.updated_at, datetime) else datetime.now(UTC)
|
||||
if existing is None:
|
||||
session.add(
|
||||
TaskDeeplinkThreadRow(
|
||||
thread_id=thread.thread_id,
|
||||
task_id=binding["task_id"],
|
||||
source_task_id=binding["source_task_id"],
|
||||
go_path=binding["go_path"],
|
||||
agent_id=binding["agent_id"],
|
||||
created_by=thread.user_id,
|
||||
created_at=(thread.created_at if isinstance(thread.created_at, datetime) else now),
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
existing.task_id = binding["task_id"]
|
||||
existing.source_task_id = binding["source_task_id"]
|
||||
existing.go_path = binding["go_path"]
|
||||
existing.agent_id = binding["agent_id"]
|
||||
existing.updated_at = now
|
||||
|
||||
|
||||
async def delete_task_deeplink_index(session: AsyncSession, thread_id: str) -> None:
|
||||
await session.execute(delete(TaskDeeplinkThreadRow).where(TaskDeeplinkThreadRow.thread_id == thread_id))
|
||||
|
||||
|
||||
class TaskDeeplinkThreadRepository:
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_threads(
|
||||
self,
|
||||
task_id: str,
|
||||
go_path: str,
|
||||
*,
|
||||
limit: int = 100,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""List one task/goPath partition without scanning thread metadata JSON."""
|
||||
|
||||
# Local import avoids a package-initialization cycle: thread_meta.sql
|
||||
# imports the sync helpers above for atomic dual writes.
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
|
||||
stmt = (
|
||||
select(TaskDeeplinkThreadRow, ThreadMetaRow)
|
||||
.join(
|
||||
ThreadMetaRow,
|
||||
ThreadMetaRow.thread_id == TaskDeeplinkThreadRow.thread_id,
|
||||
)
|
||||
.where(
|
||||
TaskDeeplinkThreadRow.task_id == task_id,
|
||||
TaskDeeplinkThreadRow.go_path == go_path,
|
||||
)
|
||||
.order_by(
|
||||
ThreadMetaRow.updated_at.desc(),
|
||||
ThreadMetaRow.thread_id.desc(),
|
||||
)
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
|
||||
return [
|
||||
{
|
||||
"thread_id": thread.thread_id,
|
||||
"status": thread.status,
|
||||
"created_at": thread.created_at,
|
||||
"updated_at": thread.updated_at,
|
||||
"metadata": thread.metadata_json or {},
|
||||
"values": ({"title": thread.display_name} if thread.display_name else {}),
|
||||
"interrupts": {},
|
||||
}
|
||||
for _index, thread in rows
|
||||
]
|
||||
+31
-1
@@ -30,19 +30,49 @@ EXCLUDED_SYSTEM_THREAD_TYPES = frozenset(
|
||||
"deep_research",
|
||||
"writing_setup",
|
||||
"report_structure_setup",
|
||||
"task-analysis",
|
||||
"task-analysis-qa",
|
||||
"task_deeplink",
|
||||
}
|
||||
)
|
||||
|
||||
# 这些系统线程 id 只存在于编排内部,不进「最近的对话」;loopback 派活必须能跨
|
||||
# 「点击触发的 HTTP 用户」和「绑定 created_by 签发的 JWT」读写,否则会 404 not accessible。
|
||||
SHARED_ACCESS_THREAD_TYPES = frozenset(
|
||||
{
|
||||
"roundtable",
|
||||
"task-analysis",
|
||||
"task-analysis-qa",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def is_excluded_system_thread(metadata: dict[str, Any] | None) -> bool:
|
||||
"""Return True when a thread should be hidden from user conversation lists."""
|
||||
"""Return True when a thread should be hidden from user conversation lists.
|
||||
|
||||
Task-analysis seat runs created before ``thread_type`` was stamped still
|
||||
carry ``task_analysis_seat``. Those rows stay in the execution record and
|
||||
out of recent chats.
|
||||
"""
|
||||
md = metadata or {}
|
||||
if md.get("system"):
|
||||
return True
|
||||
seat = md.get("task_analysis_seat")
|
||||
if isinstance(seat, str) and seat.strip():
|
||||
return True
|
||||
thread_type = md.get("thread_type")
|
||||
return isinstance(thread_type, str) and thread_type in EXCLUDED_SYSTEM_THREAD_TYPES
|
||||
|
||||
|
||||
def is_shared_access_thread(metadata: dict[str, Any] | None) -> bool:
|
||||
"""Return True when any authenticated caller may read/write this thread."""
|
||||
md = metadata or {}
|
||||
if md.get("taskId"):
|
||||
return True
|
||||
thread_type = md.get("thread_type")
|
||||
return isinstance(thread_type, str) and thread_type in SHARED_ACCESS_THREAD_TYPES
|
||||
|
||||
|
||||
class ThreadMetaStore(abc.ABC):
|
||||
@abc.abstractmethod
|
||||
async def create(
|
||||
|
||||
+14
-3
@@ -11,7 +11,15 @@ from typing import Any
|
||||
|
||||
from langgraph.store.base import BaseStore
|
||||
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread
|
||||
from deerflow.persistence.task_deeplink_threads.memory import (
|
||||
delete_task_deeplink_index,
|
||||
sync_task_deeplink_index,
|
||||
)
|
||||
from deerflow.persistence.thread_meta.base import (
|
||||
ThreadMetaStore,
|
||||
is_excluded_system_thread,
|
||||
is_shared_access_thread,
|
||||
)
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
from deerflow.utils.time import coerce_iso, now_iso
|
||||
|
||||
@@ -64,6 +72,7 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
"updated_at": now,
|
||||
}
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
await sync_task_deeplink_index(self._store, record)
|
||||
return record
|
||||
|
||||
async def get(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> dict | None:
|
||||
@@ -183,8 +192,8 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
item = await self._store.aget(THREADS_NS, thread_id)
|
||||
if item is None:
|
||||
return not require_existing
|
||||
# 任务深链对话(metadata.taskId)全局共享、不区分用户:任何登录用户都可读写/续聊。
|
||||
if (item.value.get("metadata") or {}).get("taskId"):
|
||||
# 任务深链 / 圆桌 / 任务分析系统线程:全局共享(与 SQL store 同口径)。
|
||||
if is_shared_access_thread(item.value.get("metadata") or {}):
|
||||
return True
|
||||
record_user_id = item.value.get("user_id")
|
||||
if record_user_id is None:
|
||||
@@ -224,11 +233,13 @@ class MemoryThreadMetaStore(ThreadMetaStore):
|
||||
record["metadata"] = merged
|
||||
record["updated_at"] = now_iso()
|
||||
await self._store.aput(THREADS_NS, thread_id, record)
|
||||
await sync_task_deeplink_index(self._store, record)
|
||||
|
||||
async def delete(self, thread_id: str, *, user_id: str | None | _AutoSentinel = AUTO) -> None:
|
||||
record = await self._get_owned_record(thread_id, user_id, "MemoryThreadMetaStore.delete")
|
||||
if record is None:
|
||||
return
|
||||
await delete_task_deeplink_index(self._store, thread_id)
|
||||
await self._store.adelete(THREADS_NS, thread_id)
|
||||
|
||||
@staticmethod
|
||||
|
||||
+18
-3
@@ -16,12 +16,27 @@ class ThreadMetaRow(Base):
|
||||
__table_args__ = (
|
||||
Index("ix_threads_meta_created_user", "created_at", "user_id"),
|
||||
Index("ix_threads_meta_user_created", "user_id", "created_at"),
|
||||
# Task deep-link history is globally shared and therefore cannot use
|
||||
# the normal ``user_id`` scope. Keep its high-cardinality lookup
|
||||
# fields out of ``metadata_json`` so task entry never has to scan and
|
||||
# Python-filter every thread in the installation.
|
||||
Index("ix_threads_meta_task_agent_updated", "task_id", "agent_id", "updated_at", "thread_id"),
|
||||
Index("ix_threads_meta_task_action_agent_updated", "task_id", "action_id", "agent_id", "updated_at", "thread_id"),
|
||||
# The ordinary recent-chat path has an owner filter and the same
|
||||
# updated-at ordering. This prevents a full table sort there too.
|
||||
Index("ix_threads_meta_user_updated", "user_id", "updated_at", "thread_id"),
|
||||
)
|
||||
|
||||
thread_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
assistant_id: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
display_name: Mapped[str | None] = mapped_column(String(256))
|
||||
assistant_id: Mapped[str | None] = mapped_column(String(128), index=True)
|
||||
user_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
# Indexed projections of the corresponding metadata values. The JSON
|
||||
# blob remains authoritative and supports arbitrary metadata filters;
|
||||
# these columns exist solely for the task-workspace hot path.
|
||||
task_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
action_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
display_name: Mapped[str | None] = mapped_column(String(256))
|
||||
status: Mapped[str] = mapped_column(String(20), default="idle")
|
||||
metadata_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
+108
-32
@@ -8,12 +8,69 @@ from typing import Any
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread
|
||||
from deerflow.persistence.task_deeplink_threads.sql import (
|
||||
delete_task_deeplink_index,
|
||||
sync_task_deeplink_index,
|
||||
)
|
||||
from deerflow.persistence.thread_meta.base import ThreadMetaStore, is_excluded_system_thread, is_shared_access_thread
|
||||
from deerflow.persistence.thread_meta.model import ThreadMetaRow
|
||||
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
|
||||
|
||||
|
||||
class ThreadMetaRepository(ThreadMetaStore):
|
||||
_INDEXED_METADATA_COLUMNS = {
|
||||
"taskId": ThreadMetaRow.task_id,
|
||||
"agent_id": ThreadMetaRow.agent_id,
|
||||
"actionId": ThreadMetaRow.action_id,
|
||||
}
|
||||
|
||||
_INDEXED_METADATA_ATTRIBUTES = {
|
||||
"taskId": "task_id",
|
||||
"agent_id": "agent_id",
|
||||
"actionId": "action_id",
|
||||
}
|
||||
|
||||
|
||||
def _indexed_metadata_values(metadata: dict[str, Any] | None) -> dict[str, str]:
|
||||
"""Return metadata predicates that have a maintained SQL projection.
|
||||
|
||||
Task deep-link callers use string ids. Restrict the SQL shortcut to
|
||||
strings so legacy callers that intentionally store another JSON type keep
|
||||
the exact Python-side equality semantics of the generic metadata filter.
|
||||
"""
|
||||
return {
|
||||
key: value
|
||||
for key, value in (metadata or {}).items()
|
||||
if key in _INDEXED_METADATA_COLUMNS and isinstance(value, str)
|
||||
}
|
||||
|
||||
|
||||
def _indexed_thread_metadata(metadata: dict[str, Any] | None) -> dict[str, str | None]:
|
||||
"""Build ORM column values from authoritative thread metadata.
|
||||
|
||||
Always include every projection so a metadata update that changes a value
|
||||
from a string to another JSON type cannot leave a stale indexed value.
|
||||
"""
|
||||
values = _indexed_metadata_values(metadata)
|
||||
return {
|
||||
attribute: values.get(metadata_key)
|
||||
for metadata_key, attribute in _INDEXED_METADATA_ATTRIBUTES.items()
|
||||
}
|
||||
|
||||
|
||||
def _apply_indexed_metadata_filters(stmt: Any, metadata: dict[str, Any] | None) -> tuple[Any, set[str]]:
|
||||
"""Apply known metadata predicates in SQL and return their keys.
|
||||
|
||||
The caller still verifies the full metadata object in Python whenever a
|
||||
generic filter or system-thread exclusion is involved. This keeps the
|
||||
public API's exact-match contract while reducing that verification set to
|
||||
rows for the requested task/agent/action instead of the entire table.
|
||||
"""
|
||||
values = _indexed_metadata_values(metadata)
|
||||
for key, value in values.items():
|
||||
stmt = stmt.where(_INDEXED_METADATA_COLUMNS[key] == value)
|
||||
return stmt, set(values)
|
||||
|
||||
|
||||
class ThreadMetaRepository(ThreadMetaStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
@@ -40,18 +97,20 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
# creates an orphan row (used by migration scripts).
|
||||
resolved_user_id = resolve_user_id(user_id, method_name="ThreadMetaRepository.create")
|
||||
now = datetime.now(UTC)
|
||||
row = ThreadMetaRow(
|
||||
thread_id=thread_id,
|
||||
assistant_id=assistant_id,
|
||||
user_id=resolved_user_id,
|
||||
display_name=display_name,
|
||||
status="idle",
|
||||
metadata_json=metadata or {},
|
||||
row = ThreadMetaRow(
|
||||
thread_id=thread_id,
|
||||
assistant_id=assistant_id,
|
||||
user_id=resolved_user_id,
|
||||
**_indexed_thread_metadata(metadata),
|
||||
display_name=display_name,
|
||||
status="idle",
|
||||
metadata_json=metadata or {},
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await sync_task_deeplink_index(session, row)
|
||||
await session.commit()
|
||||
# Deliberately no session.refresh(): every column above is assigned
|
||||
# explicitly and the session factory uses expire_on_commit=False, so
|
||||
@@ -108,10 +167,10 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
md = row.metadata_json or {}
|
||||
# 全局共享、不区分用户(任何登录用户都可读写/续聊):
|
||||
# ① 任务深链对话(metadata.taskId);
|
||||
# ② 圆桌会商系统线程(metadata.thread_type=="roundtable")——其 Step2/Step3
|
||||
# 沙箱产物(md/流程图/报告)需让历史草稿或其他账号打开同一会商时也能查看,
|
||||
# 与任务深链对话一致(线程 id 仅存在于草稿中,不会被无关账号发现)。
|
||||
if md.get("taskId") or md.get("thread_type") == "roundtable":
|
||||
# ② 圆桌 / 任务分析等系统编排线程(thread_type 见 SHARED_ACCESS_THREAD_TYPES)
|
||||
# —— loopback 派活签发的 JWT 可能与建 thread 时 HTTP 上下文用户不是同一人,
|
||||
# 若按属主拦截会把刚建好的总控 thread 打成 404 not accessible。
|
||||
if is_shared_access_thread(md):
|
||||
return True
|
||||
if row.user_id is None:
|
||||
return True
|
||||
@@ -144,16 +203,24 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
stmt = stmt.where(ThreadMetaRow.user_id == resolved_user_id)
|
||||
if status:
|
||||
stmt = stmt.where(ThreadMetaRow.status == status)
|
||||
if query and query.strip():
|
||||
if query and query.strip():
|
||||
# Case-insensitive title search. display_name is a real column, so
|
||||
# this filter runs in SQL even when the JSON filters below fall
|
||||
# back to Python-side scanning.
|
||||
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
|
||||
if not metadata and not exclude_system:
|
||||
stmt = stmt.limit(limit).offset(offset)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
stmt = stmt.where(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
|
||||
# ``metadata_json`` is deliberately portable, so arbitrary filters
|
||||
# still receive Python-side equality checks below. These three task
|
||||
# identifiers are maintained projections and can safely narrow the
|
||||
# SQL result first. A deep-link query that used to scan every thread
|
||||
# now reads only that task's rows.
|
||||
stmt, indexed_metadata_keys = _apply_indexed_metadata_filters(stmt, metadata)
|
||||
remaining_metadata_keys = set(metadata or {}) - indexed_metadata_keys
|
||||
|
||||
if not remaining_metadata_keys and not exclude_system:
|
||||
stmt = stmt.limit(limit).offset(offset)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [self._row_to_dict(r) for r in result.scalars()]
|
||||
|
||||
# Metadata filters live inside the JSON blob, so rows are scanned in
|
||||
@@ -205,12 +272,17 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
conditions.append(ThreadMetaRow.user_id == resolved_user_id)
|
||||
if status:
|
||||
conditions.append(ThreadMetaRow.status == status)
|
||||
if query and query.strip():
|
||||
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
|
||||
if not metadata and not exclude_system:
|
||||
stmt = select(func.count()).select_from(ThreadMetaRow)
|
||||
for condition in conditions:
|
||||
if query and query.strip():
|
||||
conditions.append(func.lower(ThreadMetaRow.display_name).contains(query.strip().lower()))
|
||||
|
||||
indexed_metadata = _indexed_metadata_values(metadata)
|
||||
for key, value in indexed_metadata.items():
|
||||
conditions.append(_INDEXED_METADATA_COLUMNS[key] == value)
|
||||
remaining_metadata_keys = set(metadata or {}) - set(indexed_metadata)
|
||||
|
||||
if not remaining_metadata_keys and not exclude_system:
|
||||
stmt = select(func.count()).select_from(ThreadMetaRow)
|
||||
for condition in conditions:
|
||||
stmt = stmt.where(condition)
|
||||
async with self._sf() as session:
|
||||
return int((await session.execute(stmt)).scalar_one())
|
||||
@@ -318,11 +390,14 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
task_shared = bool((row.metadata_json or {}).get("taskId"))
|
||||
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
|
||||
return
|
||||
merged = dict(row.metadata_json or {})
|
||||
merged.update(metadata)
|
||||
row.metadata_json = merged
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
merged = dict(row.metadata_json or {})
|
||||
merged.update(metadata)
|
||||
row.metadata_json = merged
|
||||
for key, value in _indexed_thread_metadata(merged).items():
|
||||
setattr(row, key, value)
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await sync_task_deeplink_index(session, row)
|
||||
await session.commit()
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
@@ -339,5 +414,6 @@ class ThreadMetaRepository(ThreadMetaStore):
|
||||
task_shared = bool((row.metadata_json or {}).get("taskId"))
|
||||
if resolved_user_id is not None and not task_shared and row.user_id != resolved_user_id:
|
||||
return
|
||||
await delete_task_deeplink_index(session, thread_id)
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
@@ -43,7 +43,17 @@ class PortableJSON(TypeDecorator):
|
||||
return value
|
||||
if not value:
|
||||
return None
|
||||
return json.loads(value)
|
||||
try:
|
||||
return json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
from deerflow.persistence.gauss_mysql_compat import (
|
||||
decode_gauss_mysql_json,
|
||||
is_gauss_mysql_dialect,
|
||||
)
|
||||
|
||||
if not is_gauss_mysql_dialect(dialect):
|
||||
raise
|
||||
return decode_gauss_mysql_json(value)
|
||||
|
||||
|
||||
class PortableLongText(TypeDecorator):
|
||||
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Global database-backed upload policy persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.upload_policy.base import UploadPolicyStore
|
||||
from deerflow.persistence.upload_policy.memory import MemoryUploadPolicyStore
|
||||
from deerflow.persistence.upload_policy.model import UploadPolicyRow
|
||||
from deerflow.persistence.upload_policy.sql import UploadPolicyRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryUploadPolicyStore",
|
||||
"UploadPolicyRepository",
|
||||
"UploadPolicyRow",
|
||||
"UploadPolicyStore",
|
||||
"make_upload_policy_store",
|
||||
]
|
||||
|
||||
|
||||
def make_upload_policy_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryUploadPolicyStore()
|
||||
return UploadPolicyRepository(session_factory)
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
"""Persistence contract for the global upload policy document."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class UploadPolicyStore(ABC):
|
||||
@abstractmethod
|
||||
async def ensure_seed(self, policy: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Create the default row if missing without overwriting saved edits."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_policy(self) -> dict[str, Any] | None:
|
||||
"""Return the stored policy, or ``None`` before the first save."""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_policy(self, policy: dict[str, Any], *, updated_by: str | None = None) -> dict[str, Any]:
|
||||
"""Atomically replace the global policy document."""
|
||||
|
||||
raise NotImplementedError
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
"""In-memory upload-policy store used without a configured SQL database."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.upload_policy.base import UploadPolicyStore
|
||||
|
||||
|
||||
class MemoryUploadPolicyStore(UploadPolicyStore):
|
||||
def __init__(self) -> None:
|
||||
self._policy: dict[str, Any] | None = None
|
||||
self._updated_by: str | None = None
|
||||
self._updated_at: str | None = None
|
||||
self._revision = 0
|
||||
|
||||
async def ensure_seed(self, policy: dict[str, Any]) -> dict[str, Any]:
|
||||
if self._policy is None:
|
||||
return await self.set_policy(policy, updated_by=None)
|
||||
return (await self.get_policy()) or {}
|
||||
|
||||
async def get_policy(self) -> dict[str, Any] | None:
|
||||
if self._policy is None:
|
||||
return None
|
||||
return {
|
||||
"policy": deepcopy(self._policy),
|
||||
"updated_by": self._updated_by,
|
||||
"updated_at": self._updated_at,
|
||||
"revision": self._revision,
|
||||
}
|
||||
|
||||
async def set_policy(self, policy: dict[str, Any], *, updated_by: str | None = None) -> dict[str, Any]:
|
||||
self._policy = deepcopy(policy)
|
||||
self._updated_by = updated_by
|
||||
self._updated_at = datetime.now(UTC).isoformat()
|
||||
self._revision += 1
|
||||
return (await self.get_policy()) or {}
|
||||
Loaded 100 of 1651 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user