sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -0,0 +1,26 @@
"""Persisted read index for the generated-file library."""
from __future__ import annotations
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
from deerflow.persistence.artifact_library.memory import MemoryArtifactLibraryStore
from deerflow.persistence.artifact_library.model import (
ArtifactLibraryItemRow,
ArtifactLibrarySyncStateRow,
)
from deerflow.persistence.artifact_library.sql import SqlArtifactLibraryStore
__all__ = [
"ArtifactLibraryItemRow",
"ArtifactLibraryStore",
"ArtifactLibrarySyncStateRow",
"MemoryArtifactLibraryStore",
"SqlArtifactLibraryStore",
"make_artifact_library_store",
]
def make_artifact_library_store(session_factory):
if session_factory is None:
return MemoryArtifactLibraryStore()
return SqlArtifactLibraryStore(session_factory)
@@ -0,0 +1,61 @@
"""Storage contract for the generated-file library index."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class ArtifactLibraryStore(ABC):
"""A user-scoped, replace-on-sync artifact metadata index."""
@abstractmethod
async def claim_sync(
self,
user_id: str,
*,
force: bool = False,
stale_after_seconds: float = 15.0,
lease_seconds: float = 300.0,
) -> str | None:
"""Claim one background refresh and return its generation token.
``None`` means another worker owns a live lease or the persisted index
is still fresh enough. Implementations must make this operation safe
when multiple Gateway workers receive the same request concurrently.
"""
raise NotImplementedError
@abstractmethod
async def finish_sync(
self,
user_id: str,
generation: str,
items: list[dict[str, Any]],
) -> bool:
"""Atomically publish ``items`` when ``generation`` still owns the lease."""
raise NotImplementedError
@abstractmethod
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
"""Record a failed refresh when ``generation`` still owns the lease."""
raise NotImplementedError
@abstractmethod
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
"""Return the persisted refresh state for one user."""
raise NotImplementedError
@abstractmethod
async def list_items(
self,
user_id: str,
*,
query: str | None = None,
file_kind: str = "all",
source_type: str = "all",
offset: int = 0,
limit: int = 50,
) -> tuple[int, list[dict[str, Any]]]:
"""Return an indexed, newest-first page and its filtered total."""
raise NotImplementedError
@@ -0,0 +1,158 @@
"""In-memory artifact-library index used by memory-mode deployments and tests."""
from __future__ import annotations
import asyncio
import hashlib
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _artifact_id(user_id: str, item: dict[str, Any]) -> str:
identity = "\0".join((user_id, str(item.get("thread_id") or ""), str(item.get("path") or "")))
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
class MemoryArtifactLibraryStore(ArtifactLibraryStore):
def __init__(self) -> None:
self._items: dict[str, dict[str, Any]] = {}
self._states: dict[str, dict[str, Any]] = {}
self._lock = asyncio.Lock()
async def claim_sync(
self,
user_id: str,
*,
force: bool = False,
stale_after_seconds: float = 15.0,
lease_seconds: float = 300.0,
) -> str | None:
now = datetime.now(UTC)
async with self._lock:
state = self._states.get(user_id)
if state is not None:
lease = state.get("lease_expires_at")
if state.get("status") == "syncing" and isinstance(lease, datetime) and lease > now:
return None
freshness_anchor = state.get("completed_at")
if state.get("status") == "error":
freshness_anchor = state.get("started_at")
if not force and isinstance(freshness_anchor, datetime) and freshness_anchor + timedelta(seconds=stale_after_seconds) > now:
return None
generation = str(uuid4())
self._states[user_id] = {
"status": "syncing",
"generation": generation,
"started_at": now,
"lease_expires_at": now + timedelta(seconds=lease_seconds),
"completed_at": state.get("completed_at") if state else None,
"last_error": None,
}
return generation
async def finish_sync(
self,
user_id: str,
generation: str,
items: list[dict[str, Any]],
) -> bool:
now = datetime.now(UTC)
async with self._lock:
state = self._states.get(user_id)
if state is None or state.get("generation") != generation or state.get("status") != "syncing":
return False
self._items = {key: value for key, value in self._items.items() if value.get("user_id") != user_id}
for item in items:
row = dict(item)
row["artifact_id"] = _artifact_id(user_id, item)
row["user_id"] = user_id
row["sync_generation"] = generation
row["indexed_at"] = now.isoformat()
self._items[row["artifact_id"]] = row
state.update(
status="ready",
completed_at=now,
lease_expires_at=None,
last_error=None,
)
return True
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
async with self._lock:
state = self._states.get(user_id)
if state is None or state.get("generation") != generation or state.get("status") != "syncing":
return False
state.update(
status="error",
lease_expires_at=None,
last_error=(error or "Artifact library refresh failed")[:1024],
)
return True
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
async with self._lock:
state = self._states.get(user_id)
if state is None:
return {
"status": "idle",
"generation": None,
"started_at": None,
"completed_at": None,
"last_error": None,
}
return {
"status": str(state.get("status") or "idle"),
"generation": state.get("generation"),
"started_at": _iso(state.get("started_at")),
"completed_at": _iso(state.get("completed_at")),
"last_error": state.get("last_error"),
}
async def list_items(
self,
user_id: str,
*,
query: str | None = None,
file_kind: str = "all",
source_type: str = "all",
offset: int = 0,
limit: int = 50,
) -> tuple[int, list[dict[str, Any]]]:
normalized = (query or "").strip().casefold()
async with self._lock:
rows = [dict(row) for row in self._items.values() if row.get("user_id") == user_id]
def matches(row: dict[str, Any]) -> bool:
if file_kind != "all" and row.get("file_kind") != file_kind:
return False
if source_type != "all" and row.get("source_type") != source_type:
return False
if not normalized:
return True
return any(
normalized in str(row.get(field) or "").casefold()
for field in (
"name",
"thread_title",
"agent_name",
"strategy_name",
"strategy_project_id",
)
)
filtered = [row for row in rows if matches(row)]
filtered.sort(
key=lambda row: (
str(row.get("modified_at") or ""),
str(row.get("artifact_id") or ""),
),
reverse=True,
)
return len(filtered), filtered[offset : offset + limit]
@@ -0,0 +1,78 @@
"""ORM rows for the generated-file library read index."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import BigInteger, Boolean, Index, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime
class ArtifactLibraryItemRow(Base):
"""One generated file projected into the fast library listing."""
__tablename__ = "artifact_library_items"
__table_args__ = (
Index(
"ix_artifact_library_user_modified",
"user_id",
"modified_at",
"artifact_id",
),
Index(
"ix_artifact_library_user_kind",
"user_id",
"file_kind",
"modified_at",
"artifact_id",
),
Index(
"ix_artifact_library_user_source",
"user_id",
"source_type",
"modified_at",
"artifact_id",
),
Index("ix_artifact_library_user_generation", "user_id", "sync_generation"),
)
# A deterministic SHA-256 of (user_id, thread_id, virtual path) avoids a
# long composite primary key that would exceed older MySQL index limits.
artifact_id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str] = mapped_column(String(64), nullable=False)
thread_id: Mapped[str] = mapped_column(String(128), nullable=False)
thread_title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
agent_name: Mapped[str | None] = mapped_column(String(512), nullable=True)
thread_type: Mapped[str | None] = mapped_column(String(64), nullable=True)
source_type: Mapped[str] = mapped_column(String(32), nullable=False)
strategy_project_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
strategy_name: Mapped[str | None] = mapped_column(String(512), nullable=True)
strategy_stale: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
strategy_source_kind: Mapped[str | None] = mapped_column(String(32), nullable=True)
download_url: Mapped[str | None] = mapped_column(Text, nullable=True)
path: Mapped[str] = mapped_column(Text, nullable=False)
name: Mapped[str] = mapped_column(String(512), nullable=False)
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
mime_type: Mapped[str | None] = mapped_column(String(255), nullable=True)
modified_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False)
file_kind: Mapped[str] = mapped_column(String(32), nullable=False)
sync_generation: Mapped[str] = mapped_column(String(36), nullable=False)
indexed_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
class ArtifactLibrarySyncStateRow(Base):
"""Per-user lease and last-published generation metadata."""
__tablename__ = "artifact_library_sync_states"
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
status: Mapped[str] = mapped_column(String(16), nullable=False, default="idle")
generation: Mapped[str | None] = mapped_column(String(36), nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error: Mapped[str | None] = mapped_column(String(1024), nullable=True)
@@ -0,0 +1,346 @@
"""SQLAlchemy-backed generated-file library index."""
from __future__ import annotations
import hashlib
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import and_, delete, func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
from deerflow.persistence.artifact_library.model import (
ArtifactLibraryItemRow,
ArtifactLibrarySyncStateRow,
)
_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
def _artifact_id(user_id: str, item: dict[str, Any]) -> str:
identity = "\0".join((user_id, str(item.get("thread_id") or ""), str(item.get("path") or "")))
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
def _parse_datetime(value: object) -> datetime:
if isinstance(value, datetime):
parsed = value
else:
text = str(value or "").strip()
if not text:
return _EPOCH
try:
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError:
return _EPOCH
if parsed.tzinfo is None:
return parsed.replace(tzinfo=UTC)
return parsed.astimezone(UTC)
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _state_dict(row: ArtifactLibrarySyncStateRow | None) -> dict[str, Any]:
if row is None:
return {
"status": "idle",
"generation": None,
"started_at": None,
"completed_at": None,
"last_error": None,
}
return {
"status": row.status,
"generation": row.generation,
"started_at": _iso(row.started_at),
"completed_at": _iso(row.completed_at),
"last_error": row.last_error,
}
def _item_dict(row: ArtifactLibraryItemRow) -> dict[str, Any]:
return {
"thread_id": row.thread_id,
"thread_title": row.thread_title,
"agent_id": row.agent_id,
"agent_name": row.agent_name,
"thread_type": row.thread_type,
"source_type": row.source_type,
"strategy_project_id": row.strategy_project_id,
"strategy_name": row.strategy_name,
"strategy_stale": row.strategy_stale,
"strategy_source_kind": row.strategy_source_kind,
"download_url": row.download_url,
"path": row.path,
"name": row.name,
"size_bytes": row.size_bytes,
"mime_type": row.mime_type,
"modified_at": row.modified_at.isoformat(),
"file_kind": row.file_kind,
}
def _apply_item(
row: ArtifactLibraryItemRow,
*,
user_id: str,
generation: str,
item: dict[str, Any],
indexed_at: datetime,
) -> None:
row.user_id = user_id
row.thread_id = str(item.get("thread_id") or "")[:128]
row.thread_title = str(item.get("thread_title") or "")[:512]
row.agent_id = str(item.get("agent_id"))[:128] if item.get("agent_id") else None
row.agent_name = str(item.get("agent_name"))[:512] if item.get("agent_name") else None
row.thread_type = str(item.get("thread_type"))[:64] if item.get("thread_type") else None
row.source_type = str(item.get("source_type") or "general_qa")[:32]
row.strategy_project_id = str(item.get("strategy_project_id"))[:128] if item.get("strategy_project_id") else None
row.strategy_name = str(item.get("strategy_name"))[:512] if item.get("strategy_name") else None
row.strategy_stale = item.get("strategy_stale") is True
row.strategy_source_kind = str(item.get("strategy_source_kind"))[:32] if item.get("strategy_source_kind") else None
row.download_url = str(item.get("download_url")) if item.get("download_url") else None
row.path = str(item.get("path") or "")
row.name = str(item.get("name") or "")[:512]
row.size_bytes = max(0, int(item.get("size_bytes") or 0))
row.mime_type = str(item.get("mime_type"))[:255] if item.get("mime_type") else None
row.modified_at = _parse_datetime(item.get("modified_at"))
row.file_kind = str(item.get("file_kind") or "other")[:32]
row.sync_generation = generation
row.indexed_at = indexed_at
class SqlArtifactLibraryStore(ArtifactLibraryStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def claim_sync(
self,
user_id: str,
*,
force: bool = False,
stale_after_seconds: float = 15.0,
lease_seconds: float = 300.0,
) -> str | None:
now = datetime.now(UTC)
generation = str(uuid4())
try:
async with self._sf() as session:
async with session.begin():
claim_filters = [
ArtifactLibrarySyncStateRow.user_id == user_id,
# An expired (or legacy missing) lease may be reclaimed.
# A live lease remains exclusive even for force=True.
or_(
ArtifactLibrarySyncStateRow.status != "syncing",
ArtifactLibrarySyncStateRow.lease_expires_at.is_(None),
ArtifactLibrarySyncStateRow.lease_expires_at <= now,
),
]
if not force:
stale_before = now - timedelta(seconds=stale_after_seconds)
claim_filters.append(
or_(
and_(
ArtifactLibrarySyncStateRow.status == "error",
or_(
ArtifactLibrarySyncStateRow.started_at.is_(None),
ArtifactLibrarySyncStateRow.started_at <= stale_before,
),
),
and_(
ArtifactLibrarySyncStateRow.status != "error",
or_(
ArtifactLibrarySyncStateRow.completed_at.is_(None),
ArtifactLibrarySyncStateRow.completed_at <= stale_before,
),
),
)
)
# A conditional UPDATE is the portable compare-and-swap.
# SQLite ignores SELECT ... FOR UPDATE, whereas this write
# serializes claimers there and also works on MySQL,
# PostgreSQL, and openGauss.
result = await session.execute(
update(ArtifactLibrarySyncStateRow)
.where(*claim_filters)
.values(
status="syncing",
generation=generation,
started_at=now,
lease_expires_at=now + timedelta(seconds=lease_seconds),
last_error=None,
)
)
if result.rowcount == 1:
return generation
row = await session.get(ArtifactLibrarySyncStateRow, user_id)
if row is not None:
return None
session.add(
ArtifactLibrarySyncStateRow(
user_id=user_id,
status="syncing",
generation=generation,
started_at=now,
lease_expires_at=now + timedelta(seconds=lease_seconds),
last_error=None,
)
)
return generation
except IntegrityError:
# Two workers can race while inserting a user's first state row.
# The winner owns the lease; the loser simply serves the same
# persisted index without starting a duplicate filesystem scan.
async with self._sf() as session:
if await session.get(ArtifactLibrarySyncStateRow, user_id) is not None:
return None
raise
async def finish_sync(
self,
user_id: str,
generation: str,
items: list[dict[str, Any]],
) -> bool:
now = datetime.now(UTC)
deduped = {_artifact_id(user_id, item): item for item in items}
async with self._sf() as session:
async with session.begin():
# Write first so SQLite also serializes this publisher against
# a lease takeover. Requiring status=syncing makes completion
# one-shot even if the same callback is accidentally invoked
# twice with an old generation token.
fence = await session.execute(
update(ArtifactLibrarySyncStateRow)
.where(
ArtifactLibrarySyncStateRow.user_id == user_id,
ArtifactLibrarySyncStateRow.generation == generation,
ArtifactLibrarySyncStateRow.status == "syncing",
)
.values(lease_expires_at=now + timedelta(seconds=300))
)
if fence.rowcount != 1:
return False
existing_result = await session.execute(select(ArtifactLibraryItemRow).where(ArtifactLibraryItemRow.user_id == user_id))
existing = {row.artifact_id: row for row in existing_result.scalars()}
for artifact_id, item in deduped.items():
row = existing.get(artifact_id)
if row is None:
row = ArtifactLibraryItemRow(
artifact_id=artifact_id,
user_id=user_id,
thread_id="",
thread_title="",
source_type="general_qa",
path="",
name="",
size_bytes=0,
modified_at=_EPOCH,
file_kind="other",
sync_generation=generation,
indexed_at=now,
)
session.add(row)
_apply_item(
row,
user_id=user_id,
generation=generation,
item=item,
indexed_at=now,
)
await session.flush()
await session.execute(
delete(ArtifactLibraryItemRow).where(
ArtifactLibraryItemRow.user_id == user_id,
ArtifactLibraryItemRow.sync_generation != generation,
)
)
await session.execute(
update(ArtifactLibrarySyncStateRow)
.where(
ArtifactLibrarySyncStateRow.user_id == user_id,
ArtifactLibrarySyncStateRow.generation == generation,
ArtifactLibrarySyncStateRow.status == "syncing",
)
.values(
status="ready",
completed_at=now,
lease_expires_at=None,
last_error=None,
)
)
return True
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
async with self._sf() as session:
async with session.begin():
result = await session.execute(
update(ArtifactLibrarySyncStateRow)
.where(
ArtifactLibrarySyncStateRow.user_id == user_id,
ArtifactLibrarySyncStateRow.generation == generation,
ArtifactLibrarySyncStateRow.status == "syncing",
)
.values(
status="error",
lease_expires_at=None,
last_error=(error or "Artifact library refresh failed")[:1024],
)
)
return result.rowcount == 1
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
async with self._sf() as session:
row = await session.get(ArtifactLibrarySyncStateRow, user_id)
return _state_dict(row)
async def list_items(
self,
user_id: str,
*,
query: str | None = None,
file_kind: str = "all",
source_type: str = "all",
offset: int = 0,
limit: int = 50,
) -> tuple[int, list[dict[str, Any]]]:
filters = [ArtifactLibraryItemRow.user_id == user_id]
if file_kind != "all":
filters.append(ArtifactLibraryItemRow.file_kind == file_kind)
if source_type != "all":
filters.append(ArtifactLibraryItemRow.source_type == source_type)
normalized = (query or "").strip().casefold()
if normalized:
searchable = (
ArtifactLibraryItemRow.name,
ArtifactLibraryItemRow.thread_title,
ArtifactLibraryItemRow.agent_name,
ArtifactLibraryItemRow.strategy_name,
ArtifactLibraryItemRow.strategy_project_id,
)
filters.append(or_(*(func.lower(func.coalesce(column, "")).contains(normalized, autoescape=True) for column in searchable)))
count_stmt = select(func.count()).select_from(ArtifactLibraryItemRow).where(*filters)
page_stmt = (
select(ArtifactLibraryItemRow)
.where(*filters)
.order_by(
ArtifactLibraryItemRow.modified_at.desc(),
ArtifactLibraryItemRow.artifact_id.desc(),
)
.offset(offset)
.limit(limit)
)
async with self._sf() as session:
total = int((await session.execute(count_stmt)).scalar_one())
rows = (await session.execute(page_stmt)).scalars().all()
return total, [_item_dict(row) for row in rows]