sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+26
@@ -0,0 +1,26 @@
|
||||
"""Persisted read index for the generated-file library."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
|
||||
from deerflow.persistence.artifact_library.memory import MemoryArtifactLibraryStore
|
||||
from deerflow.persistence.artifact_library.model import (
|
||||
ArtifactLibraryItemRow,
|
||||
ArtifactLibrarySyncStateRow,
|
||||
)
|
||||
from deerflow.persistence.artifact_library.sql import SqlArtifactLibraryStore
|
||||
|
||||
__all__ = [
|
||||
"ArtifactLibraryItemRow",
|
||||
"ArtifactLibraryStore",
|
||||
"ArtifactLibrarySyncStateRow",
|
||||
"MemoryArtifactLibraryStore",
|
||||
"SqlArtifactLibraryStore",
|
||||
"make_artifact_library_store",
|
||||
]
|
||||
|
||||
|
||||
def make_artifact_library_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryArtifactLibraryStore()
|
||||
return SqlArtifactLibraryStore(session_factory)
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
"""Storage contract for the generated-file library index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class ArtifactLibraryStore(ABC):
|
||||
"""A user-scoped, replace-on-sync artifact metadata index."""
|
||||
|
||||
@abstractmethod
|
||||
async def claim_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
force: bool = False,
|
||||
stale_after_seconds: float = 15.0,
|
||||
lease_seconds: float = 300.0,
|
||||
) -> str | None:
|
||||
"""Claim one background refresh and return its generation token.
|
||||
|
||||
``None`` means another worker owns a live lease or the persisted index
|
||||
is still fresh enough. Implementations must make this operation safe
|
||||
when multiple Gateway workers receive the same request concurrently.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def finish_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
generation: str,
|
||||
items: list[dict[str, Any]],
|
||||
) -> bool:
|
||||
"""Atomically publish ``items`` when ``generation`` still owns the lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
|
||||
"""Record a failed refresh when ``generation`` still owns the lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
|
||||
"""Return the persisted refresh state for one user."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_items(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
query: str | None = None,
|
||||
file_kind: str = "all",
|
||||
source_type: str = "all",
|
||||
offset: int = 0,
|
||||
limit: int = 50,
|
||||
) -> tuple[int, list[dict[str, Any]]]:
|
||||
"""Return an indexed, newest-first page and its filtered total."""
|
||||
raise NotImplementedError
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
"""In-memory artifact-library index used by memory-mode deployments and tests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _artifact_id(user_id: str, item: dict[str, Any]) -> str:
|
||||
identity = "\0".join((user_id, str(item.get("thread_id") or ""), str(item.get("path") or "")))
|
||||
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class MemoryArtifactLibraryStore(ArtifactLibraryStore):
|
||||
def __init__(self) -> None:
|
||||
self._items: dict[str, dict[str, Any]] = {}
|
||||
self._states: dict[str, dict[str, Any]] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def claim_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
force: bool = False,
|
||||
stale_after_seconds: float = 15.0,
|
||||
lease_seconds: float = 300.0,
|
||||
) -> str | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._lock:
|
||||
state = self._states.get(user_id)
|
||||
if state is not None:
|
||||
lease = state.get("lease_expires_at")
|
||||
if state.get("status") == "syncing" and isinstance(lease, datetime) and lease > now:
|
||||
return None
|
||||
freshness_anchor = state.get("completed_at")
|
||||
if state.get("status") == "error":
|
||||
freshness_anchor = state.get("started_at")
|
||||
if not force and isinstance(freshness_anchor, datetime) and freshness_anchor + timedelta(seconds=stale_after_seconds) > now:
|
||||
return None
|
||||
generation = str(uuid4())
|
||||
self._states[user_id] = {
|
||||
"status": "syncing",
|
||||
"generation": generation,
|
||||
"started_at": now,
|
||||
"lease_expires_at": now + timedelta(seconds=lease_seconds),
|
||||
"completed_at": state.get("completed_at") if state else None,
|
||||
"last_error": None,
|
||||
}
|
||||
return generation
|
||||
|
||||
async def finish_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
generation: str,
|
||||
items: list[dict[str, Any]],
|
||||
) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
async with self._lock:
|
||||
state = self._states.get(user_id)
|
||||
if state is None or state.get("generation") != generation or state.get("status") != "syncing":
|
||||
return False
|
||||
self._items = {key: value for key, value in self._items.items() if value.get("user_id") != user_id}
|
||||
for item in items:
|
||||
row = dict(item)
|
||||
row["artifact_id"] = _artifact_id(user_id, item)
|
||||
row["user_id"] = user_id
|
||||
row["sync_generation"] = generation
|
||||
row["indexed_at"] = now.isoformat()
|
||||
self._items[row["artifact_id"]] = row
|
||||
state.update(
|
||||
status="ready",
|
||||
completed_at=now,
|
||||
lease_expires_at=None,
|
||||
last_error=None,
|
||||
)
|
||||
return True
|
||||
|
||||
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
|
||||
async with self._lock:
|
||||
state = self._states.get(user_id)
|
||||
if state is None or state.get("generation") != generation or state.get("status") != "syncing":
|
||||
return False
|
||||
state.update(
|
||||
status="error",
|
||||
lease_expires_at=None,
|
||||
last_error=(error or "Artifact library refresh failed")[:1024],
|
||||
)
|
||||
return True
|
||||
|
||||
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
|
||||
async with self._lock:
|
||||
state = self._states.get(user_id)
|
||||
if state is None:
|
||||
return {
|
||||
"status": "idle",
|
||||
"generation": None,
|
||||
"started_at": None,
|
||||
"completed_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
return {
|
||||
"status": str(state.get("status") or "idle"),
|
||||
"generation": state.get("generation"),
|
||||
"started_at": _iso(state.get("started_at")),
|
||||
"completed_at": _iso(state.get("completed_at")),
|
||||
"last_error": state.get("last_error"),
|
||||
}
|
||||
|
||||
async def list_items(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
query: str | None = None,
|
||||
file_kind: str = "all",
|
||||
source_type: str = "all",
|
||||
offset: int = 0,
|
||||
limit: int = 50,
|
||||
) -> tuple[int, list[dict[str, Any]]]:
|
||||
normalized = (query or "").strip().casefold()
|
||||
async with self._lock:
|
||||
rows = [dict(row) for row in self._items.values() if row.get("user_id") == user_id]
|
||||
|
||||
def matches(row: dict[str, Any]) -> bool:
|
||||
if file_kind != "all" and row.get("file_kind") != file_kind:
|
||||
return False
|
||||
if source_type != "all" and row.get("source_type") != source_type:
|
||||
return False
|
||||
if not normalized:
|
||||
return True
|
||||
return any(
|
||||
normalized in str(row.get(field) or "").casefold()
|
||||
for field in (
|
||||
"name",
|
||||
"thread_title",
|
||||
"agent_name",
|
||||
"strategy_name",
|
||||
"strategy_project_id",
|
||||
)
|
||||
)
|
||||
|
||||
filtered = [row for row in rows if matches(row)]
|
||||
filtered.sort(
|
||||
key=lambda row: (
|
||||
str(row.get("modified_at") or ""),
|
||||
str(row.get("artifact_id") or ""),
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return len(filtered), filtered[offset : offset + limit]
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
"""ORM rows for the generated-file library read index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, Index, String, Text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime
|
||||
|
||||
|
||||
class ArtifactLibraryItemRow(Base):
|
||||
"""One generated file projected into the fast library listing."""
|
||||
|
||||
__tablename__ = "artifact_library_items"
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"ix_artifact_library_user_modified",
|
||||
"user_id",
|
||||
"modified_at",
|
||||
"artifact_id",
|
||||
),
|
||||
Index(
|
||||
"ix_artifact_library_user_kind",
|
||||
"user_id",
|
||||
"file_kind",
|
||||
"modified_at",
|
||||
"artifact_id",
|
||||
),
|
||||
Index(
|
||||
"ix_artifact_library_user_source",
|
||||
"user_id",
|
||||
"source_type",
|
||||
"modified_at",
|
||||
"artifact_id",
|
||||
),
|
||||
Index("ix_artifact_library_user_generation", "user_id", "sync_generation"),
|
||||
)
|
||||
|
||||
# A deterministic SHA-256 of (user_id, thread_id, virtual path) avoids a
|
||||
# long composite primary key that would exceed older MySQL index limits.
|
||||
artifact_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
thread_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
thread_title: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
agent_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
agent_name: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
thread_type: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
source_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
strategy_project_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
strategy_name: Mapped[str | None] = mapped_column(String(512), nullable=True)
|
||||
strategy_stale: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
strategy_source_kind: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||
download_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
path: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
mime_type: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
modified_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False)
|
||||
file_kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
sync_generation: Mapped[str] = mapped_column(String(36), nullable=False)
|
||||
indexed_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class ArtifactLibrarySyncStateRow(Base):
|
||||
"""Per-user lease and last-published generation metadata."""
|
||||
|
||||
__tablename__ = "artifact_library_sync_states"
|
||||
|
||||
user_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="idle")
|
||||
generation: Mapped[str | None] = mapped_column(String(36), nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(String(1024), nullable=True)
|
||||
+346
@@ -0,0 +1,346 @@
|
||||
"""SQLAlchemy-backed generated-file library index."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import and_, delete, func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.artifact_library.base import ArtifactLibraryStore
|
||||
from deerflow.persistence.artifact_library.model import (
|
||||
ArtifactLibraryItemRow,
|
||||
ArtifactLibrarySyncStateRow,
|
||||
)
|
||||
|
||||
_EPOCH = datetime(1970, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
def _artifact_id(user_id: str, item: dict[str, Any]) -> str:
|
||||
identity = "\0".join((user_id, str(item.get("thread_id") or ""), str(item.get("path") or "")))
|
||||
return hashlib.sha256(identity.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _parse_datetime(value: object) -> datetime:
|
||||
if isinstance(value, datetime):
|
||||
parsed = value
|
||||
else:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return _EPOCH
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return _EPOCH
|
||||
if parsed.tzinfo is None:
|
||||
return parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _state_dict(row: ArtifactLibrarySyncStateRow | None) -> dict[str, Any]:
|
||||
if row is None:
|
||||
return {
|
||||
"status": "idle",
|
||||
"generation": None,
|
||||
"started_at": None,
|
||||
"completed_at": None,
|
||||
"last_error": None,
|
||||
}
|
||||
return {
|
||||
"status": row.status,
|
||||
"generation": row.generation,
|
||||
"started_at": _iso(row.started_at),
|
||||
"completed_at": _iso(row.completed_at),
|
||||
"last_error": row.last_error,
|
||||
}
|
||||
|
||||
|
||||
def _item_dict(row: ArtifactLibraryItemRow) -> dict[str, Any]:
|
||||
return {
|
||||
"thread_id": row.thread_id,
|
||||
"thread_title": row.thread_title,
|
||||
"agent_id": row.agent_id,
|
||||
"agent_name": row.agent_name,
|
||||
"thread_type": row.thread_type,
|
||||
"source_type": row.source_type,
|
||||
"strategy_project_id": row.strategy_project_id,
|
||||
"strategy_name": row.strategy_name,
|
||||
"strategy_stale": row.strategy_stale,
|
||||
"strategy_source_kind": row.strategy_source_kind,
|
||||
"download_url": row.download_url,
|
||||
"path": row.path,
|
||||
"name": row.name,
|
||||
"size_bytes": row.size_bytes,
|
||||
"mime_type": row.mime_type,
|
||||
"modified_at": row.modified_at.isoformat(),
|
||||
"file_kind": row.file_kind,
|
||||
}
|
||||
|
||||
|
||||
def _apply_item(
|
||||
row: ArtifactLibraryItemRow,
|
||||
*,
|
||||
user_id: str,
|
||||
generation: str,
|
||||
item: dict[str, Any],
|
||||
indexed_at: datetime,
|
||||
) -> None:
|
||||
row.user_id = user_id
|
||||
row.thread_id = str(item.get("thread_id") or "")[:128]
|
||||
row.thread_title = str(item.get("thread_title") or "")[:512]
|
||||
row.agent_id = str(item.get("agent_id"))[:128] if item.get("agent_id") else None
|
||||
row.agent_name = str(item.get("agent_name"))[:512] if item.get("agent_name") else None
|
||||
row.thread_type = str(item.get("thread_type"))[:64] if item.get("thread_type") else None
|
||||
row.source_type = str(item.get("source_type") or "general_qa")[:32]
|
||||
row.strategy_project_id = str(item.get("strategy_project_id"))[:128] if item.get("strategy_project_id") else None
|
||||
row.strategy_name = str(item.get("strategy_name"))[:512] if item.get("strategy_name") else None
|
||||
row.strategy_stale = item.get("strategy_stale") is True
|
||||
row.strategy_source_kind = str(item.get("strategy_source_kind"))[:32] if item.get("strategy_source_kind") else None
|
||||
row.download_url = str(item.get("download_url")) if item.get("download_url") else None
|
||||
row.path = str(item.get("path") or "")
|
||||
row.name = str(item.get("name") or "")[:512]
|
||||
row.size_bytes = max(0, int(item.get("size_bytes") or 0))
|
||||
row.mime_type = str(item.get("mime_type"))[:255] if item.get("mime_type") else None
|
||||
row.modified_at = _parse_datetime(item.get("modified_at"))
|
||||
row.file_kind = str(item.get("file_kind") or "other")[:32]
|
||||
row.sync_generation = generation
|
||||
row.indexed_at = indexed_at
|
||||
|
||||
|
||||
class SqlArtifactLibraryStore(ArtifactLibraryStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def claim_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
force: bool = False,
|
||||
stale_after_seconds: float = 15.0,
|
||||
lease_seconds: float = 300.0,
|
||||
) -> str | None:
|
||||
now = datetime.now(UTC)
|
||||
generation = str(uuid4())
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
async with session.begin():
|
||||
claim_filters = [
|
||||
ArtifactLibrarySyncStateRow.user_id == user_id,
|
||||
# An expired (or legacy missing) lease may be reclaimed.
|
||||
# A live lease remains exclusive even for force=True.
|
||||
or_(
|
||||
ArtifactLibrarySyncStateRow.status != "syncing",
|
||||
ArtifactLibrarySyncStateRow.lease_expires_at.is_(None),
|
||||
ArtifactLibrarySyncStateRow.lease_expires_at <= now,
|
||||
),
|
||||
]
|
||||
if not force:
|
||||
stale_before = now - timedelta(seconds=stale_after_seconds)
|
||||
claim_filters.append(
|
||||
or_(
|
||||
and_(
|
||||
ArtifactLibrarySyncStateRow.status == "error",
|
||||
or_(
|
||||
ArtifactLibrarySyncStateRow.started_at.is_(None),
|
||||
ArtifactLibrarySyncStateRow.started_at <= stale_before,
|
||||
),
|
||||
),
|
||||
and_(
|
||||
ArtifactLibrarySyncStateRow.status != "error",
|
||||
or_(
|
||||
ArtifactLibrarySyncStateRow.completed_at.is_(None),
|
||||
ArtifactLibrarySyncStateRow.completed_at <= stale_before,
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# A conditional UPDATE is the portable compare-and-swap.
|
||||
# SQLite ignores SELECT ... FOR UPDATE, whereas this write
|
||||
# serializes claimers there and also works on MySQL,
|
||||
# PostgreSQL, and openGauss.
|
||||
result = await session.execute(
|
||||
update(ArtifactLibrarySyncStateRow)
|
||||
.where(*claim_filters)
|
||||
.values(
|
||||
status="syncing",
|
||||
generation=generation,
|
||||
started_at=now,
|
||||
lease_expires_at=now + timedelta(seconds=lease_seconds),
|
||||
last_error=None,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 1:
|
||||
return generation
|
||||
|
||||
row = await session.get(ArtifactLibrarySyncStateRow, user_id)
|
||||
if row is not None:
|
||||
return None
|
||||
session.add(
|
||||
ArtifactLibrarySyncStateRow(
|
||||
user_id=user_id,
|
||||
status="syncing",
|
||||
generation=generation,
|
||||
started_at=now,
|
||||
lease_expires_at=now + timedelta(seconds=lease_seconds),
|
||||
last_error=None,
|
||||
)
|
||||
)
|
||||
return generation
|
||||
except IntegrityError:
|
||||
# Two workers can race while inserting a user's first state row.
|
||||
# The winner owns the lease; the loser simply serves the same
|
||||
# persisted index without starting a duplicate filesystem scan.
|
||||
async with self._sf() as session:
|
||||
if await session.get(ArtifactLibrarySyncStateRow, user_id) is not None:
|
||||
return None
|
||||
raise
|
||||
|
||||
async def finish_sync(
|
||||
self,
|
||||
user_id: str,
|
||||
generation: str,
|
||||
items: list[dict[str, Any]],
|
||||
) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
deduped = {_artifact_id(user_id, item): item for item in items}
|
||||
async with self._sf() as session:
|
||||
async with session.begin():
|
||||
# Write first so SQLite also serializes this publisher against
|
||||
# a lease takeover. Requiring status=syncing makes completion
|
||||
# one-shot even if the same callback is accidentally invoked
|
||||
# twice with an old generation token.
|
||||
fence = await session.execute(
|
||||
update(ArtifactLibrarySyncStateRow)
|
||||
.where(
|
||||
ArtifactLibrarySyncStateRow.user_id == user_id,
|
||||
ArtifactLibrarySyncStateRow.generation == generation,
|
||||
ArtifactLibrarySyncStateRow.status == "syncing",
|
||||
)
|
||||
.values(lease_expires_at=now + timedelta(seconds=300))
|
||||
)
|
||||
if fence.rowcount != 1:
|
||||
return False
|
||||
|
||||
existing_result = await session.execute(select(ArtifactLibraryItemRow).where(ArtifactLibraryItemRow.user_id == user_id))
|
||||
existing = {row.artifact_id: row for row in existing_result.scalars()}
|
||||
for artifact_id, item in deduped.items():
|
||||
row = existing.get(artifact_id)
|
||||
if row is None:
|
||||
row = ArtifactLibraryItemRow(
|
||||
artifact_id=artifact_id,
|
||||
user_id=user_id,
|
||||
thread_id="",
|
||||
thread_title="",
|
||||
source_type="general_qa",
|
||||
path="",
|
||||
name="",
|
||||
size_bytes=0,
|
||||
modified_at=_EPOCH,
|
||||
file_kind="other",
|
||||
sync_generation=generation,
|
||||
indexed_at=now,
|
||||
)
|
||||
session.add(row)
|
||||
_apply_item(
|
||||
row,
|
||||
user_id=user_id,
|
||||
generation=generation,
|
||||
item=item,
|
||||
indexed_at=now,
|
||||
)
|
||||
await session.flush()
|
||||
await session.execute(
|
||||
delete(ArtifactLibraryItemRow).where(
|
||||
ArtifactLibraryItemRow.user_id == user_id,
|
||||
ArtifactLibraryItemRow.sync_generation != generation,
|
||||
)
|
||||
)
|
||||
await session.execute(
|
||||
update(ArtifactLibrarySyncStateRow)
|
||||
.where(
|
||||
ArtifactLibrarySyncStateRow.user_id == user_id,
|
||||
ArtifactLibrarySyncStateRow.generation == generation,
|
||||
ArtifactLibrarySyncStateRow.status == "syncing",
|
||||
)
|
||||
.values(
|
||||
status="ready",
|
||||
completed_at=now,
|
||||
lease_expires_at=None,
|
||||
last_error=None,
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
async def fail_sync(self, user_id: str, generation: str, error: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
async with session.begin():
|
||||
result = await session.execute(
|
||||
update(ArtifactLibrarySyncStateRow)
|
||||
.where(
|
||||
ArtifactLibrarySyncStateRow.user_id == user_id,
|
||||
ArtifactLibrarySyncStateRow.generation == generation,
|
||||
ArtifactLibrarySyncStateRow.status == "syncing",
|
||||
)
|
||||
.values(
|
||||
status="error",
|
||||
lease_expires_at=None,
|
||||
last_error=(error or "Artifact library refresh failed")[:1024],
|
||||
)
|
||||
)
|
||||
return result.rowcount == 1
|
||||
|
||||
async def get_sync_state(self, user_id: str) -> dict[str, Any]:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(ArtifactLibrarySyncStateRow, user_id)
|
||||
return _state_dict(row)
|
||||
|
||||
async def list_items(
|
||||
self,
|
||||
user_id: str,
|
||||
*,
|
||||
query: str | None = None,
|
||||
file_kind: str = "all",
|
||||
source_type: str = "all",
|
||||
offset: int = 0,
|
||||
limit: int = 50,
|
||||
) -> tuple[int, list[dict[str, Any]]]:
|
||||
filters = [ArtifactLibraryItemRow.user_id == user_id]
|
||||
if file_kind != "all":
|
||||
filters.append(ArtifactLibraryItemRow.file_kind == file_kind)
|
||||
if source_type != "all":
|
||||
filters.append(ArtifactLibraryItemRow.source_type == source_type)
|
||||
normalized = (query or "").strip().casefold()
|
||||
if normalized:
|
||||
searchable = (
|
||||
ArtifactLibraryItemRow.name,
|
||||
ArtifactLibraryItemRow.thread_title,
|
||||
ArtifactLibraryItemRow.agent_name,
|
||||
ArtifactLibraryItemRow.strategy_name,
|
||||
ArtifactLibraryItemRow.strategy_project_id,
|
||||
)
|
||||
filters.append(or_(*(func.lower(func.coalesce(column, "")).contains(normalized, autoescape=True) for column in searchable)))
|
||||
|
||||
count_stmt = select(func.count()).select_from(ArtifactLibraryItemRow).where(*filters)
|
||||
page_stmt = (
|
||||
select(ArtifactLibraryItemRow)
|
||||
.where(*filters)
|
||||
.order_by(
|
||||
ArtifactLibraryItemRow.modified_at.desc(),
|
||||
ArtifactLibraryItemRow.artifact_id.desc(),
|
||||
)
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
total = int((await session.execute(count_stmt)).scalar_one())
|
||||
rows = (await session.execute(page_stmt)).scalars().all()
|
||||
return total, [_item_dict(row) for row in rows]
|
||||
Reference in new issue
Block a user