sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
@@ -4,8 +4,10 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from deerflow.utils.time import now_iso as _now_iso
|
||||
@@ -36,6 +38,7 @@ class RunRecord:
|
||||
abort_event: asyncio.Event = field(default_factory=asyncio.Event, repr=False)
|
||||
abort_action: str = "interrupt"
|
||||
error: str | None = None
|
||||
superseded: bool = field(default=False, repr=False)
|
||||
|
||||
|
||||
class RunManager:
|
||||
@@ -50,6 +53,8 @@ class RunManager:
|
||||
self._runs: dict[str, RunRecord] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
self._store = store
|
||||
self._finished_at: dict[str, float] = {}
|
||||
self._retention_task: asyncio.Task | None = None
|
||||
|
||||
async def _persist_to_store(self, record: RunRecord) -> None:
|
||||
"""Best-effort persist run record to backing store."""
|
||||
@@ -103,6 +108,9 @@ class RunManager:
|
||||
updated_at=now,
|
||||
)
|
||||
async with self._lock:
|
||||
for previous in self._runs.values():
|
||||
if previous.thread_id == thread_id:
|
||||
previous.superseded = True
|
||||
self._runs[run_id] = record
|
||||
await self._persist_to_store(record)
|
||||
logger.info("Run created: run_id=%s thread_id=%s", run_id, thread_id)
|
||||
@@ -138,7 +146,7 @@ class RunManager:
|
||||
error=data.get("error"),
|
||||
)
|
||||
|
||||
async def get_persisted(self, run_id: str) -> RunRecord | None:
|
||||
async def get_persisted(self, run_id: str, *, authorized_thread_id: str | None = None) -> RunRecord | None:
|
||||
"""Load a run from the backing store (process-restart fallback).
|
||||
|
||||
The in-memory registry is wiped on every Gateway restart, but the
|
||||
@@ -150,20 +158,36 @@ class RunManager:
|
||||
if self._store is None:
|
||||
return None
|
||||
try:
|
||||
data = await self._store.get(run_id)
|
||||
# Only an already-authorized thread route may bypass row ownership.
|
||||
# This keeps task-shared runs readable after retiring their RAM copy.
|
||||
data = await self._store.get(run_id, user_id=None) if authorized_thread_id else await self._store.get(run_id)
|
||||
except Exception:
|
||||
logger.warning("Failed to load run %s from store", run_id, exc_info=True)
|
||||
return None
|
||||
if not data:
|
||||
return None
|
||||
if authorized_thread_id is not None and data.get("thread_id") != authorized_thread_id:
|
||||
return None
|
||||
return self._record_from_store(data)
|
||||
|
||||
async def list_by_thread(self, thread_id: str) -> list[RunRecord]:
|
||||
"""Return all runs for a given thread, newest first."""
|
||||
"""Return live and stored runs in the original creation order."""
|
||||
async with self._lock:
|
||||
# Dict insertion order matches creation order, so reversing it gives
|
||||
# us deterministic newest-first results even when timestamps tie.
|
||||
return [r for r in self._runs.values() if r.thread_id == thread_id]
|
||||
# Keep live object identity when the same run also exists in SQL.
|
||||
records = {r.run_id: r for r in self._runs.values() if r.thread_id == thread_id}
|
||||
if self._store is not None:
|
||||
try:
|
||||
# Callers authorize the thread, as for the original RAM registry.
|
||||
for row in await self._store.list_by_thread(thread_id, user_id=None, limit=1000):
|
||||
records.setdefault(row["run_id"], self._record_from_store(row))
|
||||
except Exception:
|
||||
logger.warning("Failed to list persisted runs for thread %s", thread_id, exc_info=True)
|
||||
return sorted(records.values(), key=lambda record: datetime.fromisoformat(record.created_at.replace("Z", "+00:00")).timestamp())
|
||||
|
||||
def is_latest(self, record: RunRecord) -> bool:
|
||||
"""Completion fencing stays in RAM; it must not wait on history IO."""
|
||||
latest = next((item for item in reversed(self._runs.values()) if item.thread_id == record.thread_id), None)
|
||||
return not record.superseded and latest is record
|
||||
|
||||
async def set_status(self, run_id: str, status: RunStatus, *, error: str | None = None) -> None:
|
||||
"""Transition a run to a new status."""
|
||||
@@ -176,6 +200,8 @@ class RunManager:
|
||||
record.updated_at = _now_iso()
|
||||
if error is not None:
|
||||
record.error = error
|
||||
if status not in (RunStatus.pending, RunStatus.running):
|
||||
self._finished_at[run_id] = time.monotonic()
|
||||
if self._store is not None:
|
||||
try:
|
||||
await self._store.update_status(run_id, status.value, error=error)
|
||||
@@ -290,6 +316,9 @@ class RunManager:
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
for previous in self._runs.values():
|
||||
if previous.thread_id == thread_id:
|
||||
previous.superseded = True
|
||||
self._runs[run_id] = record
|
||||
|
||||
await self._persist_to_store(record)
|
||||
@@ -316,13 +345,75 @@ class RunManager:
|
||||
return [r for r in self._runs.values() if r.status in (RunStatus.pending, RunStatus.running)]
|
||||
|
||||
async def cleanup(self, run_id: str, *, delay: float = 300) -> None:
|
||||
"""Remove a run record after an optional delay."""
|
||||
"""Retire only a stopped run whose terminal snapshot is safely stored."""
|
||||
if delay > 0:
|
||||
await asyncio.sleep(delay)
|
||||
record = self._runs.get(run_id)
|
||||
if record is None:
|
||||
self._finished_at.pop(run_id, None)
|
||||
return
|
||||
if self._store is None or record.status in (RunStatus.pending, RunStatus.running):
|
||||
return
|
||||
if record.task is not None and not record.task.done():
|
||||
return
|
||||
try:
|
||||
saved = await self._store.get(run_id, user_id=None)
|
||||
except Exception:
|
||||
logger.warning("Retaining run %s: durable lookup unavailable", run_id, exc_info=True)
|
||||
return
|
||||
if not saved or saved.get("thread_id") != record.thread_id or saved.get("status") != record.status.value or saved.get("error") != record.error:
|
||||
return
|
||||
# Never evict a user input that was not successfully persisted.
|
||||
if saved.get("kwargs", {}) != record.kwargs or saved.get("metadata", {}) != record.metadata:
|
||||
try:
|
||||
if not await self._store.update_snapshot(run_id, metadata=record.metadata, kwargs=record.kwargs):
|
||||
return
|
||||
saved = await self._store.get(run_id, user_id=None)
|
||||
except Exception:
|
||||
logger.warning("Retaining run %s: final snapshot could not be stored", run_id, exc_info=True)
|
||||
return
|
||||
if not saved or saved.get("kwargs", {}) != record.kwargs or saved.get("metadata", {}) != record.metadata:
|
||||
return
|
||||
async with self._lock:
|
||||
self._runs.pop(run_id, None)
|
||||
if self._runs.get(run_id) is record and (record.task is None or record.task.done()):
|
||||
self._runs.pop(run_id, None)
|
||||
self._finished_at.pop(run_id, None)
|
||||
logger.debug("Run record %s cleaned up", run_id)
|
||||
|
||||
def watch_completion(self, record: RunRecord) -> None:
|
||||
def finished(_):
|
||||
if self._runs.get(record.run_id) is record:
|
||||
self._finished_at[record.run_id] = time.monotonic()
|
||||
|
||||
if record.task is not None:
|
||||
record.task.add_done_callback(finished)
|
||||
|
||||
async def reap_completed(self, retention_seconds: float = 300) -> None:
|
||||
cutoff = time.monotonic() - retention_seconds
|
||||
for run_id in [key for key, finished in self._finished_at.items() if finished <= cutoff][:200]:
|
||||
await self.cleanup(run_id, delay=0)
|
||||
if run_id in self._finished_at:
|
||||
self._finished_at[run_id] = time.monotonic()
|
||||
|
||||
def start_retention(self, *, retention_seconds=300, interval_seconds=10) -> None:
|
||||
if self._retention_task is not None and not self._retention_task.done():
|
||||
return
|
||||
|
||||
async def sweep():
|
||||
while True:
|
||||
await asyncio.sleep(interval_seconds)
|
||||
try:
|
||||
await self.reap_completed(retention_seconds)
|
||||
except Exception:
|
||||
logger.exception("Completed run retention failed; retaining in-memory records")
|
||||
|
||||
self._retention_task = asyncio.create_task(sweep(), name="completed-run-retention")
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._retention_task is not None:
|
||||
self._retention_task.cancel()
|
||||
await asyncio.gather(self._retention_task, return_exceptions=True)
|
||||
|
||||
|
||||
class ConflictError(Exception):
|
||||
"""Raised when multitask_strategy=reject and thread has inflight runs."""
|
||||
|
||||
Reference in new issue
Block a user