sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -4,8 +4,10 @@ from __future__ import annotations
import asyncio
import logging
import time
import uuid
from dataclasses import dataclass, field
from datetime import datetime
from typing import TYPE_CHECKING
from deerflow.utils.time import now_iso as _now_iso
@@ -36,6 +38,7 @@ class RunRecord:
abort_event: asyncio.Event = field(default_factory=asyncio.Event, repr=False)
abort_action: str = "interrupt"
error: str | None = None
superseded: bool = field(default=False, repr=False)
class RunManager:
@@ -50,6 +53,8 @@ class RunManager:
self._runs: dict[str, RunRecord] = {}
self._lock = asyncio.Lock()
self._store = store
self._finished_at: dict[str, float] = {}
self._retention_task: asyncio.Task | None = None
async def _persist_to_store(self, record: RunRecord) -> None:
"""Best-effort persist run record to backing store."""
@@ -103,6 +108,9 @@ class RunManager:
updated_at=now,
)
async with self._lock:
for previous in self._runs.values():
if previous.thread_id == thread_id:
previous.superseded = True
self._runs[run_id] = record
await self._persist_to_store(record)
logger.info("Run created: run_id=%s thread_id=%s", run_id, thread_id)
@@ -138,7 +146,7 @@ class RunManager:
error=data.get("error"),
)
async def get_persisted(self, run_id: str) -> RunRecord | None:
async def get_persisted(self, run_id: str, *, authorized_thread_id: str | None = None) -> RunRecord | None:
"""Load a run from the backing store (process-restart fallback).
The in-memory registry is wiped on every Gateway restart, but the
@@ -150,20 +158,36 @@ class RunManager:
if self._store is None:
return None
try:
data = await self._store.get(run_id)
# Only an already-authorized thread route may bypass row ownership.
# This keeps task-shared runs readable after retiring their RAM copy.
data = await self._store.get(run_id, user_id=None) if authorized_thread_id else await self._store.get(run_id)
except Exception:
logger.warning("Failed to load run %s from store", run_id, exc_info=True)
return None
if not data:
return None
if authorized_thread_id is not None and data.get("thread_id") != authorized_thread_id:
return None
return self._record_from_store(data)
async def list_by_thread(self, thread_id: str) -> list[RunRecord]:
"""Return all runs for a given thread, newest first."""
"""Return live and stored runs in the original creation order."""
async with self._lock:
# Dict insertion order matches creation order, so reversing it gives
# us deterministic newest-first results even when timestamps tie.
return [r for r in self._runs.values() if r.thread_id == thread_id]
# Keep live object identity when the same run also exists in SQL.
records = {r.run_id: r for r in self._runs.values() if r.thread_id == thread_id}
if self._store is not None:
try:
# Callers authorize the thread, as for the original RAM registry.
for row in await self._store.list_by_thread(thread_id, user_id=None, limit=1000):
records.setdefault(row["run_id"], self._record_from_store(row))
except Exception:
logger.warning("Failed to list persisted runs for thread %s", thread_id, exc_info=True)
return sorted(records.values(), key=lambda record: datetime.fromisoformat(record.created_at.replace("Z", "+00:00")).timestamp())
def is_latest(self, record: RunRecord) -> bool:
"""Completion fencing stays in RAM; it must not wait on history IO."""
latest = next((item for item in reversed(self._runs.values()) if item.thread_id == record.thread_id), None)
return not record.superseded and latest is record
async def set_status(self, run_id: str, status: RunStatus, *, error: str | None = None) -> None:
"""Transition a run to a new status."""
@@ -176,6 +200,8 @@ class RunManager:
record.updated_at = _now_iso()
if error is not None:
record.error = error
if status not in (RunStatus.pending, RunStatus.running):
self._finished_at[run_id] = time.monotonic()
if self._store is not None:
try:
await self._store.update_status(run_id, status.value, error=error)
@@ -290,6 +316,9 @@ class RunManager:
created_at=now,
updated_at=now,
)
for previous in self._runs.values():
if previous.thread_id == thread_id:
previous.superseded = True
self._runs[run_id] = record
await self._persist_to_store(record)
@@ -316,13 +345,75 @@ class RunManager:
return [r for r in self._runs.values() if r.status in (RunStatus.pending, RunStatus.running)]
async def cleanup(self, run_id: str, *, delay: float = 300) -> None:
"""Remove a run record after an optional delay."""
"""Retire only a stopped run whose terminal snapshot is safely stored."""
if delay > 0:
await asyncio.sleep(delay)
record = self._runs.get(run_id)
if record is None:
self._finished_at.pop(run_id, None)
return
if self._store is None or record.status in (RunStatus.pending, RunStatus.running):
return
if record.task is not None and not record.task.done():
return
try:
saved = await self._store.get(run_id, user_id=None)
except Exception:
logger.warning("Retaining run %s: durable lookup unavailable", run_id, exc_info=True)
return
if not saved or saved.get("thread_id") != record.thread_id or saved.get("status") != record.status.value or saved.get("error") != record.error:
return
# Never evict a user input that was not successfully persisted.
if saved.get("kwargs", {}) != record.kwargs or saved.get("metadata", {}) != record.metadata:
try:
if not await self._store.update_snapshot(run_id, metadata=record.metadata, kwargs=record.kwargs):
return
saved = await self._store.get(run_id, user_id=None)
except Exception:
logger.warning("Retaining run %s: final snapshot could not be stored", run_id, exc_info=True)
return
if not saved or saved.get("kwargs", {}) != record.kwargs or saved.get("metadata", {}) != record.metadata:
return
async with self._lock:
self._runs.pop(run_id, None)
if self._runs.get(run_id) is record and (record.task is None or record.task.done()):
self._runs.pop(run_id, None)
self._finished_at.pop(run_id, None)
logger.debug("Run record %s cleaned up", run_id)
def watch_completion(self, record: RunRecord) -> None:
def finished(_):
if self._runs.get(record.run_id) is record:
self._finished_at[record.run_id] = time.monotonic()
if record.task is not None:
record.task.add_done_callback(finished)
async def reap_completed(self, retention_seconds: float = 300) -> None:
cutoff = time.monotonic() - retention_seconds
for run_id in [key for key, finished in self._finished_at.items() if finished <= cutoff][:200]:
await self.cleanup(run_id, delay=0)
if run_id in self._finished_at:
self._finished_at[run_id] = time.monotonic()
def start_retention(self, *, retention_seconds=300, interval_seconds=10) -> None:
if self._retention_task is not None and not self._retention_task.done():
return
async def sweep():
while True:
await asyncio.sleep(interval_seconds)
try:
await self.reap_completed(retention_seconds)
except Exception:
logger.exception("Completed run retention failed; retaining in-memory records")
self._retention_task = asyncio.create_task(sweep(), name="completed-run-retention")
async def aclose(self) -> None:
if self._retention_task is not None:
self._retention_task.cancel()
await asyncio.gather(self._retention_task, return_exceptions=True)
class ConflictError(Exception):
"""Raised when multitask_strategy=reject and thread has inflight runs."""