初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
@@ -0,0 +1,151 @@
|
||||
"""Lease-based dispatcher for workflow runs.
|
||||
|
||||
A single background loop scans for claimable runs (``queued``, or ``running``
|
||||
with an expired lease) and hands each successfully claimed run to the executor.
|
||||
Because the claim is an atomic conditional UPDATE, several workers can run this
|
||||
loop concurrently and a run that dies with its worker is picked up by whoever
|
||||
notices the expired lease first — that is the restart-recovery story.
|
||||
|
||||
``nudge()`` is called by the start/resume routes so a fresh run does not wait
|
||||
for the next poll tick.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from app.gateway.workflow_executor import LeaseParams, WorkflowRunExecutor
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
from deerflow.workflows.runtime import PersistingWorkflowEventSink
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_POLL_INTERVAL_SECONDS = 2.0
|
||||
_CLAIM_BATCH = 5
|
||||
|
||||
|
||||
def _worker_id() -> str:
|
||||
return f"{socket.gethostname()[:24]}-{os.getpid()}-{uuid.uuid4().hex[:6]}"
|
||||
|
||||
|
||||
class WorkflowRunDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
store: WorkflowRunStore,
|
||||
executor: WorkflowRunExecutor,
|
||||
*,
|
||||
event_store: WorkflowEventStore | None = None,
|
||||
live_publisher: Any = None,
|
||||
config: WorkflowConfig | None = None,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._executor = executor
|
||||
self._events = event_store
|
||||
self._publish = live_publisher
|
||||
self._config = config or WorkflowConfig()
|
||||
self._worker_id = _worker_id()
|
||||
self._wake = asyncio.Event()
|
||||
self._task: asyncio.Task | None = None
|
||||
self._stopping = False
|
||||
|
||||
@property
|
||||
def worker_id(self) -> str:
|
||||
return self._worker_id
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is not None and not self._task.done():
|
||||
return
|
||||
self._stopping = False
|
||||
self._task = asyncio.create_task(self._loop(), name="workflow-run-dispatcher")
|
||||
logger.info("workflow dispatcher started (worker=%s)", self._worker_id)
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._stopping = True
|
||||
self._wake.set()
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
self._task = None
|
||||
await self._executor.drain()
|
||||
|
||||
def nudge(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
async def _loop(self) -> None:
|
||||
await self._safe_dispatch()
|
||||
while not self._stopping:
|
||||
try:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=_POLL_INTERVAL_SECONDS)
|
||||
except TimeoutError:
|
||||
pass
|
||||
self._wake.clear()
|
||||
if self._stopping:
|
||||
return
|
||||
await self._safe_dispatch()
|
||||
|
||||
async def _safe_dispatch(self) -> None:
|
||||
try:
|
||||
await self.dispatch_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - the loop must survive a bad round
|
||||
logger.exception("workflow dispatch round failed")
|
||||
|
||||
async def dispatch_once(self) -> int:
|
||||
now = datetime.now(UTC)
|
||||
await self._reap_cancel_requested()
|
||||
candidates = await self._store.list_claimable(now=now, limit=_CLAIM_BATCH)
|
||||
if not candidates:
|
||||
return 0
|
||||
ttl = max(30, self._config.lease_ttl_seconds)
|
||||
started = 0
|
||||
for run_id in candidates:
|
||||
row = await self._store.claim_run(
|
||||
run_id,
|
||||
lease_owner=self._worker_id,
|
||||
lease_until=now + timedelta(seconds=ttl),
|
||||
)
|
||||
if row is None:
|
||||
continue
|
||||
if self._executor.start(LeaseParams(run_id=run_id, lease_owner=self._worker_id, attempt=int(row.get("attempt") or 1))):
|
||||
started += 1
|
||||
return started
|
||||
|
||||
async def _reap_cancel_requested(self) -> None:
|
||||
"""Finalize cancels whose worker is gone (no live lease to notice them)."""
|
||||
rows = await self._store.list_runs(status="cancel_requested", limit=20)
|
||||
now = datetime.now(UTC)
|
||||
for row in rows:
|
||||
lease_until = row.get("lease_until")
|
||||
if lease_until:
|
||||
try:
|
||||
if datetime.fromisoformat(str(lease_until)) > now:
|
||||
continue # its worker is alive and will handle the cancel
|
||||
except ValueError:
|
||||
pass
|
||||
finalized = await self._store.finalize_cancel(row["id"])
|
||||
if finalized is not None and self._events is not None:
|
||||
# Nobody owns this run, so the dispatcher owes the stream its
|
||||
# terminal frame — otherwise open SSE connections hang.
|
||||
sink = PersistingWorkflowEventSink(
|
||||
self._events,
|
||||
run_id=row["id"],
|
||||
workflow_id=row["workflow_id"],
|
||||
version_id=row["workflow_version_id"],
|
||||
publish=self._publish,
|
||||
)
|
||||
await sink.emit("run.cancelled", data={"reapedBy": self._worker_id})
|
||||
|
||||
|
||||
__all__ = ["WorkflowRunDispatcher"]
|
||||
Reference in new issue
Block a user