sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
+5
-2
@@ -5,14 +5,17 @@ from deerflow.runtime.events.store.memory import MemoryRunEventStore
|
||||
def make_run_event_store(config=None) -> RunEventStore:
|
||||
"""Create a RunEventStore based on run_events.backend configuration."""
|
||||
if config is None or config.backend == "memory":
|
||||
return MemoryRunEventStore()
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.runtime.events.store.spool import SpoolingRunEventStore
|
||||
|
||||
return SpoolingRunEventStore(runtime_home() / "run-events" / "events.sqlite3")
|
||||
if config.backend == "db":
|
||||
from deerflow.persistence.engine import get_session_factory
|
||||
|
||||
sf = get_session_factory()
|
||||
if sf is None:
|
||||
# database.backend=memory but run_events.backend=db -> fallback
|
||||
return MemoryRunEventStore()
|
||||
return make_run_event_store()
|
||||
from deerflow.runtime.events.store.db import DbRunEventStore
|
||||
|
||||
return DbRunEventStore(sf, max_trace_content=config.max_trace_content)
|
||||
|
||||
+45
-9
@@ -16,19 +16,44 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
from contextlib import ExitStack
|
||||
from datetime import UTC, datetime
|
||||
from functools import wraps
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.runtime.events.store.base import RunEventStore
|
||||
from deerflow.utils.file_io import event_io
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SAFE_ID_PATTERN = re.compile(r"^[A-Za-z0-9_\-]+$")
|
||||
|
||||
|
||||
def _disk_operation(method):
|
||||
@wraps(method)
|
||||
async def asynchronous(self, *args, **kwargs):
|
||||
if method.__name__ == "put_batch":
|
||||
events = kwargs.get("events", args[0] if args else [])
|
||||
threads = {event["thread_id"] for event in events}
|
||||
else:
|
||||
threads = {kwargs.get("thread_id", args[0] if args else "")}
|
||||
stripes = sorted({hash(thread) % len(self._locks) for thread in threads})
|
||||
|
||||
def operation():
|
||||
with ExitStack() as stack:
|
||||
for stripe in stripes:
|
||||
stack.enter_context(self._locks[stripe])
|
||||
return method(self, *args, **kwargs)
|
||||
|
||||
return await event_io(operation)
|
||||
return asynchronous
|
||||
|
||||
|
||||
class JsonlRunEventStore(RunEventStore):
|
||||
def __init__(self, base_dir: str | Path | None = None):
|
||||
self._base_dir = Path(base_dir) if base_dir else Path(".deer-flow")
|
||||
self._locks = [threading.RLock() for _ in range(64)]
|
||||
self._seq_counters: dict[str, int] = {} # thread_id -> current max seq
|
||||
|
||||
@staticmethod
|
||||
@@ -108,7 +133,7 @@ class JsonlRunEventStore(RunEventStore):
|
||||
events.sort(key=lambda e: e.get("seq", 0))
|
||||
return events
|
||||
|
||||
async def put(self, *, thread_id, run_id, event_type, category, content="", metadata=None, created_at=None):
|
||||
def _put(self, *, thread_id, run_id, event_type, category, content="", metadata=None, created_at=None):
|
||||
self._ensure_seq_loaded(thread_id)
|
||||
seq = self._next_seq(thread_id)
|
||||
record = {
|
||||
@@ -124,16 +149,22 @@ class JsonlRunEventStore(RunEventStore):
|
||||
self._write_record(record)
|
||||
return record
|
||||
|
||||
async def put_batch(self, events):
|
||||
@_disk_operation
|
||||
def put(self, **kwargs):
|
||||
return self._put(**kwargs)
|
||||
|
||||
@_disk_operation
|
||||
def put_batch(self, events):
|
||||
if not events:
|
||||
return []
|
||||
results = []
|
||||
for ev in events:
|
||||
record = await self.put(**ev)
|
||||
record = self._put(**ev)
|
||||
results.append(record)
|
||||
return results
|
||||
|
||||
async def list_messages(self, thread_id, *, limit=50, before_seq=None, after_seq=None):
|
||||
@_disk_operation
|
||||
def list_messages(self, thread_id, *, limit=50, before_seq=None, after_seq=None):
|
||||
all_events = self._read_thread_events(thread_id)
|
||||
messages = [e for e in all_events if e.get("category") == "message"]
|
||||
|
||||
@@ -146,13 +177,15 @@ class JsonlRunEventStore(RunEventStore):
|
||||
else:
|
||||
return messages[-limit:]
|
||||
|
||||
async def list_events(self, thread_id, run_id, *, event_types=None, limit=500):
|
||||
@_disk_operation
|
||||
def list_events(self, thread_id, run_id, *, event_types=None, limit=500):
|
||||
events = self._read_run_events(thread_id, run_id)
|
||||
if event_types is not None:
|
||||
events = [e for e in events if e.get("event_type") in event_types]
|
||||
return events[:limit]
|
||||
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
|
||||
@_disk_operation
|
||||
def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
|
||||
# JSONL events carry no per-event owner; user_id is accepted for
|
||||
# interface parity with DbRunEventStore but ignored.
|
||||
events = self._read_run_events(thread_id, run_id)
|
||||
@@ -166,11 +199,13 @@ class JsonlRunEventStore(RunEventStore):
|
||||
else:
|
||||
return filtered[-limit:] if len(filtered) > limit else filtered
|
||||
|
||||
async def count_messages(self, thread_id):
|
||||
@_disk_operation
|
||||
def count_messages(self, thread_id):
|
||||
all_events = self._read_thread_events(thread_id)
|
||||
return sum(1 for e in all_events if e.get("category") == "message")
|
||||
|
||||
async def delete_by_thread(self, thread_id):
|
||||
@_disk_operation
|
||||
def delete_by_thread(self, thread_id):
|
||||
all_events = self._read_thread_events(thread_id)
|
||||
count = len(all_events)
|
||||
thread_dir = self._thread_dir(thread_id)
|
||||
@@ -180,7 +215,8 @@ class JsonlRunEventStore(RunEventStore):
|
||||
self._seq_counters.pop(thread_id, None)
|
||||
return count
|
||||
|
||||
async def delete_by_run(self, thread_id, run_id):
|
||||
@_disk_operation
|
||||
def delete_by_run(self, thread_id, run_id):
|
||||
events = self._read_run_events(thread_id, run_id)
|
||||
count = len(events)
|
||||
path = self._run_file(thread_id, run_id)
|
||||
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
"""Disk spool for the memory event backend; never discard cold messages.
|
||||
|
||||
The gateway is single-process. SQLite transactions give concurrent journal
|
||||
writers stable thread cursors without keeping all completed events in RAM.
|
||||
Thread access remains enforced by the existing gateway authorization layer,
|
||||
matching MemoryRunEventStore/JsonlRunEventStore semantics.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import threading
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.runtime.events.store.base import RunEventStore
|
||||
from deerflow.utils.file_io import event_io
|
||||
|
||||
_initialization_locks = [threading.Lock() for _ in range(64)]
|
||||
|
||||
|
||||
class SpoolingRunEventStore(RunEventStore):
|
||||
def __init__(self, path: Path):
|
||||
self.path = Path(path)
|
||||
self._initialized = False
|
||||
|
||||
def _connect(self):
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
connection = sqlite3.connect(self.path, timeout=30)
|
||||
try:
|
||||
if not self._initialized:
|
||||
with _initialization_locks[hash(str(self.path.resolve())) % len(_initialization_locks)]:
|
||||
if not self._initialized:
|
||||
connection.execute("PRAGMA journal_mode=WAL")
|
||||
connection.execute("CREATE TABLE IF NOT EXISTS cursors (thread_id TEXT PRIMARY KEY, seq INTEGER NOT NULL)")
|
||||
connection.execute("CREATE TABLE IF NOT EXISTS events "
|
||||
"(thread_id TEXT NOT NULL, seq INTEGER NOT NULL, run_id TEXT NOT NULL, "
|
||||
"category TEXT NOT NULL, event_type TEXT NOT NULL, payload TEXT NOT NULL, PRIMARY KEY(thread_id,seq))")
|
||||
connection.execute("CREATE INDEX IF NOT EXISTS events_run ON events(thread_id,run_id,seq)")
|
||||
self._initialized = True
|
||||
return connection
|
||||
except BaseException:
|
||||
connection.close()
|
||||
raise
|
||||
|
||||
def _put_batch(self, events):
|
||||
prepared = []
|
||||
for event in events:
|
||||
record = {"content": "", "metadata": {}, "created_at": datetime.now(UTC).isoformat(), **event}
|
||||
# Validate/serialize every input before opening the transaction.
|
||||
prepared.append(json.loads(json.dumps(record, ensure_ascii=False, default=str)))
|
||||
connection = self._connect()
|
||||
try:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
for record in prepared:
|
||||
thread = record["thread_id"]
|
||||
connection.execute("INSERT INTO cursors(thread_id,seq) VALUES (?,0) ON CONFLICT(thread_id) DO NOTHING", (thread,))
|
||||
connection.execute("UPDATE cursors SET seq=seq+1 WHERE thread_id=?", (thread,))
|
||||
record["seq"] = connection.execute("SELECT seq FROM cursors WHERE thread_id=?", (thread,)).fetchone()[0]
|
||||
connection.execute("INSERT INTO events VALUES (?,?,?,?,?,?)", (thread, record["seq"], record["run_id"], record["category"], record["event_type"], json.dumps(record, ensure_ascii=False)))
|
||||
connection.commit()
|
||||
return prepared
|
||||
except BaseException:
|
||||
connection.rollback()
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
async def put(self, *, thread_id, run_id, event_type, category, content="", metadata=None, created_at=None):
|
||||
return (await self.put_batch([dict(thread_id=thread_id, run_id=run_id, event_type=event_type, category=category, content=content, metadata=metadata or {}, created_at=created_at or datetime.now(UTC).isoformat())]))[0]
|
||||
|
||||
async def put_batch(self, events):
|
||||
return await event_io(self._put_batch, events) if events else []
|
||||
|
||||
def _query(self, conditions, args, *, limit, descending=False):
|
||||
connection = self._connect()
|
||||
try:
|
||||
order = "DESC" if descending else "ASC"
|
||||
rows = connection.execute(f"SELECT payload FROM events WHERE {' AND '.join(conditions)} ORDER BY seq {order} LIMIT ?", [*args, max(0, int(limit))]).fetchall()
|
||||
data = [json.loads(row[0]) for row in rows]
|
||||
return list(reversed(data)) if descending else data
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
async def _messages(self, thread_id, *, run_id=None, limit=50, before_seq=None, after_seq=None):
|
||||
conditions, args = ["thread_id=?", "category='message'"], [thread_id]
|
||||
if run_id is not None:
|
||||
conditions.append("run_id=?")
|
||||
args.append(run_id)
|
||||
if before_seq is not None:
|
||||
conditions.append("seq<?")
|
||||
args.append(before_seq)
|
||||
if after_seq is not None:
|
||||
conditions.append("seq>?")
|
||||
args.append(after_seq)
|
||||
return await event_io(self._query, conditions, args, limit=limit, descending=after_seq is None)
|
||||
|
||||
async def list_messages(self, thread_id, *, limit=50, before_seq=None, after_seq=None):
|
||||
return await self._messages(thread_id, limit=limit, before_seq=before_seq, after_seq=after_seq)
|
||||
|
||||
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
|
||||
return await self._messages(thread_id, run_id=run_id, limit=limit, before_seq=before_seq, after_seq=after_seq)
|
||||
|
||||
async def list_events(self, thread_id, run_id, *, event_types=None, limit=500):
|
||||
conditions, args = ["thread_id=?", "run_id=?"], [thread_id, run_id]
|
||||
if event_types is not None:
|
||||
if not event_types:
|
||||
return []
|
||||
conditions.append(f"event_type IN ({','.join('?' for _ in event_types)})")
|
||||
args.extend(event_types)
|
||||
return await event_io(self._query, conditions, args, limit=limit)
|
||||
|
||||
def _count_or_delete(self, thread_id, run_id=None, *, delete=False):
|
||||
connection = self._connect()
|
||||
try:
|
||||
conditions, args = "thread_id=?", [thread_id]
|
||||
if run_id is not None:
|
||||
conditions += " AND run_id=?"
|
||||
args.append(run_id)
|
||||
if delete:
|
||||
connection.execute("BEGIN IMMEDIATE")
|
||||
count = connection.execute(f"SELECT COUNT(*) FROM events WHERE {conditions}" + ("" if delete else " AND category='message'"), args).fetchone()[0]
|
||||
if delete:
|
||||
connection.execute(f"DELETE FROM events WHERE {conditions}", args)
|
||||
if run_id is None:
|
||||
connection.execute("DELETE FROM cursors WHERE thread_id=?", (thread_id,))
|
||||
connection.commit()
|
||||
return count
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
async def count_messages(self, thread_id):
|
||||
return await event_io(self._count_or_delete, thread_id)
|
||||
|
||||
async def delete_by_thread(self, thread_id):
|
||||
return await event_io(self._count_or_delete, thread_id, delete=True)
|
||||
|
||||
async def delete_by_run(self, thread_id, run_id):
|
||||
return await event_io(self._count_or_delete, thread_id, run_id, delete=True)
|
||||
Reference in new issue
Block a user