sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -5,14 +5,17 @@ from deerflow.runtime.events.store.memory import MemoryRunEventStore
def make_run_event_store(config=None) -> RunEventStore:
"""Create a RunEventStore based on run_events.backend configuration."""
if config is None or config.backend == "memory":
return MemoryRunEventStore()
from deerflow.config.runtime_paths import runtime_home
from deerflow.runtime.events.store.spool import SpoolingRunEventStore
return SpoolingRunEventStore(runtime_home() / "run-events" / "events.sqlite3")
if config.backend == "db":
from deerflow.persistence.engine import get_session_factory
sf = get_session_factory()
if sf is None:
# database.backend=memory but run_events.backend=db -> fallback
return MemoryRunEventStore()
return make_run_event_store()
from deerflow.runtime.events.store.db import DbRunEventStore
return DbRunEventStore(sf, max_trace_content=config.max_trace_content)
@@ -16,19 +16,44 @@ from __future__ import annotations
import json
import logging
import re
import threading
from contextlib import ExitStack
from datetime import UTC, datetime
from functools import wraps
from pathlib import Path
from deerflow.runtime.events.store.base import RunEventStore
from deerflow.utils.file_io import event_io
logger = logging.getLogger(__name__)
_SAFE_ID_PATTERN = re.compile(r"^[A-Za-z0-9_\-]+$")
def _disk_operation(method):
@wraps(method)
async def asynchronous(self, *args, **kwargs):
if method.__name__ == "put_batch":
events = kwargs.get("events", args[0] if args else [])
threads = {event["thread_id"] for event in events}
else:
threads = {kwargs.get("thread_id", args[0] if args else "")}
stripes = sorted({hash(thread) % len(self._locks) for thread in threads})
def operation():
with ExitStack() as stack:
for stripe in stripes:
stack.enter_context(self._locks[stripe])
return method(self, *args, **kwargs)
return await event_io(operation)
return asynchronous
class JsonlRunEventStore(RunEventStore):
def __init__(self, base_dir: str | Path | None = None):
self._base_dir = Path(base_dir) if base_dir else Path(".deer-flow")
self._locks = [threading.RLock() for _ in range(64)]
self._seq_counters: dict[str, int] = {} # thread_id -> current max seq
@staticmethod
@@ -108,7 +133,7 @@ class JsonlRunEventStore(RunEventStore):
events.sort(key=lambda e: e.get("seq", 0))
return events
async def put(self, *, thread_id, run_id, event_type, category, content="", metadata=None, created_at=None):
def _put(self, *, thread_id, run_id, event_type, category, content="", metadata=None, created_at=None):
self._ensure_seq_loaded(thread_id)
seq = self._next_seq(thread_id)
record = {
@@ -124,16 +149,22 @@ class JsonlRunEventStore(RunEventStore):
self._write_record(record)
return record
async def put_batch(self, events):
@_disk_operation
def put(self, **kwargs):
return self._put(**kwargs)
@_disk_operation
def put_batch(self, events):
if not events:
return []
results = []
for ev in events:
record = await self.put(**ev)
record = self._put(**ev)
results.append(record)
return results
async def list_messages(self, thread_id, *, limit=50, before_seq=None, after_seq=None):
@_disk_operation
def list_messages(self, thread_id, *, limit=50, before_seq=None, after_seq=None):
all_events = self._read_thread_events(thread_id)
messages = [e for e in all_events if e.get("category") == "message"]
@@ -146,13 +177,15 @@ class JsonlRunEventStore(RunEventStore):
else:
return messages[-limit:]
async def list_events(self, thread_id, run_id, *, event_types=None, limit=500):
@_disk_operation
def list_events(self, thread_id, run_id, *, event_types=None, limit=500):
events = self._read_run_events(thread_id, run_id)
if event_types is not None:
events = [e for e in events if e.get("event_type") in event_types]
return events[:limit]
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
@_disk_operation
def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
# JSONL events carry no per-event owner; user_id is accepted for
# interface parity with DbRunEventStore but ignored.
events = self._read_run_events(thread_id, run_id)
@@ -166,11 +199,13 @@ class JsonlRunEventStore(RunEventStore):
else:
return filtered[-limit:] if len(filtered) > limit else filtered
async def count_messages(self, thread_id):
@_disk_operation
def count_messages(self, thread_id):
all_events = self._read_thread_events(thread_id)
return sum(1 for e in all_events if e.get("category") == "message")
async def delete_by_thread(self, thread_id):
@_disk_operation
def delete_by_thread(self, thread_id):
all_events = self._read_thread_events(thread_id)
count = len(all_events)
thread_dir = self._thread_dir(thread_id)
@@ -180,7 +215,8 @@ class JsonlRunEventStore(RunEventStore):
self._seq_counters.pop(thread_id, None)
return count
async def delete_by_run(self, thread_id, run_id):
@_disk_operation
def delete_by_run(self, thread_id, run_id):
events = self._read_run_events(thread_id, run_id)
count = len(events)
path = self._run_file(thread_id, run_id)
@@ -0,0 +1,140 @@
"""Disk spool for the memory event backend; never discard cold messages.
The gateway is single-process. SQLite transactions give concurrent journal
writers stable thread cursors without keeping all completed events in RAM.
Thread access remains enforced by the existing gateway authorization layer,
matching MemoryRunEventStore/JsonlRunEventStore semantics.
"""
from __future__ import annotations
import json
import sqlite3
import threading
from datetime import UTC, datetime
from pathlib import Path
from deerflow.runtime.events.store.base import RunEventStore
from deerflow.utils.file_io import event_io
_initialization_locks = [threading.Lock() for _ in range(64)]
class SpoolingRunEventStore(RunEventStore):
def __init__(self, path: Path):
self.path = Path(path)
self._initialized = False
def _connect(self):
self.path.parent.mkdir(parents=True, exist_ok=True)
connection = sqlite3.connect(self.path, timeout=30)
try:
if not self._initialized:
with _initialization_locks[hash(str(self.path.resolve())) % len(_initialization_locks)]:
if not self._initialized:
connection.execute("PRAGMA journal_mode=WAL")
connection.execute("CREATE TABLE IF NOT EXISTS cursors (thread_id TEXT PRIMARY KEY, seq INTEGER NOT NULL)")
connection.execute("CREATE TABLE IF NOT EXISTS events "
"(thread_id TEXT NOT NULL, seq INTEGER NOT NULL, run_id TEXT NOT NULL, "
"category TEXT NOT NULL, event_type TEXT NOT NULL, payload TEXT NOT NULL, PRIMARY KEY(thread_id,seq))")
connection.execute("CREATE INDEX IF NOT EXISTS events_run ON events(thread_id,run_id,seq)")
self._initialized = True
return connection
except BaseException:
connection.close()
raise
def _put_batch(self, events):
prepared = []
for event in events:
record = {"content": "", "metadata": {}, "created_at": datetime.now(UTC).isoformat(), **event}
# Validate/serialize every input before opening the transaction.
prepared.append(json.loads(json.dumps(record, ensure_ascii=False, default=str)))
connection = self._connect()
try:
connection.execute("BEGIN IMMEDIATE")
for record in prepared:
thread = record["thread_id"]
connection.execute("INSERT INTO cursors(thread_id,seq) VALUES (?,0) ON CONFLICT(thread_id) DO NOTHING", (thread,))
connection.execute("UPDATE cursors SET seq=seq+1 WHERE thread_id=?", (thread,))
record["seq"] = connection.execute("SELECT seq FROM cursors WHERE thread_id=?", (thread,)).fetchone()[0]
connection.execute("INSERT INTO events VALUES (?,?,?,?,?,?)", (thread, record["seq"], record["run_id"], record["category"], record["event_type"], json.dumps(record, ensure_ascii=False)))
connection.commit()
return prepared
except BaseException:
connection.rollback()
raise
finally:
connection.close()
async def put(self, *, thread_id, run_id, event_type, category, content="", metadata=None, created_at=None):
return (await self.put_batch([dict(thread_id=thread_id, run_id=run_id, event_type=event_type, category=category, content=content, metadata=metadata or {}, created_at=created_at or datetime.now(UTC).isoformat())]))[0]
async def put_batch(self, events):
return await event_io(self._put_batch, events) if events else []
def _query(self, conditions, args, *, limit, descending=False):
connection = self._connect()
try:
order = "DESC" if descending else "ASC"
rows = connection.execute(f"SELECT payload FROM events WHERE {' AND '.join(conditions)} ORDER BY seq {order} LIMIT ?", [*args, max(0, int(limit))]).fetchall()
data = [json.loads(row[0]) for row in rows]
return list(reversed(data)) if descending else data
finally:
connection.close()
async def _messages(self, thread_id, *, run_id=None, limit=50, before_seq=None, after_seq=None):
conditions, args = ["thread_id=?", "category='message'"], [thread_id]
if run_id is not None:
conditions.append("run_id=?")
args.append(run_id)
if before_seq is not None:
conditions.append("seq<?")
args.append(before_seq)
if after_seq is not None:
conditions.append("seq>?")
args.append(after_seq)
return await event_io(self._query, conditions, args, limit=limit, descending=after_seq is None)
async def list_messages(self, thread_id, *, limit=50, before_seq=None, after_seq=None):
return await self._messages(thread_id, limit=limit, before_seq=before_seq, after_seq=after_seq)
async def list_messages_by_run(self, thread_id, run_id, *, limit=50, before_seq=None, after_seq=None, user_id=None):
return await self._messages(thread_id, run_id=run_id, limit=limit, before_seq=before_seq, after_seq=after_seq)
async def list_events(self, thread_id, run_id, *, event_types=None, limit=500):
conditions, args = ["thread_id=?", "run_id=?"], [thread_id, run_id]
if event_types is not None:
if not event_types:
return []
conditions.append(f"event_type IN ({','.join('?' for _ in event_types)})")
args.extend(event_types)
return await event_io(self._query, conditions, args, limit=limit)
def _count_or_delete(self, thread_id, run_id=None, *, delete=False):
connection = self._connect()
try:
conditions, args = "thread_id=?", [thread_id]
if run_id is not None:
conditions += " AND run_id=?"
args.append(run_id)
if delete:
connection.execute("BEGIN IMMEDIATE")
count = connection.execute(f"SELECT COUNT(*) FROM events WHERE {conditions}" + ("" if delete else " AND category='message'"), args).fetchone()[0]
if delete:
connection.execute(f"DELETE FROM events WHERE {conditions}", args)
if run_id is None:
connection.execute("DELETE FROM cursors WHERE thread_id=?", (thread_id,))
connection.commit()
return count
finally:
connection.close()
async def count_messages(self, thread_id):
return await event_io(self._count_or_delete, thread_id)
async def delete_by_thread(self, thread_id):
return await event_io(self._count_or_delete, thread_id, delete=True)
async def delete_by_run(self, thread_id, run_id):
return await event_io(self._count_or_delete, thread_id, run_id, delete=True)