This commit is contained in:
董宏凯 committed 2026-06-05 19:56:51 +08:00
commit 7b6dd9ba6f
23 files changed
+1888 -572

No files matched your search

@@ -2,7 +2,7 @@
from typing import Literal
from pydantic import BaseModel, Field
from pydantic import BaseModel, Field
CheckpointerType = Literal["memory", "sqlite", "postgres"]
@@ -16,13 +16,41 @@ class CheckpointerConfig(BaseModel):
"'sqlite' persists to a local file (requires langgraph-checkpoint-sqlite). "
"'postgres' persists to PostgreSQL (requires langgraph-checkpoint-postgres)."
)
connection_string: str | None = Field(
default=None,
description="Connection string for sqlite (file path) or postgres (DSN). "
connection_string: str | None = Field(
default=None,
description="Connection string for sqlite (file path) or postgres (DSN). "
"Required for sqlite and postgres types. "
"For sqlite, use a file path like '.deer-flow/checkpoints.db' or ':memory:' for in-memory. "
"For postgres, use a DSN like 'postgresql://user:pass@localhost:5432/db'.",
)
"For sqlite, use a file path like '.deer-flow/checkpoints.db' or ':memory:' for in-memory. "
"For postgres, use a DSN like 'postgresql://user:pass@localhost:5432/db'.",
)
use_postgres: bool = Field(
default=True,
description=(
"When type='postgres', set false to keep the Postgres DSN in config "
"but run the checkpointer on SQLite instead."
),
)
sqlite_connection_string: str | None = Field(
default=None,
description=(
"SQLite file used when type='postgres' and use_postgres=false. "
"Defaults to the first dual_read_sqlite_paths entry or 'store.db'."
),
)
dual_read_enabled: bool = Field(
default=False,
description=(
"When type='postgres', read from Postgres first and fall back to legacy "
"SQLite checkpoint files when Postgres does not have the requested data."
),
)
dual_read_sqlite_paths: list[str] = Field(
default_factory=list,
description=(
"Legacy SQLite checkpoint database paths used for read fallback. "
"Plain paths are resolved the same way as SQLite connection_string."
),
)
# Global configuration instance — None means no checkpointer is configured.
@@ -51,6 +51,14 @@ class AppearanceSettings(BaseModel):
default="light",
description="Theme applied for users who haven't picked their own (light / dark)",
)
default_skill_view: Literal["card", "list"] = Field(
default="list",
description="Default skill list display mode for users who haven't picked their own (card / list)",
)
default_agent_view: Literal["card", "list"] = Field(
default="card",
description="Default agent list display mode for users who haven't picked their own (card / list)",
)
model_config = ConfigDict(extra="ignore")
@@ -17,10 +17,12 @@ For sync usage see :mod:`deerflow.runtime.checkpointer.provider`.
from __future__ import annotations
import asyncio
import contextlib
import logging
from collections.abc import AsyncIterator
import asyncio
import contextlib
import logging
from collections.abc import AsyncIterator
import os
from pathlib import Path
from langgraph.types import Checkpointer
@@ -30,13 +32,70 @@ from deerflow.runtime.checkpointer.provider import (
POSTGRES_INSTALL,
SQLITE_INSTALL,
)
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Async factory
# ---------------------------------------------------------------------------
logger = logging.getLogger(__name__)
def _csv_env(name: str) -> list[str]:
raw = os.getenv(name, "")
return [part.strip() for part in raw.split(",") if part.strip()]
def _dual_read_enabled(config) -> bool:
if getattr(config, "dual_read_enabled", False):
return True
return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
def _use_postgres_enabled(config) -> bool:
raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
if raw:
return raw.strip().lower() not in {"0", "false", "no", "off"}
return bool(getattr(config, "use_postgres", True))
def _dual_read_paths(config) -> list[str]:
paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
resolved: list[str] = []
seen: set[str] = set()
for path in paths:
conn_str = resolve_sqlite_conn_str(path)
if conn_str == ":memory:" or conn_str.startswith("file:"):
logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
continue
if conn_str not in seen:
resolved.append(conn_str)
seen.add(conn_str)
return resolved
def _sqlite_conn_str_for_postgres_fallback(config) -> str:
configured = getattr(config, "sqlite_connection_string", None)
if configured:
return configured
paths = _dual_read_paths(config)
return paths[0] if paths else "store.db"
@contextlib.asynccontextmanager
async def _sqlite_checkpointer_from_conn_str(raw: str) -> AsyncIterator[Checkpointer]:
try:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = resolve_sqlite_conn_str(raw)
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
await saver.setup()
yield saver
# ---------------------------------------------------------------------------
# Async factory
# ---------------------------------------------------------------------------
@contextlib.asynccontextmanager
@@ -48,32 +107,71 @@ async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
yield InMemorySaver()
return
if config.type == "sqlite":
try:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = resolve_sqlite_conn_str(config.connection_string or "store.db")
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
await saver.setup()
yield saver
return
if config.type == "postgres":
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
if config.type == "sqlite":
async with _sqlite_checkpointer_from_conn_str(config.connection_string or "store.db") as saver:
yield saver
return
if config.type == "postgres":
if not _use_postgres_enabled(config):
conn_str = _sqlite_conn_str_for_postgres_fallback(config)
logger.info("Checkpointer: use_postgres=false; using SQLite fallback (%s)", conn_str)
async with _sqlite_checkpointer_from_conn_str(conn_str) as saver:
yield saver
return
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
except ImportError as exc:
raise ImportError(POSTGRES_INSTALL) from exc
if not config.connection_string:
raise ValueError(POSTGRES_CONN_REQUIRED)
async with AsyncPostgresSaver.from_conn_string(config.connection_string) as saver:
await saver.setup()
yield saver
return
if not config.connection_string:
raise ValueError(POSTGRES_CONN_REQUIRED)
async with AsyncPostgresSaver.from_conn_string(config.connection_string) as saver:
await saver.setup()
if not _dual_read_enabled(config):
yield saver
return
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from deerflow.runtime.checkpointer.checkpoint_migration import (
DualReadCheckpointer,
SQLiteCheckpointSource,
)
sources: list[SQLiteCheckpointSource] = []
for sqlite_path in _dual_read_paths(config):
if not Path(sqlite_path).exists():
logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
continue
ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
source_saver = await ctx.__aenter__()
await source_saver.setup()
sources.append(
SQLiteCheckpointSource(
path=sqlite_path,
saver=source_saver,
context=ctx,
progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
)
)
if not sources:
logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
yield saver
return
proxy = DualReadCheckpointer(
saver,
sources,
)
try:
yield proxy
finally:
await proxy.aclose_sources()
return
raise ValueError(f"Unknown checkpointer type: {config.type!r}")
@@ -0,0 +1,276 @@
"""SQLite-to-Postgres checkpoint migration and read fallback helpers.
The live gateway can wrap a Postgres checkpointer with ``DualReadCheckpointer``.
New writes go to Postgres, reads prefer Postgres and fall back to legacy SQLite.
Manual migration scripts reuse the copy helpers in this module.
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
import sqlite3
from dataclasses import dataclass
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
@dataclass
class SQLiteCheckpointSource:
path: str
saver: Any
context: Any
progress_path: Path
closed: bool = False
async def close(self) -> None:
if self.closed:
return
self.closed = True
try:
await self.context.__aexit__(None, None, None)
except Exception:
logger.warning("Failed to close legacy SQLite checkpointer: %s", self.path, exc_info=True)
def _sqlite_db_exists(path: str) -> bool:
return Path(path).exists()
def _sqlite_threads(path: str) -> list[str]:
with sqlite3.connect(path) as conn:
rows = conn.execute("SELECT DISTINCT thread_id FROM checkpoints ORDER BY thread_id").fetchall()
return [str(row[0]) for row in rows if row and row[0]]
def _sqlite_has_table(path: str, table: str) -> bool:
with sqlite3.connect(path) as conn:
row = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
(table,),
).fetchone()
return row is not None
def sqlite_recent_threads(path: str, *, since: str | None = None, until: str | None = None) -> list[str]:
"""Return threads updated since an ISO timestamp when ``threads_meta`` exists."""
if not _sqlite_has_table(path, "threads_meta"):
return []
query = "SELECT thread_id FROM threads_meta"
where: list[str] = []
params: list[str] = []
if since:
where.append("updated_at >= ?")
params.append(since)
if until:
where.append("updated_at <= ?")
params.append(until)
if where:
query += " WHERE " + " AND ".join(where)
query += " ORDER BY updated_at DESC"
with sqlite3.connect(path) as conn:
rows = conn.execute(query, tuple(params)).fetchall()
return [str(row[0]) for row in rows if row and row[0]]
def _sqlite_thread_checkpoint_count(path: str, thread_id: str) -> int:
with sqlite3.connect(path) as conn:
row = conn.execute(
"SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?",
(thread_id,),
).fetchone()
return int(row[0] if row else 0)
def _load_progress(path: Path) -> set[str]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError:
return set()
except Exception:
logger.warning("Ignoring unreadable checkpoint migration progress file: %s", path, exc_info=True)
return set()
migrated = payload.get("migrated_threads", [])
return {str(x) for x in migrated if x}
def _save_progress(path: Path, migrated: set[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(
json.dumps({"migrated_threads": sorted(migrated)}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
tmp.replace(path)
async def _copy_thread_state(source, target, thread_id: str) -> bool:
"""Copy one thread through the public LangGraph checkpointer API."""
config = {"configurable": {"thread_id": thread_id}}
tuples = [t async for t in source.alist(config)]
if not tuples:
return False
for tup in reversed(tuples):
configurable = (tup.config or {}).get("configurable", {}) or {}
checkpoint_ns = configurable.get("checkpoint_ns", "") or ""
parent_id = ((tup.parent_config or {}).get("configurable", {}) or {}).get("checkpoint_id")
put_config: dict[str, Any] = {"configurable": {"thread_id": thread_id, "checkpoint_ns": checkpoint_ns}}
if parent_id:
put_config["configurable"]["checkpoint_id"] = parent_id
new_versions = (tup.checkpoint or {}).get("channel_versions", {}) or {}
await target.aput(put_config, tup.checkpoint, tup.metadata or {}, new_versions)
if tup.pending_writes:
by_task: dict[str, list] = {}
for task_id, channel, value in tup.pending_writes:
by_task.setdefault(task_id, []).append((channel, value))
checkpoint_id = (tup.checkpoint or {}).get("id")
for task_id, writes in by_task.items():
writes_config = {
"configurable": {
"thread_id": thread_id,
"checkpoint_ns": checkpoint_ns,
"checkpoint_id": checkpoint_id,
}
}
await target.aput_writes(writes_config, writes, task_id)
return True
async def copy_thread_state(source, target, thread_id: str) -> bool:
return await _copy_thread_state(source, target, thread_id)
async def _target_thread_checkpoint_count(target, thread_id: str) -> int:
config = {"configurable": {"thread_id": thread_id}}
count = 0
async for _ in target.alist(config):
count += 1
return count
async def target_has_thread(target, thread_id: str) -> bool:
config = {"configurable": {"thread_id": thread_id}}
async for _ in target.alist(config, limit=1):
return True
return False
class DualReadCheckpointer:
"""Postgres-first checkpointer with SQLite read fallback."""
def __init__(self, primary, sources: list[SQLiteCheckpointSource], *, sleep_seconds: float = 0.05):
self.primary = primary
self.sources = sources
self.sleep_seconds = max(0.0, sleep_seconds)
self._closed = False
def __getattr__(self, name: str):
return getattr(self.primary, name)
async def aclose_sources(self) -> None:
self._closed = True
for source in self.sources:
await source.close()
async def aget_tuple(self, config):
found = await self.primary.aget_tuple(config)
if found is not None:
return found
for source in self.sources:
if source.closed:
continue
found = await source.saver.aget_tuple(config)
if found is not None:
return found
return None
async def aget(self, config):
found = await self.primary.aget(config)
if found is not None:
return found
for source in self.sources:
if source.closed:
continue
found = await source.saver.aget(config)
if found is not None:
return found
return None
async def alist(self, config, *, filter=None, before=None, limit=None):
seen: set[tuple[str, str, str]] = set()
yielded = 0
async for tup in self.primary.alist(config, filter=filter, before=before, limit=limit):
key = self._tuple_key(tup)
seen.add(key)
yield tup
yielded += 1
if limit is not None and yielded >= limit:
return
remaining = None if limit is None else max(0, limit - yielded)
for source in self.sources:
if source.closed:
continue
async for tup in source.saver.alist(config, filter=filter, before=before, limit=remaining):
key = self._tuple_key(tup)
if key in seen:
continue
seen.add(key)
yield tup
yielded += 1
if limit is not None:
remaining = max(0, limit - yielded)
if remaining <= 0:
return
async def aput(self, *args, **kwargs):
return await self.primary.aput(*args, **kwargs)
async def aput_writes(self, *args, **kwargs):
return await self.primary.aput_writes(*args, **kwargs)
async def adelete_thread(self, thread_id: str) -> None:
delete = getattr(self.primary, "adelete_thread", None)
if delete is not None:
await delete(thread_id)
for source in self.sources:
if source.closed:
continue
delete = getattr(source.saver, "adelete_thread", None)
if delete is not None:
await delete(thread_id)
@staticmethod
def _tuple_key(tup) -> tuple[str, str, str]:
configurable = (tup.config or {}).get("configurable", {}) or {}
return (
str(configurable.get("thread_id", "")),
str(configurable.get("checkpoint_ns", "") or ""),
str(configurable.get("checkpoint_id", "") or (tup.checkpoint or {}).get("id", "")),
)
async def _verify_source(self, source: SQLiteCheckpointSource, threads: list[str]) -> bool:
for thread_id in threads:
source_count = await asyncio.to_thread(_sqlite_thread_checkpoint_count, source.path, thread_id)
target_count = await _target_thread_checkpoint_count(self.primary, thread_id)
if target_count < source_count:
logger.warning(
"Checkpoint migration verification failed: source=%s thread_id=%s source_count=%d target_count=%d",
source.path,
thread_id,
source_count,
target_count,
)
return False
return True
@@ -31,9 +31,12 @@ dependencies = [
"ddgs>=9.10.0",
"duckdb>=1.4.4",
"langchain-google-genai>=4.2.1",
"langgraph-checkpoint-sqlite>=3.0.3",
"langgraph-sdk>=0.1.51",
"sqlalchemy[asyncio]>=2.0,<3.0",
"langgraph-checkpoint-sqlite>=3.0.3",
"langgraph-checkpoint-postgres>=3.0.5",
"langgraph-sdk>=0.1.51",
"psycopg[binary]>=3.3.3",
"psycopg-pool>=3.3.0",
"sqlalchemy[asyncio]>=2.0,<3.0",
"aiosqlite>=0.19",
"alembic>=1.13",
]