初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,9 @@
from .async_provider import make_checkpointer
from .provider import checkpointer_context, get_checkpointer, reset_checkpointer
__all__ = [
"get_checkpointer",
"reset_checkpointer",
"checkpointer_context",
"make_checkpointer",
]
@@ -0,0 +1,302 @@
"""Async checkpointer factory.
Provides an **async context manager** for long-running async servers that need
proper resource cleanup.
Supported backends: memory, sqlite, postgres.
Usage (e.g. FastAPI lifespan)::
from deerflow.runtime.checkpointer.async_provider import make_checkpointer
async with make_checkpointer() as checkpointer:
app.state.checkpointer = checkpointer # InMemorySaver if not configured
For sync usage see :mod:`deerflow.runtime.checkpointer.provider`.
"""
from __future__ import annotations
import asyncio
import contextlib
import logging
from collections.abc import AsyncIterator
import os
from pathlib import Path
from langgraph.types import Checkpointer
from deerflow.config.app_config import AppConfig, get_app_config
from deerflow.runtime.checkpointer.provider import (
POSTGRES_CONN_REQUIRED,
POSTGRES_INSTALL,
SQLITE_INSTALL,
)
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
logger = logging.getLogger(__name__)
def _csv_env(name: str) -> list[str]:
raw = os.getenv(name, "")
return [part.strip() for part in raw.split(",") if part.strip()]
def _dual_read_enabled(config) -> bool:
if getattr(config, "dual_read_enabled", False):
return True
return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
def _use_postgres_enabled(config) -> bool:
raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
if raw:
return raw.strip().lower() not in {"0", "false", "no", "off"}
return bool(getattr(config, "use_postgres", True))
def _dual_read_paths(config) -> list[str]:
paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
resolved: list[str] = []
seen: set[str] = set()
for path in paths:
conn_str = resolve_sqlite_conn_str(path)
if conn_str == ":memory:" or conn_str.startswith("file:"):
logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
continue
if conn_str not in seen:
resolved.append(conn_str)
seen.add(conn_str)
return resolved
def _sqlite_conn_str_for_postgres_fallback(config) -> str:
configured = getattr(config, "sqlite_connection_string", None)
if configured:
return configured
paths = _dual_read_paths(config)
return paths[0] if paths else "store.db"
def _pool_size_env(name: str, default: int) -> int:
raw = os.getenv(name, "").strip()
if not raw:
return default
try:
return max(0, int(raw))
except ValueError:
logger.warning("Invalid %s=%r; using default %d", name, raw, default)
return default
@contextlib.asynccontextmanager
async def _postgres_saver_from_pool(conn_string: str) -> AsyncIterator[Checkpointer]:
"""Yield an ``AsyncPostgresSaver`` backed by a self-healing connection pool.
``AsyncPostgresSaver.from_conn_string`` opens a *single* long-lived
connection. If that one connection dies (Postgres restart, idle timeout,
network blip, or a prior broken pipeline), every later checkpointer call
fails permanently with "the connection is closed" until the process is
restarted. A pool with ``check=check_connection`` validates each connection
on checkout and transparently replaces dead ones, so the runtime recovers
on its own.
"""
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool
pool = AsyncConnectionPool(
conninfo=conn_string,
min_size=_pool_size_env("DEER_FLOW_PG_POOL_MIN", 1),
max_size=_pool_size_env("DEER_FLOW_PG_POOL_MAX", 10),
max_idle=300.0,
max_lifetime=3600.0,
check=AsyncConnectionPool.check_connection,
kwargs={"autocommit": True, "prepare_threshold": 0, "row_factory": dict_row},
open=False,
)
await pool.open(wait=True)
try:
saver = AsyncPostgresSaver(pool)
await saver.setup()
yield saver
finally:
await pool.close()
@contextlib.asynccontextmanager
async def _sqlite_checkpointer_from_conn_str(raw: str) -> AsyncIterator[Checkpointer]:
try:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = resolve_sqlite_conn_str(raw)
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
await saver.setup()
yield saver
# ---------------------------------------------------------------------------
# Async factory
# ---------------------------------------------------------------------------
@contextlib.asynccontextmanager
async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
"""Async context manager that constructs and tears down a checkpointer."""
if config.type == "memory":
from langgraph.checkpoint.memory import InMemorySaver
yield InMemorySaver()
return
if config.type == "sqlite":
async with _sqlite_checkpointer_from_conn_str(config.connection_string or "store.db") as saver:
yield saver
return
if config.type == "postgres":
if not _use_postgres_enabled(config):
conn_str = _sqlite_conn_str_for_postgres_fallback(config)
logger.info("Checkpointer: use_postgres=false; using SQLite fallback (%s)", conn_str)
async with _sqlite_checkpointer_from_conn_str(conn_str) as saver:
yield saver
return
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver # noqa: F401
except ImportError as exc:
raise ImportError(POSTGRES_INSTALL) from exc
if not config.connection_string:
raise ValueError(POSTGRES_CONN_REQUIRED)
async with _postgres_saver_from_pool(config.connection_string) as saver:
if not _dual_read_enabled(config):
yield saver
return
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from deerflow.runtime.checkpointer.checkpoint_migration import (
DualReadCheckpointer,
SQLiteCheckpointSource,
)
sources: list[SQLiteCheckpointSource] = []
for sqlite_path in _dual_read_paths(config):
if not Path(sqlite_path).exists():
logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
continue
ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
source_saver = await ctx.__aenter__()
await source_saver.setup()
sources.append(
SQLiteCheckpointSource(
path=sqlite_path,
saver=source_saver,
context=ctx,
progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
)
)
if not sources:
logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
yield saver
return
proxy = DualReadCheckpointer(
saver,
sources,
)
try:
yield proxy
finally:
await proxy.aclose_sources()
return
raise ValueError(f"Unknown checkpointer type: {config.type!r}")
# ---------------------------------------------------------------------------
# Public async context manager
# ---------------------------------------------------------------------------
@contextlib.asynccontextmanager
async def _async_checkpointer_from_database(db_config) -> AsyncIterator[Checkpointer]:
"""Async context manager that constructs a checkpointer from unified DatabaseConfig."""
if db_config.backend == "memory":
from langgraph.checkpoint.memory import InMemorySaver
yield InMemorySaver()
return
if db_config.backend == "sqlite":
try:
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = db_config.checkpointer_sqlite_path
ensure_sqlite_parent_dir(conn_str)
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
await saver.setup()
yield saver
return
if db_config.backend == "postgres":
try:
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver # noqa: F401
except ImportError as exc:
raise ImportError(POSTGRES_INSTALL) from exc
if not db_config.postgres_url:
raise ValueError("database.postgres_url is required for the postgres backend")
async with _postgres_saver_from_pool(db_config.postgres_url) as saver:
yield saver
return
raise ValueError(f"Unknown database backend: {db_config.backend!r}")
@contextlib.asynccontextmanager
async def make_checkpointer(app_config: AppConfig | None = None) -> AsyncIterator[Checkpointer]:
"""Async context manager that yields a checkpointer for the caller's lifetime.
Resources are opened on enter and closed on exit -- no global state::
async with make_checkpointer(app_config) as checkpointer:
app.state.checkpointer = checkpointer
Yields an ``InMemorySaver`` when no checkpointer is configured in *config.yaml*.
Priority:
1. Legacy ``checkpointer:`` config section (backward compatible)
2. Unified ``database:`` config section
3. Default InMemorySaver
"""
if app_config is None:
app_config = get_app_config()
# Legacy: standalone checkpointer config takes precedence
if app_config.checkpointer is not None:
async with _async_checkpointer(app_config.checkpointer) as saver:
yield saver
return
# Unified database config
db_config = getattr(app_config, "database", None)
if db_config is not None and db_config.backend != "memory":
async with _async_checkpointer_from_database(db_config) as saver:
yield saver
return
# Default: in-memory
from langgraph.checkpoint.memory import InMemorySaver
yield InMemorySaver()
@@ -0,0 +1,276 @@
"""SQLite-to-Postgres checkpoint migration and read fallback helpers.
The live gateway can wrap a Postgres checkpointer with ``DualReadCheckpointer``.
New writes go to Postgres, reads prefer Postgres and fall back to legacy SQLite.
Manual migration scripts reuse the copy helpers in this module.
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
import sqlite3
from dataclasses import dataclass
from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
@dataclass
class SQLiteCheckpointSource:
path: str
saver: Any
context: Any
progress_path: Path
closed: bool = False
async def close(self) -> None:
if self.closed:
return
self.closed = True
try:
await self.context.__aexit__(None, None, None)
except Exception:
logger.warning("Failed to close legacy SQLite checkpointer: %s", self.path, exc_info=True)
def _sqlite_db_exists(path: str) -> bool:
return Path(path).exists()
def _sqlite_threads(path: str) -> list[str]:
with sqlite3.connect(path) as conn:
rows = conn.execute("SELECT DISTINCT thread_id FROM checkpoints ORDER BY thread_id").fetchall()
return [str(row[0]) for row in rows if row and row[0]]
def _sqlite_has_table(path: str, table: str) -> bool:
with sqlite3.connect(path) as conn:
row = conn.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
(table,),
).fetchone()
return row is not None
def sqlite_recent_threads(path: str, *, since: str | None = None, until: str | None = None) -> list[str]:
"""Return threads updated since an ISO timestamp when ``threads_meta`` exists."""
if not _sqlite_has_table(path, "threads_meta"):
return []
query = "SELECT thread_id FROM threads_meta"
where: list[str] = []
params: list[str] = []
if since:
where.append("updated_at >= ?")
params.append(since)
if until:
where.append("updated_at <= ?")
params.append(until)
if where:
query += " WHERE " + " AND ".join(where)
query += " ORDER BY updated_at DESC"
with sqlite3.connect(path) as conn:
rows = conn.execute(query, tuple(params)).fetchall()
return [str(row[0]) for row in rows if row and row[0]]
def _sqlite_thread_checkpoint_count(path: str, thread_id: str) -> int:
with sqlite3.connect(path) as conn:
row = conn.execute(
"SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?",
(thread_id,),
).fetchone()
return int(row[0] if row else 0)
def _load_progress(path: Path) -> set[str]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError:
return set()
except Exception:
logger.warning("Ignoring unreadable checkpoint migration progress file: %s", path, exc_info=True)
return set()
migrated = payload.get("migrated_threads", [])
return {str(x) for x in migrated if x}
def _save_progress(path: Path, migrated: set[str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_suffix(path.suffix + ".tmp")
tmp.write_text(
json.dumps({"migrated_threads": sorted(migrated)}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
tmp.replace(path)
async def _copy_thread_state(source, target, thread_id: str) -> bool:
"""Copy one thread through the public LangGraph checkpointer API."""
config = {"configurable": {"thread_id": thread_id}}
tuples = [t async for t in source.alist(config)]
if not tuples:
return False
for tup in reversed(tuples):
configurable = (tup.config or {}).get("configurable", {}) or {}
checkpoint_ns = configurable.get("checkpoint_ns", "") or ""
parent_id = ((tup.parent_config or {}).get("configurable", {}) or {}).get("checkpoint_id")
put_config: dict[str, Any] = {"configurable": {"thread_id": thread_id, "checkpoint_ns": checkpoint_ns}}
if parent_id:
put_config["configurable"]["checkpoint_id"] = parent_id
new_versions = (tup.checkpoint or {}).get("channel_versions", {}) or {}
await target.aput(put_config, tup.checkpoint, tup.metadata or {}, new_versions)
if tup.pending_writes:
by_task: dict[str, list] = {}
for task_id, channel, value in tup.pending_writes:
by_task.setdefault(task_id, []).append((channel, value))
checkpoint_id = (tup.checkpoint or {}).get("id")
for task_id, writes in by_task.items():
writes_config = {
"configurable": {
"thread_id": thread_id,
"checkpoint_ns": checkpoint_ns,
"checkpoint_id": checkpoint_id,
}
}
await target.aput_writes(writes_config, writes, task_id)
return True
async def copy_thread_state(source, target, thread_id: str) -> bool:
return await _copy_thread_state(source, target, thread_id)
async def _target_thread_checkpoint_count(target, thread_id: str) -> int:
config = {"configurable": {"thread_id": thread_id}}
count = 0
async for _ in target.alist(config):
count += 1
return count
async def target_has_thread(target, thread_id: str) -> bool:
config = {"configurable": {"thread_id": thread_id}}
async for _ in target.alist(config, limit=1):
return True
return False
class DualReadCheckpointer:
"""Postgres-first checkpointer with SQLite read fallback."""
def __init__(self, primary, sources: list[SQLiteCheckpointSource], *, sleep_seconds: float = 0.05):
self.primary = primary
self.sources = sources
self.sleep_seconds = max(0.0, sleep_seconds)
self._closed = False
def __getattr__(self, name: str):
return getattr(self.primary, name)
async def aclose_sources(self) -> None:
self._closed = True
for source in self.sources:
await source.close()
async def aget_tuple(self, config):
found = await self.primary.aget_tuple(config)
if found is not None:
return found
for source in self.sources:
if source.closed:
continue
found = await source.saver.aget_tuple(config)
if found is not None:
return found
return None
async def aget(self, config):
found = await self.primary.aget(config)
if found is not None:
return found
for source in self.sources:
if source.closed:
continue
found = await source.saver.aget(config)
if found is not None:
return found
return None
async def alist(self, config, *, filter=None, before=None, limit=None):
seen: set[tuple[str, str, str]] = set()
yielded = 0
async for tup in self.primary.alist(config, filter=filter, before=before, limit=limit):
key = self._tuple_key(tup)
seen.add(key)
yield tup
yielded += 1
if limit is not None and yielded >= limit:
return
remaining = None if limit is None else max(0, limit - yielded)
for source in self.sources:
if source.closed:
continue
async for tup in source.saver.alist(config, filter=filter, before=before, limit=remaining):
key = self._tuple_key(tup)
if key in seen:
continue
seen.add(key)
yield tup
yielded += 1
if limit is not None:
remaining = max(0, limit - yielded)
if remaining <= 0:
return
async def aput(self, *args, **kwargs):
return await self.primary.aput(*args, **kwargs)
async def aput_writes(self, *args, **kwargs):
return await self.primary.aput_writes(*args, **kwargs)
async def adelete_thread(self, thread_id: str) -> None:
delete = getattr(self.primary, "adelete_thread", None)
if delete is not None:
await delete(thread_id)
for source in self.sources:
if source.closed:
continue
delete = getattr(source.saver, "adelete_thread", None)
if delete is not None:
await delete(thread_id)
@staticmethod
def _tuple_key(tup) -> tuple[str, str, str]:
configurable = (tup.config or {}).get("configurable", {}) or {}
return (
str(configurable.get("thread_id", "")),
str(configurable.get("checkpoint_ns", "") or ""),
str(configurable.get("checkpoint_id", "") or (tup.checkpoint or {}).get("id", "")),
)
async def _verify_source(self, source: SQLiteCheckpointSource, threads: list[str]) -> bool:
for thread_id in threads:
source_count = await asyncio.to_thread(_sqlite_thread_checkpoint_count, source.path, thread_id)
target_count = await _target_thread_checkpoint_count(self.primary, thread_id)
if target_count < source_count:
logger.warning(
"Checkpoint migration verification failed: source=%s thread_id=%s source_count=%d target_count=%d",
source.path,
thread_id,
source_count,
target_count,
)
return False
return True
@@ -0,0 +1,192 @@
"""Sync checkpointer factory.
Provides a **sync singleton** and a **sync context manager** for LangGraph
graph compilation and CLI tools.
Supported backends: memory, sqlite, postgres.
Usage::
from deerflow.runtime.checkpointer.provider import get_checkpointer, checkpointer_context
# Singleton — reused across calls, closed on process exit
cp = get_checkpointer()
# One-shot — fresh connection, closed on block exit
with checkpointer_context() as cp:
graph.invoke(input, config={"configurable": {"thread_id": "1"}})
"""
from __future__ import annotations
import contextlib
import logging
from collections.abc import Iterator
from langgraph.types import Checkpointer
from deerflow.config.app_config import get_app_config
from deerflow.config.checkpointer_config import CheckpointerConfig
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Error message constants — imported by aio.provider too
# ---------------------------------------------------------------------------
SQLITE_INSTALL = "langgraph-checkpoint-sqlite is required for the SQLite checkpointer. Install it with: uv add langgraph-checkpoint-sqlite"
POSTGRES_INSTALL = "langgraph-checkpoint-postgres is required for the PostgreSQL checkpointer. Install it with: uv add langgraph-checkpoint-postgres psycopg[binary] psycopg-pool"
POSTGRES_CONN_REQUIRED = "checkpointer.connection_string is required for the postgres backend"
# ---------------------------------------------------------------------------
# Sync factory
# ---------------------------------------------------------------------------
@contextlib.contextmanager
def _sync_checkpointer_cm(config: CheckpointerConfig) -> Iterator[Checkpointer]:
"""Context manager that creates and tears down a sync checkpointer.
Returns a configured ``Checkpointer`` instance. Resource cleanup for any
underlying connections or pools is handled by higher-level helpers in
this module (such as the singleton factory or context manager); this
function does not return a separate cleanup callback.
"""
if config.type == "memory":
from langgraph.checkpoint.memory import InMemorySaver
logger.info("Checkpointer: using InMemorySaver (in-process, not persistent)")
yield InMemorySaver()
return
if config.type == "sqlite":
try:
from langgraph.checkpoint.sqlite import SqliteSaver
except ImportError as exc:
raise ImportError(SQLITE_INSTALL) from exc
conn_str = resolve_sqlite_conn_str(config.connection_string or "store.db")
ensure_sqlite_parent_dir(conn_str)
with SqliteSaver.from_conn_string(conn_str) as saver:
saver.setup()
logger.info("Checkpointer: using SqliteSaver (%s)", conn_str)
yield saver
return
if config.type == "postgres":
try:
from langgraph.checkpoint.postgres import PostgresSaver
except ImportError as exc:
raise ImportError(POSTGRES_INSTALL) from exc
if not config.connection_string:
raise ValueError(POSTGRES_CONN_REQUIRED)
with PostgresSaver.from_conn_string(config.connection_string) as saver:
saver.setup()
logger.info("Checkpointer: using PostgresSaver")
yield saver
return
raise ValueError(f"Unknown checkpointer type: {config.type!r}")
# ---------------------------------------------------------------------------
# Sync singleton
# ---------------------------------------------------------------------------
_checkpointer: Checkpointer | None = None
_checkpointer_ctx = None # open context manager keeping the connection alive
def get_checkpointer() -> Checkpointer:
"""Return the global sync checkpointer singleton, creating it on first call.
Returns an ``InMemorySaver`` when no checkpointer is configured in *config.yaml*.
Raises:
ImportError: If the required package for the configured backend is not installed.
ValueError: If ``connection_string`` is missing for a backend that requires it.
"""
global _checkpointer, _checkpointer_ctx
if _checkpointer is not None:
return _checkpointer
# Ensure app config is loaded before checking checkpointer config
# This prevents returning InMemorySaver when config.yaml actually has a checkpointer section
# but hasn't been loaded yet
from deerflow.config.app_config import _app_config
from deerflow.config.checkpointer_config import get_checkpointer_config
config = get_checkpointer_config()
if config is None and _app_config is None:
# Only load app config lazily when neither the app config nor an explicit
# checkpointer config has been initialized yet. This keeps tests that
# intentionally set the global checkpointer config isolated from any
# ambient config.yaml on disk.
try:
get_app_config()
except FileNotFoundError:
# In test environments without config.yaml, this is expected.
pass
config = get_checkpointer_config()
if config is None:
from langgraph.checkpoint.memory import InMemorySaver
logger.info("Checkpointer: using InMemorySaver (in-process, not persistent)")
_checkpointer = InMemorySaver()
return _checkpointer
_checkpointer_ctx = _sync_checkpointer_cm(config)
_checkpointer = _checkpointer_ctx.__enter__()
return _checkpointer
def reset_checkpointer() -> None:
"""Reset the sync singleton, forcing recreation on the next call.
Closes any open backend connections and clears the cached instance.
Useful in tests or after a configuration change.
"""
global _checkpointer, _checkpointer_ctx
if _checkpointer_ctx is not None:
try:
_checkpointer_ctx.__exit__(None, None, None)
except Exception:
logger.warning("Error during checkpointer cleanup", exc_info=True)
_checkpointer_ctx = None
_checkpointer = None
# ---------------------------------------------------------------------------
# Sync context manager
# ---------------------------------------------------------------------------
@contextlib.contextmanager
def checkpointer_context() -> Iterator[Checkpointer]:
"""Sync context manager that yields a checkpointer and cleans up on exit.
Unlike :func:`get_checkpointer`, this does **not** cache the instance —
each ``with`` block creates and destroys its own connection. Use it in
CLI scripts or tests where you want deterministic cleanup::
with checkpointer_context() as cp:
graph.invoke(input, config={"configurable": {"thread_id": "1"}})
Yields an ``InMemorySaver`` when no checkpointer is configured in *config.yaml*.
"""
config = get_app_config()
if config.checkpointer is None:
from langgraph.checkpoint.memory import InMemorySaver
yield InMemorySaver()
return
with _sync_checkpointer_cm(config.checkpointer) as saver:
yield saver