feat: add checkpoint postgres migration admin
This commit is contained in:
1 parent
eaa061dd06
commit
9471e72171
10 files changed
+962
-33
No files matched your search
+86
-16
@@ -17,10 +17,12 @@ For sync usage see :mod:`deerflow.runtime.checkpointer.provider`.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from langgraph.types import Checkpointer
|
||||
|
||||
@@ -30,9 +32,37 @@ from deerflow.runtime.checkpointer.provider import (
|
||||
POSTGRES_INSTALL,
|
||||
SQLITE_INSTALL,
|
||||
)
|
||||
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
|
||||
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _csv_env(name: str) -> list[str]:
|
||||
raw = os.getenv(name, "")
|
||||
return [part.strip() for part in raw.split(",") if part.strip()]
|
||||
|
||||
|
||||
def _dual_read_enabled(config) -> bool:
|
||||
if getattr(config, "dual_read_enabled", False):
|
||||
return True
|
||||
return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _dual_read_paths(config) -> list[str]:
|
||||
paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
|
||||
paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
|
||||
resolved: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for path in paths:
|
||||
conn_str = resolve_sqlite_conn_str(path)
|
||||
if conn_str == ":memory:" or conn_str.startswith("file:"):
|
||||
logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
|
||||
continue
|
||||
if conn_str not in seen:
|
||||
resolved.append(conn_str)
|
||||
seen.add(conn_str)
|
||||
return resolved
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async factory
|
||||
@@ -61,19 +91,59 @@ async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
|
||||
yield saver
|
||||
return
|
||||
|
||||
if config.type == "postgres":
|
||||
try:
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
|
||||
if config.type == "postgres":
|
||||
try:
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
|
||||
except ImportError as exc:
|
||||
raise ImportError(POSTGRES_INSTALL) from exc
|
||||
|
||||
if not config.connection_string:
|
||||
raise ValueError(POSTGRES_CONN_REQUIRED)
|
||||
|
||||
async with AsyncPostgresSaver.from_conn_string(config.connection_string) as saver:
|
||||
await saver.setup()
|
||||
yield saver
|
||||
return
|
||||
if not config.connection_string:
|
||||
raise ValueError(POSTGRES_CONN_REQUIRED)
|
||||
|
||||
async with AsyncPostgresSaver.from_conn_string(config.connection_string) as saver:
|
||||
await saver.setup()
|
||||
if not _dual_read_enabled(config):
|
||||
yield saver
|
||||
return
|
||||
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
from deerflow.runtime.checkpointer.checkpoint_migration import (
|
||||
DualReadCheckpointer,
|
||||
SQLiteCheckpointSource,
|
||||
)
|
||||
|
||||
sources: list[SQLiteCheckpointSource] = []
|
||||
for sqlite_path in _dual_read_paths(config):
|
||||
if not Path(sqlite_path).exists():
|
||||
logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
|
||||
continue
|
||||
ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
|
||||
source_saver = await ctx.__aenter__()
|
||||
await source_saver.setup()
|
||||
sources.append(
|
||||
SQLiteCheckpointSource(
|
||||
path=sqlite_path,
|
||||
saver=source_saver,
|
||||
context=ctx,
|
||||
progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
|
||||
)
|
||||
)
|
||||
|
||||
if not sources:
|
||||
logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
|
||||
yield saver
|
||||
return
|
||||
|
||||
proxy = DualReadCheckpointer(
|
||||
saver,
|
||||
sources,
|
||||
)
|
||||
try:
|
||||
yield proxy
|
||||
finally:
|
||||
await proxy.aclose_sources()
|
||||
return
|
||||
|
||||
raise ValueError(f"Unknown checkpointer type: {config.type!r}")
|
||||
|
||||
|
||||
Reference in new issue
Block a user