Merge branch 'ji' of http://192.168.0.200/knowledge-base-bj/deerflow-server into ji
This commit is contained in:
commit
7b6dd9ba6f
23 files changed
+1888
-572
No files matched your search
@@ -30,8 +30,9 @@ from app.gateway.routers import (
|
||||
artifacts,
|
||||
assistants_compat,
|
||||
auth,
|
||||
channels,
|
||||
feedback,
|
||||
channels,
|
||||
checkpoint_migration,
|
||||
feedback,
|
||||
html_page_favorites,
|
||||
intent,
|
||||
llm_metrics,
|
||||
@@ -906,7 +907,10 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
app.include_router(tool_metrics.router)
|
||||
|
||||
# Channels API is mounted at /api/channels
|
||||
app.include_router(channels.router)
|
||||
app.include_router(channels.router)
|
||||
|
||||
# Admin checkpoint migration API: SQLite legacy checkpoints -> Postgres.
|
||||
app.include_router(checkpoint_migration.router)
|
||||
|
||||
# Assistants compatibility API (LangGraph Platform stub)
|
||||
app.include_router(assistants_compat.router)
|
||||
@@ -946,7 +950,11 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
app.include_router(recommend.router) # Step 1→2:/api/recommend/stream(无 init)
|
||||
# Step 3 产物列表/下载复用下方 artifacts.router(/api/threads/{id}/artifacts)
|
||||
|
||||
# 探活端点:无需鉴权(见 auth_middleware._PUBLIC_PATH_PREFIXES)。
|
||||
# 同时暴露裸 ``/health`` 与带 API 前缀的 ``/api/health``,方便监控按
|
||||
# 任意 base URL 直接探活,不会被认证中间件拦成 401。
|
||||
@app.get("/health", tags=["health"])
|
||||
@app.get("/api/health", tags=["health"])
|
||||
async def health_check() -> dict:
|
||||
"""Health check endpoint.
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
# safe read-only routes under it — everything there bypasses auth entirely.
|
||||
_PUBLIC_PATH_PREFIXES: tuple[str, ...] = (
|
||||
"/health",
|
||||
"/api/health",
|
||||
"/docs",
|
||||
"/redoc",
|
||||
"/openapi.json",
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Admin checkpoint migration endpoints.
|
||||
|
||||
These endpoints support the SQLite -> Postgres transition for LangGraph
|
||||
checkpoints. The live checkpointer should be Postgres with SQLite dual-read
|
||||
enabled; this router only starts manual, bounded migrations for selected time
|
||||
ranges.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.auth.disabled_mode import is_auth_disabled
|
||||
from app.gateway.deps import get_current_user_from_request
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.runtime.checkpointer.checkpoint_migration import (
|
||||
SQLiteCheckpointSource,
|
||||
copy_thread_state,
|
||||
sqlite_recent_threads,
|
||||
target_has_thread,
|
||||
)
|
||||
from deerflow.runtime.store._sqlite_utils import resolve_sqlite_conn_str
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/admin/checkpoint-migration", tags=["checkpoint-migration"])
|
||||
|
||||
_TASKS: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
class MigrationStartRequest(BaseModel):
|
||||
start_at: str = Field(description="Inclusive start datetime, ISO string.")
|
||||
end_at: str = Field(description="Inclusive end datetime, ISO string.")
|
||||
source_paths: list[str] | None = Field(default=None, description="Optional SQLite sources. Defaults to configured dual-read paths.")
|
||||
|
||||
|
||||
class MigrationTaskResponse(BaseModel):
|
||||
task_id: str
|
||||
status: str
|
||||
start_at: str
|
||||
end_at: str
|
||||
source_paths: list[str]
|
||||
total_threads: int = 0
|
||||
copied_threads: int = 0
|
||||
skipped_threads: int = 0
|
||||
failed_threads: int = 0
|
||||
skipped_ranges: int = 0
|
||||
message: str = ""
|
||||
error: str | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
async def _require_admin(request: Request) -> None:
|
||||
if is_auth_disabled():
|
||||
return
|
||||
user = await get_current_user_from_request(request)
|
||||
if getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="Checkpoint migration requires admin role")
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _parse_dt(value: str) -> datetime:
|
||||
try:
|
||||
normalized = value.replace("Z", "+00:00")
|
||||
dt = datetime.fromisoformat(normalized)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid datetime: {value}") from exc
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
def _sqlite_datetime(dt: datetime) -> str:
|
||||
# Existing SQLite rows store naive wall-clock strings like
|
||||
# "2026-05-18 06:41:10.989427"; match that format for comparisons.
|
||||
return dt.astimezone(timezone.utc).replace(tzinfo=None).isoformat(sep=" ")
|
||||
|
||||
|
||||
def _configured_sources(request: Request) -> list[str]:
|
||||
config = getattr(request.app.state, "config", None)
|
||||
cp = getattr(config, "checkpointer", None)
|
||||
paths = list(getattr(cp, "dual_read_sqlite_paths", []) or [])
|
||||
raw_env = os.getenv("DEER_FLOW_CHECKPOINT_SQLITE_PATHS", "")
|
||||
paths.extend(part.strip() for part in raw_env.split(",") if part.strip())
|
||||
resolved: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for path in paths:
|
||||
conn_str = resolve_sqlite_conn_str(path)
|
||||
if conn_str == ":memory:" or conn_str.startswith("file:"):
|
||||
continue
|
||||
if conn_str not in seen:
|
||||
resolved.append(conn_str)
|
||||
seen.add(conn_str)
|
||||
return resolved
|
||||
|
||||
|
||||
def _target_checkpointer(request: Request):
|
||||
checkpointer = getattr(request.app.state, "checkpointer", None)
|
||||
if checkpointer is None:
|
||||
raise HTTPException(status_code=503, detail="Checkpointer is not available")
|
||||
return getattr(checkpointer, "primary", checkpointer)
|
||||
|
||||
|
||||
def _state_path() -> Path:
|
||||
return runtime_home() / "checkpoint_migration_state.json"
|
||||
|
||||
|
||||
def _load_completed_ranges() -> set[str]:
|
||||
try:
|
||||
payload = json.loads(_state_path().read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return set()
|
||||
except Exception:
|
||||
logger.warning("Ignoring unreadable checkpoint migration state file", exc_info=True)
|
||||
return set()
|
||||
return {str(item) for item in payload.get("completed_ranges", [])}
|
||||
|
||||
|
||||
def _save_completed_ranges(items: set[str]) -> None:
|
||||
path = _state_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps({"completed_ranges": sorted(items)}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def _range_key(source: str, start_at: str, end_at: str) -> str:
|
||||
raw = json.dumps({"source": str(Path(source).resolve()), "start_at": start_at, "end_at": end_at}, sort_keys=True)
|
||||
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
async def _open_sqlite_source(path: str) -> SQLiteCheckpointSource:
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
ctx = AsyncSqliteSaver.from_conn_string(path)
|
||||
saver = await ctx.__aenter__()
|
||||
await saver.setup()
|
||||
return SQLiteCheckpointSource(
|
||||
path=path,
|
||||
saver=saver,
|
||||
context=ctx,
|
||||
progress_path=Path(path).with_suffix(Path(path).suffix + ".pg-migration.json"),
|
||||
)
|
||||
|
||||
|
||||
def _task_response(task: dict[str, Any]) -> MigrationTaskResponse:
|
||||
return MigrationTaskResponse(**task)
|
||||
|
||||
|
||||
async def _run_task(task_id: str, target, start_at: datetime, end_at: datetime) -> None:
|
||||
task = _TASKS[task_id]
|
||||
completed = await asyncio.to_thread(_load_completed_ranges)
|
||||
start_sql = _sqlite_datetime(start_at)
|
||||
end_sql = _sqlite_datetime(end_at)
|
||||
range_start = start_at.isoformat()
|
||||
range_end = end_at.isoformat()
|
||||
|
||||
try:
|
||||
task["status"] = "running"
|
||||
task["updated_at"] = _now_iso()
|
||||
for source_path in task["source_paths"]:
|
||||
if not Path(source_path).exists():
|
||||
task["message"] = f"SQLite source not found: {source_path}"
|
||||
task["skipped_ranges"] += 1
|
||||
continue
|
||||
|
||||
key = _range_key(source_path, range_start, range_end)
|
||||
if key in completed:
|
||||
task["skipped_ranges"] += 1
|
||||
task["message"] = "Selected range was already migrated; skipped"
|
||||
task["updated_at"] = _now_iso()
|
||||
continue
|
||||
|
||||
thread_ids = await asyncio.to_thread(sqlite_recent_threads, source_path, since=start_sql, until=end_sql)
|
||||
if not thread_ids:
|
||||
task["message"] = f"No threads found in selected range for {Path(source_path).name}"
|
||||
completed.add(key)
|
||||
await asyncio.to_thread(_save_completed_ranges, completed)
|
||||
task["updated_at"] = _now_iso()
|
||||
continue
|
||||
|
||||
task["total_threads"] += len(thread_ids)
|
||||
source = await _open_sqlite_source(source_path)
|
||||
try:
|
||||
for thread_id in thread_ids:
|
||||
if await target_has_thread(target, thread_id):
|
||||
task["skipped_threads"] += 1
|
||||
else:
|
||||
copied = await copy_thread_state(source.saver, target, thread_id)
|
||||
if copied:
|
||||
task["copied_threads"] += 1
|
||||
else:
|
||||
task["skipped_threads"] += 1
|
||||
task["updated_at"] = _now_iso()
|
||||
await asyncio.sleep(0)
|
||||
finally:
|
||||
with contextlib.suppress(Exception):
|
||||
await source.close()
|
||||
|
||||
completed.add(key)
|
||||
await asyncio.to_thread(_save_completed_ranges, completed)
|
||||
|
||||
task["status"] = "completed"
|
||||
task["message"] = task["message"] or "Migration completed"
|
||||
except Exception as exc:
|
||||
logger.exception("Checkpoint migration task failed: %s", task_id)
|
||||
task["status"] = "failed"
|
||||
task["error"] = str(exc)
|
||||
finally:
|
||||
task["updated_at"] = _now_iso()
|
||||
|
||||
|
||||
@router.post("/tasks", response_model=MigrationTaskResponse)
|
||||
async def start_migration(body: MigrationStartRequest, request: Request):
|
||||
await _require_admin(request)
|
||||
start_at = _parse_dt(body.start_at)
|
||||
end_at = _parse_dt(body.end_at)
|
||||
if end_at <= start_at:
|
||||
raise HTTPException(status_code=400, detail="end_at must be later than start_at")
|
||||
|
||||
sources = [resolve_sqlite_conn_str(p) for p in (body.source_paths or _configured_sources(request))]
|
||||
sources = [p for p in dict.fromkeys(sources) if p != ":memory:" and not p.startswith("file:")]
|
||||
if not sources:
|
||||
raise HTTPException(status_code=400, detail="No SQLite source paths configured")
|
||||
|
||||
target = _target_checkpointer(request)
|
||||
task_id = hashlib.sha256(f"{_now_iso()}:{sources}:{body.start_at}:{body.end_at}".encode("utf-8")).hexdigest()[:16]
|
||||
now = _now_iso()
|
||||
_TASKS[task_id] = {
|
||||
"task_id": task_id,
|
||||
"status": "queued",
|
||||
"start_at": start_at.isoformat(),
|
||||
"end_at": end_at.isoformat(),
|
||||
"source_paths": sources,
|
||||
"total_threads": 0,
|
||||
"copied_threads": 0,
|
||||
"skipped_threads": 0,
|
||||
"failed_threads": 0,
|
||||
"skipped_ranges": 0,
|
||||
"message": "",
|
||||
"error": None,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
asyncio.create_task(_run_task(task_id, target, start_at, end_at))
|
||||
return _task_response(_TASKS[task_id])
|
||||
|
||||
|
||||
@router.get("/tasks/{task_id}", response_model=MigrationTaskResponse)
|
||||
async def get_migration_task(task_id: str, request: Request):
|
||||
await _require_admin(request)
|
||||
task = _TASKS.get(task_id)
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail="Migration task not found")
|
||||
return _task_response(task)
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
async def list_sources(request: Request):
|
||||
await _require_admin(request)
|
||||
return {"source_paths": _configured_sources(request)}
|
||||
@@ -223,17 +223,28 @@ class AppearanceResponse(BaseModel):
|
||||
"""Effective system-wide appearance defaults."""
|
||||
|
||||
default_theme: Literal["light", "dark"] = Field(..., description="System default theme — light or dark")
|
||||
default_skill_view: Literal["card", "list"] = Field(..., description="System default skill list display mode — card or list")
|
||||
default_agent_view: Literal["card", "list"] = Field(..., description="System default agent list display mode — card or list")
|
||||
|
||||
|
||||
class AppearanceUpdateRequest(BaseModel):
|
||||
default_theme: Literal["light", "dark"] | None = Field(default=None)
|
||||
default_skill_view: Literal["card", "list"] | None = Field(default=None)
|
||||
default_agent_view: Literal["card", "list"] | None = Field(default=None)
|
||||
|
||||
|
||||
def _appearance_response(appearance) -> AppearanceResponse:
|
||||
return AppearanceResponse(
|
||||
default_theme=appearance.default_theme,
|
||||
default_skill_view=appearance.default_skill_view,
|
||||
default_agent_view=appearance.default_agent_view,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/appearance", response_model=AppearanceResponse)
|
||||
async def get_appearance_settings() -> AppearanceResponse:
|
||||
"""Return the global default theme. Any logged-in user may read this."""
|
||||
settings = load_system_settings().appearance
|
||||
return AppearanceResponse(default_theme=settings.default_theme)
|
||||
"""Return the global appearance defaults. Any logged-in user may read this."""
|
||||
return _appearance_response(load_system_settings().appearance)
|
||||
|
||||
|
||||
@router.put("/appearance", response_model=AppearanceResponse)
|
||||
@@ -246,8 +257,9 @@ async def update_appearance_settings(
|
||||
current = load_system_settings()
|
||||
fields = body.model_dump(exclude_unset=True)
|
||||
update: dict[str, object] = {}
|
||||
if "default_theme" in fields and fields["default_theme"] is not None:
|
||||
update["default_theme"] = fields["default_theme"]
|
||||
for key in ("default_theme", "default_skill_view", "default_agent_view"):
|
||||
if key in fields and fields[key] is not None:
|
||||
update[key] = fields[key]
|
||||
if update:
|
||||
updated_appearance = current.appearance.model_copy(update=update)
|
||||
saved = save_system_settings(
|
||||
@@ -255,7 +267,7 @@ async def update_appearance_settings(
|
||||
)
|
||||
else:
|
||||
saved = current
|
||||
return AppearanceResponse(default_theme=saved.appearance.default_theme)
|
||||
return _appearance_response(saved.appearance)
|
||||
|
||||
|
||||
class LogSystemResponse(BaseModel):
|
||||
|
||||
@@ -218,8 +218,16 @@ database:
|
||||
echo_sql: false
|
||||
|
||||
checkpointer:
|
||||
type: sqlite
|
||||
connection_string: .deer-flow/data/deerflow.db
|
||||
type: postgres
|
||||
# true: 新 checkpoint 写入 Postgres;false: 保留 PG 配置但继续使用 SQLite。
|
||||
# 也可临时用环境变量 DEER_FLOW_USE_PG_CHECKPOINTER=0 覆盖。
|
||||
use_postgres: true
|
||||
connection_string: postgresql://postgres:postgres123@47.88.25.99:8432/deerflow
|
||||
sqlite_connection_string: .deer-flow/data/deerflow.db
|
||||
dual_read_enabled: true
|
||||
dual_read_sqlite_paths:
|
||||
- .deer-flow/data/deerflow.db
|
||||
- ../data/.deer-flow/data/checkpoints.db
|
||||
|
||||
# 记忆子系统:跨会话持久化用户画像与 agent 私有记忆。
|
||||
# 三层覆盖优先级(高->低):
|
||||
@@ -300,4 +308,4 @@ ai_writing:
|
||||
log_system:
|
||||
enabled: true
|
||||
url: "https://your-log-system.example.com"
|
||||
label: "日志系统"
|
||||
label: "日志系统"
|
||||
+35
-7
@@ -2,7 +2,7 @@
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
CheckpointerType = Literal["memory", "sqlite", "postgres"]
|
||||
|
||||
@@ -16,13 +16,41 @@ class CheckpointerConfig(BaseModel):
|
||||
"'sqlite' persists to a local file (requires langgraph-checkpoint-sqlite). "
|
||||
"'postgres' persists to PostgreSQL (requires langgraph-checkpoint-postgres)."
|
||||
)
|
||||
connection_string: str | None = Field(
|
||||
default=None,
|
||||
description="Connection string for sqlite (file path) or postgres (DSN). "
|
||||
connection_string: str | None = Field(
|
||||
default=None,
|
||||
description="Connection string for sqlite (file path) or postgres (DSN). "
|
||||
"Required for sqlite and postgres types. "
|
||||
"For sqlite, use a file path like '.deer-flow/checkpoints.db' or ':memory:' for in-memory. "
|
||||
"For postgres, use a DSN like 'postgresql://user:pass@localhost:5432/db'.",
|
||||
)
|
||||
"For sqlite, use a file path like '.deer-flow/checkpoints.db' or ':memory:' for in-memory. "
|
||||
"For postgres, use a DSN like 'postgresql://user:pass@localhost:5432/db'.",
|
||||
)
|
||||
use_postgres: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"When type='postgres', set false to keep the Postgres DSN in config "
|
||||
"but run the checkpointer on SQLite instead."
|
||||
),
|
||||
)
|
||||
sqlite_connection_string: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"SQLite file used when type='postgres' and use_postgres=false. "
|
||||
"Defaults to the first dual_read_sqlite_paths entry or 'store.db'."
|
||||
),
|
||||
)
|
||||
dual_read_enabled: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"When type='postgres', read from Postgres first and fall back to legacy "
|
||||
"SQLite checkpoint files when Postgres does not have the requested data."
|
||||
),
|
||||
)
|
||||
dual_read_sqlite_paths: list[str] = Field(
|
||||
default_factory=list,
|
||||
description=(
|
||||
"Legacy SQLite checkpoint database paths used for read fallback. "
|
||||
"Plain paths are resolved the same way as SQLite connection_string."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
# Global configuration instance — None means no checkpointer is configured.
|
||||
|
||||
@@ -51,6 +51,14 @@ class AppearanceSettings(BaseModel):
|
||||
default="light",
|
||||
description="Theme applied for users who haven't picked their own (light / dark)",
|
||||
)
|
||||
default_skill_view: Literal["card", "list"] = Field(
|
||||
default="list",
|
||||
description="Default skill list display mode for users who haven't picked their own (card / list)",
|
||||
)
|
||||
default_agent_view: Literal["card", "list"] = Field(
|
||||
default="card",
|
||||
description="Default agent list display mode for users who haven't picked their own (card / list)",
|
||||
)
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
|
||||
+131
-33
@@ -17,10 +17,12 @@ For sync usage see :mod:`deerflow.runtime.checkpointer.provider`.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from langgraph.types import Checkpointer
|
||||
|
||||
@@ -30,13 +32,70 @@ from deerflow.runtime.checkpointer.provider import (
|
||||
POSTGRES_INSTALL,
|
||||
SQLITE_INSTALL,
|
||||
)
|
||||
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
|
||||
from deerflow.runtime.store._sqlite_utils import ensure_sqlite_parent_dir, resolve_sqlite_conn_str
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async factory
|
||||
# ---------------------------------------------------------------------------
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _csv_env(name: str) -> list[str]:
|
||||
raw = os.getenv(name, "")
|
||||
return [part.strip() for part in raw.split(",") if part.strip()]
|
||||
|
||||
|
||||
def _dual_read_enabled(config) -> bool:
|
||||
if getattr(config, "dual_read_enabled", False):
|
||||
return True
|
||||
return os.getenv("DEER_FLOW_CHECKPOINT_DUAL_READ", "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _use_postgres_enabled(config) -> bool:
|
||||
raw = os.getenv("DEER_FLOW_USE_PG_CHECKPOINTER")
|
||||
if raw:
|
||||
return raw.strip().lower() not in {"0", "false", "no", "off"}
|
||||
return bool(getattr(config, "use_postgres", True))
|
||||
|
||||
|
||||
def _dual_read_paths(config) -> list[str]:
|
||||
paths = list(getattr(config, "dual_read_sqlite_paths", []) or [])
|
||||
paths.extend(_csv_env("DEER_FLOW_CHECKPOINT_SQLITE_PATHS"))
|
||||
resolved: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for path in paths:
|
||||
conn_str = resolve_sqlite_conn_str(path)
|
||||
if conn_str == ":memory:" or conn_str.startswith("file:"):
|
||||
logger.warning("Skipping unsupported checkpoint migration SQLite path: %s", path)
|
||||
continue
|
||||
if conn_str not in seen:
|
||||
resolved.append(conn_str)
|
||||
seen.add(conn_str)
|
||||
return resolved
|
||||
|
||||
|
||||
def _sqlite_conn_str_for_postgres_fallback(config) -> str:
|
||||
configured = getattr(config, "sqlite_connection_string", None)
|
||||
if configured:
|
||||
return configured
|
||||
paths = _dual_read_paths(config)
|
||||
return paths[0] if paths else "store.db"
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _sqlite_checkpointer_from_conn_str(raw: str) -> AsyncIterator[Checkpointer]:
|
||||
try:
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
except ImportError as exc:
|
||||
raise ImportError(SQLITE_INSTALL) from exc
|
||||
|
||||
conn_str = resolve_sqlite_conn_str(raw)
|
||||
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
|
||||
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
|
||||
await saver.setup()
|
||||
yield saver
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
@@ -48,32 +107,71 @@ async def _async_checkpointer(config) -> AsyncIterator[Checkpointer]:
|
||||
yield InMemorySaver()
|
||||
return
|
||||
|
||||
if config.type == "sqlite":
|
||||
try:
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
except ImportError as exc:
|
||||
raise ImportError(SQLITE_INSTALL) from exc
|
||||
|
||||
conn_str = resolve_sqlite_conn_str(config.connection_string or "store.db")
|
||||
await asyncio.to_thread(ensure_sqlite_parent_dir, conn_str)
|
||||
async with AsyncSqliteSaver.from_conn_string(conn_str) as saver:
|
||||
await saver.setup()
|
||||
yield saver
|
||||
return
|
||||
|
||||
if config.type == "postgres":
|
||||
try:
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
|
||||
if config.type == "sqlite":
|
||||
async with _sqlite_checkpointer_from_conn_str(config.connection_string or "store.db") as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
if config.type == "postgres":
|
||||
if not _use_postgres_enabled(config):
|
||||
conn_str = _sqlite_conn_str_for_postgres_fallback(config)
|
||||
logger.info("Checkpointer: use_postgres=false; using SQLite fallback (%s)", conn_str)
|
||||
async with _sqlite_checkpointer_from_conn_str(conn_str) as saver:
|
||||
yield saver
|
||||
return
|
||||
|
||||
try:
|
||||
from langgraph.checkpoint.postgres.aio import AsyncPostgresSaver
|
||||
except ImportError as exc:
|
||||
raise ImportError(POSTGRES_INSTALL) from exc
|
||||
|
||||
if not config.connection_string:
|
||||
raise ValueError(POSTGRES_CONN_REQUIRED)
|
||||
|
||||
async with AsyncPostgresSaver.from_conn_string(config.connection_string) as saver:
|
||||
await saver.setup()
|
||||
yield saver
|
||||
return
|
||||
if not config.connection_string:
|
||||
raise ValueError(POSTGRES_CONN_REQUIRED)
|
||||
|
||||
async with AsyncPostgresSaver.from_conn_string(config.connection_string) as saver:
|
||||
await saver.setup()
|
||||
if not _dual_read_enabled(config):
|
||||
yield saver
|
||||
return
|
||||
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
from deerflow.runtime.checkpointer.checkpoint_migration import (
|
||||
DualReadCheckpointer,
|
||||
SQLiteCheckpointSource,
|
||||
)
|
||||
|
||||
sources: list[SQLiteCheckpointSource] = []
|
||||
for sqlite_path in _dual_read_paths(config):
|
||||
if not Path(sqlite_path).exists():
|
||||
logger.info("Checkpoint dual-read source does not exist, skipping: %s", sqlite_path)
|
||||
continue
|
||||
ctx = AsyncSqliteSaver.from_conn_string(sqlite_path)
|
||||
source_saver = await ctx.__aenter__()
|
||||
await source_saver.setup()
|
||||
sources.append(
|
||||
SQLiteCheckpointSource(
|
||||
path=sqlite_path,
|
||||
saver=source_saver,
|
||||
context=ctx,
|
||||
progress_path=Path(sqlite_path).with_suffix(Path(sqlite_path).suffix + ".pg-migration.json"),
|
||||
)
|
||||
)
|
||||
|
||||
if not sources:
|
||||
logger.info("Checkpoint dual-read enabled but no SQLite sources were opened; using Postgres only")
|
||||
yield saver
|
||||
return
|
||||
|
||||
proxy = DualReadCheckpointer(
|
||||
saver,
|
||||
sources,
|
||||
)
|
||||
try:
|
||||
yield proxy
|
||||
finally:
|
||||
await proxy.aclose_sources()
|
||||
return
|
||||
|
||||
raise ValueError(f"Unknown checkpointer type: {config.type!r}")
|
||||
|
||||
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
"""SQLite-to-Postgres checkpoint migration and read fallback helpers.
|
||||
|
||||
The live gateway can wrap a Postgres checkpointer with ``DualReadCheckpointer``.
|
||||
New writes go to Postgres, reads prefer Postgres and fall back to legacy SQLite.
|
||||
Manual migration scripts reuse the copy helpers in this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SQLiteCheckpointSource:
|
||||
path: str
|
||||
saver: Any
|
||||
context: Any
|
||||
progress_path: Path
|
||||
closed: bool = False
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.closed:
|
||||
return
|
||||
self.closed = True
|
||||
try:
|
||||
await self.context.__aexit__(None, None, None)
|
||||
except Exception:
|
||||
logger.warning("Failed to close legacy SQLite checkpointer: %s", self.path, exc_info=True)
|
||||
|
||||
|
||||
def _sqlite_db_exists(path: str) -> bool:
|
||||
return Path(path).exists()
|
||||
|
||||
|
||||
def _sqlite_threads(path: str) -> list[str]:
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = conn.execute("SELECT DISTINCT thread_id FROM checkpoints ORDER BY thread_id").fetchall()
|
||||
return [str(row[0]) for row in rows if row and row[0]]
|
||||
|
||||
|
||||
def _sqlite_has_table(path: str, table: str) -> bool:
|
||||
with sqlite3.connect(path) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
|
||||
(table,),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def sqlite_recent_threads(path: str, *, since: str | None = None, until: str | None = None) -> list[str]:
|
||||
"""Return threads updated since an ISO timestamp when ``threads_meta`` exists."""
|
||||
|
||||
if not _sqlite_has_table(path, "threads_meta"):
|
||||
return []
|
||||
query = "SELECT thread_id FROM threads_meta"
|
||||
where: list[str] = []
|
||||
params: list[str] = []
|
||||
if since:
|
||||
where.append("updated_at >= ?")
|
||||
params.append(since)
|
||||
if until:
|
||||
where.append("updated_at <= ?")
|
||||
params.append(until)
|
||||
if where:
|
||||
query += " WHERE " + " AND ".join(where)
|
||||
query += " ORDER BY updated_at DESC"
|
||||
with sqlite3.connect(path) as conn:
|
||||
rows = conn.execute(query, tuple(params)).fetchall()
|
||||
return [str(row[0]) for row in rows if row and row[0]]
|
||||
|
||||
|
||||
def _sqlite_thread_checkpoint_count(path: str, thread_id: str) -> int:
|
||||
with sqlite3.connect(path) as conn:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) FROM checkpoints WHERE thread_id = ?",
|
||||
(thread_id,),
|
||||
).fetchone()
|
||||
return int(row[0] if row else 0)
|
||||
|
||||
|
||||
def _load_progress(path: Path) -> set[str]:
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except FileNotFoundError:
|
||||
return set()
|
||||
except Exception:
|
||||
logger.warning("Ignoring unreadable checkpoint migration progress file: %s", path, exc_info=True)
|
||||
return set()
|
||||
migrated = payload.get("migrated_threads", [])
|
||||
return {str(x) for x in migrated if x}
|
||||
|
||||
|
||||
def _save_progress(path: Path, migrated: set[str]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(
|
||||
json.dumps({"migrated_threads": sorted(migrated)}, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
async def _copy_thread_state(source, target, thread_id: str) -> bool:
|
||||
"""Copy one thread through the public LangGraph checkpointer API."""
|
||||
|
||||
config = {"configurable": {"thread_id": thread_id}}
|
||||
tuples = [t async for t in source.alist(config)]
|
||||
if not tuples:
|
||||
return False
|
||||
|
||||
for tup in reversed(tuples):
|
||||
configurable = (tup.config or {}).get("configurable", {}) or {}
|
||||
checkpoint_ns = configurable.get("checkpoint_ns", "") or ""
|
||||
parent_id = ((tup.parent_config or {}).get("configurable", {}) or {}).get("checkpoint_id")
|
||||
|
||||
put_config: dict[str, Any] = {"configurable": {"thread_id": thread_id, "checkpoint_ns": checkpoint_ns}}
|
||||
if parent_id:
|
||||
put_config["configurable"]["checkpoint_id"] = parent_id
|
||||
|
||||
new_versions = (tup.checkpoint or {}).get("channel_versions", {}) or {}
|
||||
await target.aput(put_config, tup.checkpoint, tup.metadata or {}, new_versions)
|
||||
|
||||
if tup.pending_writes:
|
||||
by_task: dict[str, list] = {}
|
||||
for task_id, channel, value in tup.pending_writes:
|
||||
by_task.setdefault(task_id, []).append((channel, value))
|
||||
checkpoint_id = (tup.checkpoint or {}).get("id")
|
||||
for task_id, writes in by_task.items():
|
||||
writes_config = {
|
||||
"configurable": {
|
||||
"thread_id": thread_id,
|
||||
"checkpoint_ns": checkpoint_ns,
|
||||
"checkpoint_id": checkpoint_id,
|
||||
}
|
||||
}
|
||||
await target.aput_writes(writes_config, writes, task_id)
|
||||
return True
|
||||
|
||||
|
||||
async def copy_thread_state(source, target, thread_id: str) -> bool:
|
||||
return await _copy_thread_state(source, target, thread_id)
|
||||
|
||||
|
||||
async def _target_thread_checkpoint_count(target, thread_id: str) -> int:
|
||||
config = {"configurable": {"thread_id": thread_id}}
|
||||
count = 0
|
||||
async for _ in target.alist(config):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
async def target_has_thread(target, thread_id: str) -> bool:
|
||||
config = {"configurable": {"thread_id": thread_id}}
|
||||
async for _ in target.alist(config, limit=1):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class DualReadCheckpointer:
|
||||
"""Postgres-first checkpointer with SQLite read fallback."""
|
||||
|
||||
def __init__(self, primary, sources: list[SQLiteCheckpointSource], *, sleep_seconds: float = 0.05):
|
||||
self.primary = primary
|
||||
self.sources = sources
|
||||
self.sleep_seconds = max(0.0, sleep_seconds)
|
||||
self._closed = False
|
||||
|
||||
def __getattr__(self, name: str):
|
||||
return getattr(self.primary, name)
|
||||
|
||||
async def aclose_sources(self) -> None:
|
||||
self._closed = True
|
||||
for source in self.sources:
|
||||
await source.close()
|
||||
|
||||
async def aget_tuple(self, config):
|
||||
found = await self.primary.aget_tuple(config)
|
||||
if found is not None:
|
||||
return found
|
||||
for source in self.sources:
|
||||
if source.closed:
|
||||
continue
|
||||
found = await source.saver.aget_tuple(config)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
async def aget(self, config):
|
||||
found = await self.primary.aget(config)
|
||||
if found is not None:
|
||||
return found
|
||||
for source in self.sources:
|
||||
if source.closed:
|
||||
continue
|
||||
found = await source.saver.aget(config)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
async def alist(self, config, *, filter=None, before=None, limit=None):
|
||||
seen: set[tuple[str, str, str]] = set()
|
||||
yielded = 0
|
||||
|
||||
async for tup in self.primary.alist(config, filter=filter, before=before, limit=limit):
|
||||
key = self._tuple_key(tup)
|
||||
seen.add(key)
|
||||
yield tup
|
||||
yielded += 1
|
||||
if limit is not None and yielded >= limit:
|
||||
return
|
||||
|
||||
remaining = None if limit is None else max(0, limit - yielded)
|
||||
for source in self.sources:
|
||||
if source.closed:
|
||||
continue
|
||||
async for tup in source.saver.alist(config, filter=filter, before=before, limit=remaining):
|
||||
key = self._tuple_key(tup)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
yield tup
|
||||
yielded += 1
|
||||
if limit is not None:
|
||||
remaining = max(0, limit - yielded)
|
||||
if remaining <= 0:
|
||||
return
|
||||
|
||||
async def aput(self, *args, **kwargs):
|
||||
return await self.primary.aput(*args, **kwargs)
|
||||
|
||||
async def aput_writes(self, *args, **kwargs):
|
||||
return await self.primary.aput_writes(*args, **kwargs)
|
||||
|
||||
async def adelete_thread(self, thread_id: str) -> None:
|
||||
delete = getattr(self.primary, "adelete_thread", None)
|
||||
if delete is not None:
|
||||
await delete(thread_id)
|
||||
for source in self.sources:
|
||||
if source.closed:
|
||||
continue
|
||||
delete = getattr(source.saver, "adelete_thread", None)
|
||||
if delete is not None:
|
||||
await delete(thread_id)
|
||||
|
||||
@staticmethod
|
||||
def _tuple_key(tup) -> tuple[str, str, str]:
|
||||
configurable = (tup.config or {}).get("configurable", {}) or {}
|
||||
return (
|
||||
str(configurable.get("thread_id", "")),
|
||||
str(configurable.get("checkpoint_ns", "") or ""),
|
||||
str(configurable.get("checkpoint_id", "") or (tup.checkpoint or {}).get("id", "")),
|
||||
)
|
||||
|
||||
async def _verify_source(self, source: SQLiteCheckpointSource, threads: list[str]) -> bool:
|
||||
for thread_id in threads:
|
||||
source_count = await asyncio.to_thread(_sqlite_thread_checkpoint_count, source.path, thread_id)
|
||||
target_count = await _target_thread_checkpoint_count(self.primary, thread_id)
|
||||
if target_count < source_count:
|
||||
logger.warning(
|
||||
"Checkpoint migration verification failed: source=%s thread_id=%s source_count=%d target_count=%d",
|
||||
source.path,
|
||||
thread_id,
|
||||
source_count,
|
||||
target_count,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
@@ -31,9 +31,12 @@ dependencies = [
|
||||
"ddgs>=9.10.0",
|
||||
"duckdb>=1.4.4",
|
||||
"langchain-google-genai>=4.2.1",
|
||||
"langgraph-checkpoint-sqlite>=3.0.3",
|
||||
"langgraph-sdk>=0.1.51",
|
||||
"sqlalchemy[asyncio]>=2.0,<3.0",
|
||||
"langgraph-checkpoint-sqlite>=3.0.3",
|
||||
"langgraph-checkpoint-postgres>=3.0.5",
|
||||
"langgraph-sdk>=0.1.51",
|
||||
"psycopg[binary]>=3.3.3",
|
||||
"psycopg-pool>=3.3.0",
|
||||
"sqlalchemy[asyncio]>=2.0,<3.0",
|
||||
"aiosqlite>=0.19",
|
||||
"alembic>=1.13",
|
||||
]
|
||||
|
||||
@@ -67,6 +67,16 @@ models:
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
- name: Qwen/Qwen3.6-27B
|
||||
display_name: Qwen/Qwen3.6-27B
|
||||
use: langchain_openai:ChatOpenAI
|
||||
model: Qwen/Qwen3.6-27B
|
||||
api_key: sk-qwwwvjompnzxqhndctlwmmomvqdgaqqqgxvibfxchsikvgod
|
||||
base_url: https://api.siliconflow.cn/v1/
|
||||
max_tokens: 4096
|
||||
temperature: 1.0
|
||||
supports_vision: true
|
||||
supports_thinking: true
|
||||
# 识图(图片识别)配置。
|
||||
# - model_name:指定一个 models[] 中的模型名,作为“默认识图大模型”。view_image 工具会把
|
||||
# 图片交给该模型识别,并把识别出的文字结果返回给主对话模型 —— 这样即使主对话模型不支持
|
||||
@@ -187,8 +197,16 @@ checkpointer:
|
||||
# Docker 部署:把容器内 .deer-flow 目录挂卷出来即可持久化(撑住容器重启)。
|
||||
# 这里同时被「主 agent」和「AI 写作 graph」共用 —— LangGraph 用
|
||||
# (thread_id, checkpoint_ns) 做隔离,session_id 是 uuid,不会撞。
|
||||
type: sqlite
|
||||
connection_string: .deer-flow/data/checkpoints.db
|
||||
type: postgres
|
||||
# true: 新 checkpoint 写入 Postgres;false: 保留 PG 配置但继续使用 SQLite。
|
||||
# 也可临时用环境变量 DEER_FLOW_USE_PG_CHECKPOINTER=0 覆盖。
|
||||
use_postgres: true
|
||||
connection_string: postgresql://postgres:postgres123@47.88.25.99:8432/deerflow
|
||||
sqlite_connection_string: backend/.deer-flow/data/deerflow.db
|
||||
dual_read_enabled: true
|
||||
dual_read_sqlite_paths:
|
||||
- backend/.deer-flow/data/deerflow.db
|
||||
- data/.deer-flow/data/checkpoints.db
|
||||
# AI 写作子系统的会话清理配置。
|
||||
# 业务表(ai_writing_sessions @ MySQL) + checkpointer(checkpoints.db @ SQLite) 联动清理。
|
||||
# 留意:清理走 cron 表达式调度,默认每天凌晨 3 点跑一次,按 updated_at 一刷拉清。
|
||||
|
||||
Reference in new issue
Block a user