Files
deerflow-code/offline-backend-20260512/backend/app/gateway/routers/chat_history_transfer.py
T

701 lines
23 KiB
Python

"""Account-scoped chat history transfer for the conversation list page.
Exports and imports ordinary / agent chats as a JSON bundle that includes
the latest checkpoint (messages) and the thread ``user-data`` files
(uploads + outputs). One oversized or broken thread is skipped so the
rest of the batch can still succeed.
"""
from __future__ import annotations
import asyncio
import base64
import json
import logging
import re
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import FileResponse
from langgraph.checkpoint.base import empty_checkpoint
from pydantic import BaseModel, Field
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer, get_thread_store
from app.gateway.routers.threads import _strip_reserved_metadata
from app.gateway.utils import sanitize_log_param
from deerflow.config.paths import get_paths
from deerflow.config.system_settings import load_system_settings
from deerflow.persistence.thread_meta.base import is_excluded_system_thread
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.utils.time import coerce_iso, now_iso
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/chat-history", tags=["chat-history-transfer"])
FORMAT = "deerflow-chat-history-ui"
FORMAT_VERSION = 1
BATCH_LIMIT = 1
MAX_THREAD_BYTES = 50 * 1024 * 1024
MAX_FILES_PER_THREAD = 200
SAFE_REL_PATH = re.compile(r"^(?:[A-Za-z0-9._-]+/)*[A-Za-z0-9._-]+$")
_FILE_JSON_OVERHEAD = 64
_DROP_META_KEYS = frozenset(
{
"owner_id",
"user_id",
"analysis_project_id",
"analysis_project_mode",
"analysis_project_name",
"shared_from_code",
"shared_origin",
"shared_by",
"taskId",
"actionId",
}
)
class SkippedThread(BaseModel):
thread_id: str
title: str = ""
reason: str
class TransferFile(BaseModel):
path: str
content_b64: str
size: int = 0
class TransferThread(BaseModel):
thread_id: str
title: str = ""
assistant_id: str | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
created_at: str = ""
updated_at: str = ""
values: dict[str, Any] = Field(default_factory=dict)
files: list[TransferFile] = Field(default_factory=list)
skipped_files: list[str] = Field(default_factory=list)
class ExportRequest(BaseModel):
thread_ids: list[str] = Field(..., min_length=1, max_length=BATCH_LIMIT)
class ExportResponse(BaseModel):
format: str = FORMAT
version: int = FORMAT_VERSION
threads: list[TransferThread] = Field(default_factory=list)
skipped: list[SkippedThread] = Field(default_factory=list)
class ImportRequest(BaseModel):
threads: list[TransferThread] = Field(..., min_length=1, max_length=BATCH_LIMIT)
class ImportResponse(BaseModel):
imported: int = 0
skipped: list[SkippedThread] = Field(default_factory=list)
class ExportAllStartResponse(BaseModel):
job_id: str
class ExportAllStatusResponse(BaseModel):
job_id: str
status: str
total: int = 0
processed: int = 0
succeeded: int = 0
skipped: list[SkippedThread] = Field(default_factory=list)
skipped_files: int = 0
error: str | None = None
@dataclass
class ExportAllJob:
job_id: str
user_id: str
status: str = "running"
total: int = 0
processed: int = 0
succeeded: int = 0
skipped: list[SkippedThread] = field(default_factory=list)
skipped_files: int = 0
error: str | None = None
path: Path | None = None
task: asyncio.Task[None] | None = None
_EXPORT_JOBS: dict[str, ExportAllJob] = {}
_SAFE_USER_DIR = re.compile(r"[^A-Za-z0-9._-]+")
def require_transfer_enabled() -> None:
if not load_system_settings().agent_skill_migration.enabled:
raise HTTPException(status_code=403, detail="跨服务器迁移功能未开启")
def current_user_id(request: Request) -> str:
user = getattr(request.state, "user", None)
if user is not None and getattr(user, "id", None):
return str(user.id)
auth = getattr(request.state, "auth", None)
auth_user = getattr(auth, "user", None) if auth is not None else None
if auth_user is not None and getattr(auth_user, "id", None):
return str(auth_user.id)
return get_effective_user_id()
def payload_size(value: Any) -> int:
return len(json.dumps(value, ensure_ascii=False, default=str).encode("utf-8"))
def clean_export_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
return {
key: item
for key, item in _strip_reserved_metadata(metadata).items()
if key not in _DROP_META_KEYS
}
def safe_relpath(root: Path, path: Path) -> str | None:
try:
resolved = path.resolve()
relative = resolved.relative_to(root.resolve())
except ValueError:
return None
text = relative.as_posix()
if not SAFE_REL_PATH.fullmatch(text):
return None
return text
def encoded_file_cost(size: int, path: str) -> int:
return (size * 4 + 3) // 3 + len(path) + _FILE_JSON_OVERHEAD
def list_thread_file_candidates(root: Path) -> list[tuple[int, str, Path]]:
if not root.is_dir():
return []
candidates: list[tuple[int, str, Path]] = []
for path in sorted(root.rglob("*")):
if not path.is_file() or path.is_symlink():
continue
rel = safe_relpath(root, path)
if rel is None:
continue
try:
size = path.stat().st_size
except OSError:
continue
if size < 0:
continue
candidates.append((size, rel, path))
candidates.sort(key=lambda item: (item[0], item[1]))
return candidates
def collect_thread_files(root: Path, *, budget: int) -> tuple[list[dict[str, Any]], list[str]]:
candidates = list_thread_file_candidates(root)
if not candidates:
return [], []
files: list[dict[str, Any]] = []
skipped: list[str] = []
used = 0
for size, rel, path in candidates:
cost = encoded_file_cost(size, rel)
if budget <= 0 or len(files) >= MAX_FILES_PER_THREAD or used + cost > budget:
skipped.append(rel)
continue
try:
data = path.read_bytes()
except OSError:
skipped.append(rel)
continue
files.append(
{
"path": rel,
"content_b64": base64.b64encode(data).decode("ascii"),
"size": size,
}
)
used += cost
return files, skipped
def write_thread_files(
root: Path,
files: list[TransferFile],
*,
budget: int = MAX_THREAD_BYTES,
) -> list[str]:
failed: list[str] = []
used = 0
root.mkdir(parents=True, exist_ok=True)
root_resolved = root.resolve()
for item in files[:MAX_FILES_PER_THREAD]:
rel = safe_relpath(root, root / item.path) if SAFE_REL_PATH.fullmatch(item.path) else None
if rel is None:
failed.append(item.path)
continue
dest = (root / rel).resolve()
if dest != root_resolved and root_resolved not in dest.parents:
failed.append(item.path)
continue
try:
data = base64.b64decode(item.content_b64, validate=True)
if used + len(data) > budget:
failed.append(item.path)
continue
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(data)
used += len(data)
except Exception:
failed.append(item.path)
return failed
def _title_of(record: dict[str, Any], values: dict[str, Any] | None = None) -> str:
title = record.get("display_name") or (values or {}).get("title") or ""
return str(title).strip()
def bump_channel_versions(checkpointer: Any, checkpoint: dict[str, Any], keys: list[str]) -> dict[str, Any]:
"""LangGraph stores each channel as a versioned blob.
``aput(..., new_versions={})`` only keeps the in-memory dict on some
savers. Postgres/SQLite ignore unversioned channels, so imported
messages would not survive ``aget`` / ``GET /threads/{id}/state``.
"""
versions = dict(checkpoint.get("channel_versions") or {})
new_versions: dict[str, Any] = {}
get_next_version = getattr(checkpointer, "get_next_version", None)
if not callable(get_next_version):
return dict(versions)
for key in keys:
next_version = get_next_version(versions.get(key), None)
versions[key] = next_version
new_versions[key] = next_version
checkpoint["channel_versions"] = versions
return new_versions
async def write_imported_checkpoint(
checkpointer: Any,
*,
thread_id: str,
values: dict[str, Any],
metadata: dict[str, Any],
) -> None:
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
now = now_iso()
checkpoint = dict(empty_checkpoint())
checkpoint["channel_values"] = dict(values)
new_versions = bump_channel_versions(checkpointer, checkpoint, list(values.keys()))
await checkpointer.aput(
config,
checkpoint,
{
"step": 1,
"source": "update",
"created_at": now,
"updated_at": now,
**metadata,
},
new_versions,
)
written = await checkpointer.aget_tuple(config)
if written is None:
raise RuntimeError("missing checkpoint after import")
restored = serialize_channel_values(
(getattr(written, "checkpoint", {}) or {}).get("channel_values", {}) or {}
)
if values.get("messages") and not restored.get("messages"):
raise RuntimeError("checkpoint messages were not persisted")
def export_job_dir(user_id: str) -> Path:
safe = _SAFE_USER_DIR.sub("_", user_id).strip("._") or "user"
path = get_paths().base_dir / "chat-history-exports" / safe
path.mkdir(parents=True, exist_ok=True)
return path
def get_export_job(job_id: str, user_id: str) -> ExportAllJob:
job = _EXPORT_JOBS.get(job_id)
if job is None or job.user_id != user_id:
raise HTTPException(status_code=404, detail="导出任务不存在")
return job
def find_running_export_job(user_id: str) -> ExportAllJob | None:
for job in _EXPORT_JOBS.values():
if job.user_id == user_id and job.status == "running":
return job
return None
def purge_finished_export_jobs(user_id: str) -> None:
stale = [
job_id
for job_id, job in _EXPORT_JOBS.items()
if job.user_id == user_id and job.status != "running"
]
for job_id in stale:
job = _EXPORT_JOBS.pop(job_id, None)
if job is None or job.path is None:
continue
try:
job.path.unlink(missing_ok=True)
except OSError:
logger.debug("Failed to remove export file %s", job.path)
def job_status(job: ExportAllJob) -> ExportAllStatusResponse:
return ExportAllStatusResponse(
job_id=job.job_id,
status=job.status,
total=job.total,
processed=job.processed,
succeeded=job.succeeded,
skipped=list(job.skipped),
skipped_files=job.skipped_files,
error=job.error,
)
class BundleWriter:
def __init__(self, path: Path) -> None:
self.path = path
self._fh = path.open("w", encoding="utf-8")
self._first = True
self._fh.write(
'{"format":%s,"version":%s,"exported_at":%s,"threads":['
% (
json.dumps(FORMAT, ensure_ascii=False),
FORMAT_VERSION,
json.dumps(now_iso()),
)
)
def add_thread(self, thread: TransferThread) -> None:
if not self._first:
self._fh.write(",")
self._first = False
self._fh.write(thread.model_dump_json())
self._fh.flush()
def finish(self, skipped: list[SkippedThread]) -> None:
self._fh.write('],"skipped":')
self._fh.write(json.dumps([item.model_dump() for item in skipped], ensure_ascii=False))
self._fh.write("}")
self._fh.close()
def abort(self) -> None:
try:
self._fh.close()
except Exception:
pass
try:
self.path.unlink(missing_ok=True)
except OSError:
pass
async def list_exportable_thread_ids(thread_store: Any, user_id: str) -> list[str]:
ids: list[str] = []
offset = 0
page_size = 100
while True:
rows = await thread_store.search(
user_id=user_id,
exclude_system=True,
limit=page_size,
offset=offset,
)
ids.extend(str(row["thread_id"]) for row in rows if row.get("thread_id"))
if len(rows) < page_size:
break
offset += len(rows)
return ids
async def run_export_all_job(
job: ExportAllJob,
*,
thread_store: Any,
checkpointer: Any,
) -> None:
writer: BundleWriter | None = None
try:
thread_ids = await list_exportable_thread_ids(thread_store, job.user_id)
job.total = len(thread_ids)
if not thread_ids:
job.status = "done"
return
dest = export_job_dir(job.user_id) / f"{job.job_id}.json"
writer = BundleWriter(dest)
for thread_id in thread_ids:
try:
payload, skip = await export_one_thread(
thread_id=thread_id,
user_id=job.user_id,
thread_store=thread_store,
checkpointer=checkpointer,
)
except Exception:
logger.exception("Unexpected export failure for %s", sanitize_log_param(thread_id))
payload, skip = None, SkippedThread(thread_id=thread_id, reason="export_failed")
if payload is not None:
await asyncio.to_thread(writer.add_thread, payload)
job.succeeded += 1
job.skipped_files += len(payload.skipped_files)
if skip is not None:
job.skipped.append(skip)
job.processed += 1
await asyncio.to_thread(writer.finish, list(job.skipped))
job.path = dest
job.status = "done"
except Exception:
logger.exception("Background chat-history export failed for %s", sanitize_log_param(job.user_id))
if writer is not None:
writer.abort()
job.status = "error"
job.error = "导出失败"
async def export_one_thread(
*,
thread_id: str,
user_id: str,
thread_store: Any,
checkpointer: Any,
max_bytes: int = MAX_THREAD_BYTES,
) -> tuple[TransferThread | None, SkippedThread | None]:
record = await thread_store.get(thread_id, user_id=user_id)
if record is None:
return None, SkippedThread(thread_id=thread_id, reason="not_found")
if is_excluded_system_thread(record.get("metadata")):
return None, SkippedThread(
thread_id=thread_id,
title=_title_of(record),
reason="system_thread",
)
try:
checkpoint_tuple = await checkpointer.aget_tuple(
{"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
)
except Exception:
logger.exception("Failed to read checkpoint for export %s", sanitize_log_param(thread_id))
return None, SkippedThread(thread_id=thread_id, title=_title_of(record), reason="export_failed")
values: dict[str, Any] = {}
if checkpoint_tuple is not None:
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
try:
values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
except Exception:
logger.exception("Failed to serialize state for export %s", sanitize_log_param(thread_id))
return None, SkippedThread(thread_id=thread_id, title=_title_of(record), reason="export_failed")
title = _title_of(record, values)
if title and not values.get("title"):
values["title"] = title
used = payload_size(values)
if used > max_bytes:
return None, SkippedThread(thread_id=thread_id, title=title, reason="too_large")
user_data = get_paths().sandbox_user_data_dir(thread_id, user_id=user_id)
files, skipped_files = await asyncio.to_thread(
collect_thread_files, user_data, budget=max_bytes - used
)
payload = TransferThread(
thread_id=thread_id,
title=title,
assistant_id=record.get("assistant_id"),
metadata=clean_export_metadata(record.get("metadata") or {}),
created_at=coerce_iso(record.get("created_at", "")),
updated_at=coerce_iso(record.get("updated_at", "")),
values=values,
files=[TransferFile.model_validate(item) for item in files],
skipped_files=list(skipped_files),
)
while payload.files and payload_size(payload.model_dump()) > max_bytes:
largest = max(payload.files, key=lambda item: (item.size, item.path))
payload.files.remove(largest)
payload.skipped_files.append(largest.path)
if payload_size(payload.model_dump()) > max_bytes:
return None, SkippedThread(thread_id=thread_id, title=title, reason="too_large")
return payload, None
async def import_one_thread(
*,
item: TransferThread,
user_id: str,
thread_store: Any,
checkpointer: Any,
) -> SkippedThread | None:
thread_id = item.thread_id.strip()
if not thread_id or not re.fullmatch(r"[A-Za-z0-9_-]+", thread_id):
return SkippedThread(thread_id=item.thread_id or "unknown", title=item.title, reason="invalid_id")
existing = await thread_store.get(thread_id, user_id=user_id)
if existing is not None:
return SkippedThread(thread_id=thread_id, title=item.title or _title_of(existing), reason="exists")
metadata = clean_export_metadata(item.metadata)
title = (item.title or str(item.values.get("title") or "")).strip()
values = dict(item.values or {})
if title:
values["title"] = title
created = False
try:
await thread_store.create(
thread_id,
assistant_id=item.assistant_id,
user_id=user_id,
display_name=title or None,
metadata=metadata,
)
created = True
await write_imported_checkpoint(
checkpointer,
thread_id=thread_id,
values=values,
metadata=metadata,
)
if title:
try:
await thread_store.update_display_name(thread_id, title, user_id=user_id)
except Exception:
logger.debug("Failed to sync imported title for %s", sanitize_log_param(thread_id))
if item.files:
dest = get_paths().sandbox_user_data_dir(thread_id, user_id=user_id)
failed_files = await asyncio.to_thread(write_thread_files, dest, item.files)
if failed_files:
logger.warning(
"Imported thread %s with %d skipped files",
sanitize_log_param(thread_id),
len(failed_files),
)
except Exception:
logger.exception("Failed to import thread %s", sanitize_log_param(thread_id))
if created:
try:
await thread_store.delete(thread_id, user_id=user_id)
except Exception:
logger.debug("Failed to roll back imported thread %s", sanitize_log_param(thread_id))
return SkippedThread(thread_id=thread_id, title=title, reason="import_failed")
return None
@router.post("/export", response_model=ExportResponse, summary="Export a batch of conversations")
@require_permission("threads", "read")
async def export_chat_history(body: ExportRequest, request: Request) -> ExportResponse:
require_transfer_enabled()
user_id = current_user_id(request)
thread_store = get_thread_store(request)
checkpointer = get_checkpointer(request)
exported: list[TransferThread] = []
skipped: list[SkippedThread] = []
for thread_id in body.thread_ids:
try:
payload, skip = await export_one_thread(
thread_id=thread_id,
user_id=user_id,
thread_store=thread_store,
checkpointer=checkpointer,
)
except Exception:
logger.exception("Unexpected export failure for %s", sanitize_log_param(thread_id))
skipped.append(SkippedThread(thread_id=thread_id, reason="export_failed"))
continue
if payload is not None:
exported.append(payload)
if skip is not None:
skipped.append(skip)
return ExportResponse(threads=exported, skipped=skipped)
@router.post("/import", response_model=ImportResponse, summary="Import a batch of conversations")
@require_permission("threads", "write")
async def import_chat_history(body: ImportRequest, request: Request) -> ImportResponse:
require_transfer_enabled()
user_id = current_user_id(request)
thread_store = get_thread_store(request)
checkpointer = get_checkpointer(request)
imported = 0
skipped: list[SkippedThread] = []
for item in body.threads:
try:
skip = await import_one_thread(
item=item,
user_id=user_id,
thread_store=thread_store,
checkpointer=checkpointer,
)
except Exception:
logger.exception("Unexpected import failure for %s", sanitize_log_param(item.thread_id))
skipped.append(SkippedThread(thread_id=item.thread_id, title=item.title, reason="import_failed"))
continue
if skip is None:
imported += 1
else:
skipped.append(skip)
return ImportResponse(imported=imported, skipped=skipped)
@router.post("/export-all", response_model=ExportAllStartResponse, summary="Start a backend export of all conversations")
@require_permission("threads", "read")
async def start_export_all(request: Request) -> ExportAllStartResponse:
require_transfer_enabled()
user_id = current_user_id(request)
running = find_running_export_job(user_id)
if running is not None:
return ExportAllStartResponse(job_id=running.job_id)
purge_finished_export_jobs(user_id)
job = ExportAllJob(job_id=uuid.uuid4().hex, user_id=user_id)
_EXPORT_JOBS[job.job_id] = job
job.task = asyncio.create_task(
run_export_all_job(
job,
thread_store=get_thread_store(request),
checkpointer=get_checkpointer(request),
)
)
return ExportAllStartResponse(job_id=job.job_id)
@router.get("/export-all/{job_id}", response_model=ExportAllStatusResponse, summary="Poll a backend export job")
@require_permission("threads", "read")
async def get_export_all_status(job_id: str, request: Request) -> ExportAllStatusResponse:
require_transfer_enabled()
return job_status(get_export_job(job_id, current_user_id(request)))
@router.get("/export-all/{job_id}/download", summary="Download a finished backend export")
@require_permission("threads", "read")
async def download_export_all(job_id: str, request: Request) -> FileResponse:
require_transfer_enabled()
job = get_export_job(job_id, current_user_id(request))
if job.status != "done" or job.path is None or not job.path.is_file():
raise HTTPException(status_code=409, detail="导出文件还没准备好")
return FileResponse(
job.path,
media_type="application/json",
filename="deerflow-chats.json",
)