sync snapshot from LAN repo (source master-zg @ 40658c74)

This commit is contained in:
gozeros committed 2026-11-05 10:58:49 +08:00
1 parent ea9dcff9c0
commit 754b61572c
179 files changed
+26085 -8788

No files matched your search

@@ -0,0 +1,232 @@
"""Publish product-owned public skills into the configured live skill root.
The Gateway may be launched through the supported shell scripts, an image
entrypoint, Compose, or an external release repository. Startup therefore
cannot rely on one shell wrapper to populate a persistent ``/app/skills``
mount. This module is deliberately standard-library-only so both the launcher
CLI and the Gateway lifespan can use the exact same synchronization contract.
"""
from __future__ import annotations
import hashlib
import os
import shutil
from collections.abc import Iterator
from contextlib import contextmanager
from pathlib import Path
ROUTED_PAGE_SKILLS = (
"facebook-page-generator",
"x-page-generator",
"tiktok-page-generator",
"youtube-page-generator",
"threads-page-generator",
"cnn-page-generator",
"bbc-page-generator",
"xinhua-page-generator",
"white-house-page-generator",
"manila-times-page-generator",
"gma-news-page-generator",
"yahoo-japan-page-generator",
"yomiuri-news-page-generator",
)
_BUNDLED_SKILLS_ENV = "DEER_FLOW_BUNDLED_SKILLS_PATH"
_SYNC_LOCK_FILE = ".builtin-public-skills.lock"
_COPY_IGNORE = shutil.ignore_patterns("__pycache__", "*.pyc", "*.pyo")
class SkillSyncError(RuntimeError):
"""Raised when bundled skills cannot be published safely."""
@contextmanager
def _exclusive_target_lock(target_root: Path) -> Iterator[None]:
"""Serialize startup copies across Uvicorn worker processes."""
target_root.mkdir(parents=True, exist_ok=True)
lock_path = target_root / _SYNC_LOCK_FILE
with lock_path.open("a+b") as handle:
handle.seek(0, os.SEEK_END)
if handle.tell() == 0:
handle.write(b"\0")
handle.flush()
handle.seek(0)
if os.name == "nt":
import msvcrt
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
try:
yield
finally:
handle.seek(0)
if os.name == "nt":
import msvcrt
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
def sync_builtin_public_skills(
source_root: Path,
target_root: Path,
*,
required: tuple[str, ...] = (),
) -> dict[str, object]:
"""Overlay bundled ``public`` skills while preserving live custom skills."""
source_public = source_root.resolve() / "public"
target_root = target_root.resolve()
target_public = target_root / "public"
target_custom = target_root / "custom"
if not source_public.is_dir():
raise SkillSyncError(f"Bundled public skills directory not found: {source_public}")
target_public.mkdir(parents=True, exist_ok=True)
target_custom.mkdir(parents=True, exist_ok=True)
copied: list[str] = []
discovered: list[str] = []
source_is_live = source_public == target_public.resolve()
for source in sorted(source_public.iterdir(), key=lambda path: path.name):
if not source.is_dir() or source.name.startswith("."):
continue
if not source_is_live:
destination = target_public / source.name
shutil.copytree(
source,
destination,
dirs_exist_ok=True,
symlinks=True,
ignore=_COPY_IGNORE,
)
copied.append(source.name)
if (source / "SKILL.md").is_file():
discovered.append(source.name)
missing = sorted(name for name in required if not (target_public / name / "SKILL.md").is_file())
if missing:
raise SkillSyncError("Required built-in skills are unavailable after sync: " + ", ".join(missing))
return {
"status": "ok",
"source": str(source_public),
"target": str(target_public),
"sourceIsLive": source_is_live,
"copiedDirectories": copied,
"discoveredSkills": discovered,
"requiredSkills": list(required),
}
def _required_packages_digest(public_root: Path, required: tuple[str, ...]) -> str | None:
"""Hash the required packages, ignoring interpreter cache artifacts."""
digest = hashlib.sha256()
for name in required:
skill_root = public_root / name
if not (skill_root / "SKILL.md").is_file():
return None
files = sorted(
path
for path in skill_root.rglob("*")
if path.is_file()
and "__pycache__" not in path.parts
and path.suffix not in {".pyc", ".pyo"}
)
for path in files:
relative = path.relative_to(public_root).as_posix().encode("utf-8")
digest.update(len(relative).to_bytes(4, "big"))
digest.update(relative)
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def publish_bundled_public_skills(
source_root: Path,
target_root: Path,
*,
required: tuple[str, ...] = ROUTED_PAGE_SKILLS,
) -> dict[str, object]:
"""Lock, compare and publish the current bundled public skill tree."""
source_root = source_root.resolve()
target_root = target_root.resolve()
if source_root == target_root:
result = sync_builtin_public_skills(source_root, target_root, required=required)
result["skipped"] = True
return result
with _exclusive_target_lock(target_root):
source_digest = _required_packages_digest(source_root / "public", required)
if source_digest is None:
missing = sorted(
name for name in required if not (source_root / "public" / name / "SKILL.md").is_file()
)
raise SkillSyncError("Required bundled skills are unavailable: " + ", ".join(missing))
target_digest = _required_packages_digest(target_root / "public", required)
if source_digest == target_digest:
return {
"status": "ok",
"source": str(source_root / "public"),
"target": str(target_root / "public"),
"sourceIsLive": (source_root / "public") == (target_root / "public"),
"copiedDirectories": [],
"discoveredSkills": list(required),
"requiredSkills": list(required),
"skipped": True,
}
result = sync_builtin_public_skills(source_root, target_root, required=required)
result["skipped"] = False
return result
def resolve_bundled_skills_root() -> Path:
"""Return the authoritative bundled skill root for this deployment."""
explicit = os.getenv(_BUNDLED_SKILLS_ENV, "").strip()
if explicit:
candidate = Path(explicit).expanduser().resolve()
if not (candidate / "public").is_dir():
raise SkillSyncError(f"Configured bundled skills directory not found: {candidate / 'public'}")
return candidate
packaged = Path(__file__).resolve().parents[1] / "skills"
candidates = (Path("/app/bundled-skills"), packaged)
for candidate in candidates:
resolved = candidate.resolve()
if (resolved / "public").is_dir():
return resolved
raise SkillSyncError(
"Bundled public skills directory not found; checked: "
+ ", ".join(str(candidate.resolve() / "public") for candidate in candidates)
)
def ensure_bundled_public_skills(
app_config,
*,
required: tuple[str, ...] = ROUTED_PAGE_SKILLS,
) -> dict[str, object]:
"""Synchronize bundled skills into the configured root during startup."""
source_root = resolve_bundled_skills_root()
target_root = Path(app_config.skills.get_skills_path()).resolve()
return publish_bundled_public_skills(source_root, target_root, required=required)
__all__ = [
"ROUTED_PAGE_SKILLS",
"SkillSyncError",
"ensure_bundled_public_skills",
"publish_bundled_public_skills",
"resolve_bundled_skills_root",
"sync_builtin_public_skills",
]
@@ -0,0 +1,924 @@
"""Agent/skill metadata migration manifests for already-copied live folders.
The manifest deliberately contains no agent or skill payload files. Operators
copy those directories with their normal offline transfer process; these
helpers only prove that the target runtime folders match the exported inventory
before database/configuration metadata is reconciled.
"""
from __future__ import annotations
import asyncio
import hashlib
import json
import re
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath
from typing import Any, Literal
from uuid import uuid4
import yaml
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
ResourceStatus = Literal["matched", "missing", "hash_mismatch", "identity_mismatch", "invalid_path"]
OwnerStatus = Literal["matched", "unresolved"]
_TREE_HASH_RE = re.compile(r"^[0-9a-f]{64}$")
_TREE_HASH_MAX_WORKERS = 4
def _safe_relative_path(value: str) -> str:
normalized = value.strip()
if not normalized or "\\" in normalized or ":" in normalized:
raise ValueError("relative_dir must be a non-empty portable relative path")
path = PurePosixPath(normalized)
if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
raise ValueError("relative_dir must stay inside the configured runtime root")
return path.as_posix()
def compute_tree_hash(root: Path) -> str:
"""Return a deterministic SHA-256 for a directory tree.
Relative paths and bytes are both signed. Symlinks are rejected so a
manifest cannot attest content outside the selected runtime directory.
"""
root = root.resolve()
if not root.is_dir():
raise FileNotFoundError(root)
digest = hashlib.sha256()
files = sorted((path for path in root.rglob("*") if path.is_file() or path.is_symlink()), key=lambda p: p.relative_to(root).as_posix())
for path in files:
if path.is_symlink():
raise ValueError(f"Symlinks are not supported in migration trees: {path}")
relative = path.relative_to(root).as_posix().encode("utf-8")
digest.update(len(relative).to_bytes(4, "big"))
digest.update(relative)
digest.update(path.stat().st_size.to_bytes(8, "big"))
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
class OwnerReference(BaseModel):
model_config = ConfigDict(extra="forbid")
source_user_id: str | None = None
email: str | None = None
@field_validator("source_user_id", "email")
@classmethod
def _strip_optional(cls, value: str | None) -> str | None:
return value.strip() if value and value.strip() else None
class TagManifestItem(BaseModel):
model_config = ConfigDict(extra="forbid")
id: str = Field(min_length=1, max_length=64)
name: str = Field(min_length=1, max_length=128)
color: str = Field(default="gray", max_length=32)
description: str = Field(default="", max_length=1024)
class AgentManifestItem(BaseModel):
model_config = ConfigDict(extra="forbid")
id: str
relative_dir: str
tree_hash: str
name: str
description: str = ""
owner: OwnerReference | None = None
published: bool = False
is_private: bool = False
square_id: str = ""
featured_order: int | None = None
created_at: datetime | None = None
updated_at: datetime | None = None
skills: list[str] = Field(default_factory=list, max_length=5000)
tags: list[TagManifestItem] = Field(default_factory=list, max_length=200)
@field_validator("relative_dir")
@classmethod
def _relative_dir(cls, value: str) -> str:
return _safe_relative_path(value)
@field_validator("tree_hash")
@classmethod
def _tree_hash(cls, value: str) -> str:
normalized = value.lower()
if not _TREE_HASH_RE.fullmatch(normalized):
raise ValueError("tree_hash must be a SHA-256 hex digest")
return normalized
@model_validator(mode="after")
def _directory_matches_id(self):
if self.relative_dir != self.id.lower():
raise ValueError("agent relative_dir must equal the stable agent id")
return self
class SkillManifestItem(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str
category: Literal["public", "custom"]
relative_dir: str
tree_hash: str
owner: OwnerReference | None = None
published: bool = False
detail: str | None = None
name_zh: str | None = None
always_on: bool = False
square_id: str = ""
enabled: bool = True
state: Literal["active", "stale", "archived"] = "active"
pinned: bool = False
featured: bool = False
source: Literal["agent", "upload"] = "upload"
created_at: datetime | None = None
updated_at: datetime | None = None
tags: list[TagManifestItem] = Field(default_factory=list, max_length=200)
@field_validator("relative_dir")
@classmethod
def _relative_dir(cls, value: str) -> str:
normalized = _safe_relative_path(value)
if PurePosixPath(normalized).parts[0] not in {"public", "custom"}:
raise ValueError("skill relative_dir must begin with public/ or custom/")
return normalized
@field_validator("tree_hash")
@classmethod
def _tree_hash(cls, value: str) -> str:
normalized = value.lower()
if not _TREE_HASH_RE.fullmatch(normalized):
raise ValueError("tree_hash must be a SHA-256 hex digest")
return normalized
@model_validator(mode="after")
def _category_matches_directory(self):
if PurePosixPath(self.relative_dir).parts[0] != self.category:
raise ValueError("skill category must match relative_dir")
return self
class SupportDirectoryManifestItem(BaseModel):
model_config = ConfigDict(extra="forbid")
relative_dir: str
tree_hash: str
required_by: list[str] = Field(default_factory=list, max_length=5000)
@field_validator("relative_dir")
@classmethod
def _relative_dir(cls, value: str) -> str:
normalized = _safe_relative_path(value)
if PurePosixPath(normalized).parts[0] not in {"public", "custom"}:
raise ValueError("support relative_dir must begin with public/ or custom/")
return normalized
@field_validator("tree_hash")
@classmethod
def _tree_hash(cls, value: str) -> str:
normalized = value.lower()
if not _TREE_HASH_RE.fullmatch(normalized):
raise ValueError("tree_hash must be a SHA-256 hex digest")
return normalized
class MigrationManifest(BaseModel):
model_config = ConfigDict(extra="forbid")
schema_version: Literal[1] = 1
bundle_id: str = Field(min_length=1, max_length=128)
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
agents: list[AgentManifestItem] = Field(default_factory=list, max_length=1000)
skills: list[SkillManifestItem] = Field(default_factory=list, max_length=5000)
support_directories: list[SupportDirectoryManifestItem] = Field(default_factory=list, max_length=1000)
warnings: list[str] = Field(default_factory=list, max_length=1000)
class ResourcePreflightItem(BaseModel):
kind: Literal["agent", "skill", "support"]
key: str
relative_dir: str
status: ResourceStatus
message: str = ""
class OwnerMappingItem(BaseModel):
source_user_id: str | None = None
email: str | None = None
target_user_id: str | None = None
status: OwnerStatus
class MigrationPreflight(BaseModel):
ready: bool
agents: list[ResourcePreflightItem]
skills: list[ResourcePreflightItem]
support_directories: list[ResourcePreflightItem]
owner_mappings: list[OwnerMappingItem]
class MigrationApplyResult(BaseModel):
bundle_id: str
agents_applied: int
skills_applied: int
skills_skipped: int = 0
skipped_skill_names: list[str] = Field(default_factory=list)
tags_created: int
fallback_owner_count: int
def _resolved_child(root: Path, relative_dir: str) -> Path | None:
root = root.resolve()
child = root
for part in PurePosixPath(relative_dir).parts:
child /= part
if child.is_symlink():
return None
child = child.resolve()
try:
child.relative_to(root)
except ValueError:
return None
return child
def _agent_identity(directory: Path) -> str | None:
config_path = directory / "config.yaml"
if not config_path.is_file():
return None
try:
payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
except (OSError, UnicodeError, yaml.YAMLError):
return None
return str(payload.get("id") or "").strip() or None if isinstance(payload, dict) else None
def _skill_identity(directory: Path) -> str | None:
skill_path = directory / "SKILL.md"
if not skill_path.is_file():
return None
try:
text = skill_path.read_text(encoding="utf-8")
if not text.startswith("---"):
return None
parts = text.split("---", 2)
payload = yaml.safe_load(parts[1]) or {}
except (OSError, UnicodeError, yaml.YAMLError, IndexError):
return None
return str(payload.get("name") or "").strip() or None if isinstance(payload, dict) else None
def _resource_preflight(
*,
kind: Literal["agent", "skill", "support"],
key: str,
root: Path,
relative_dir: str,
expected_hash: str,
expected_identity: str | None = None,
) -> ResourcePreflightItem:
directory = _resolved_child(root, relative_dir)
if directory is None:
return ResourcePreflightItem(
kind=kind,
key=key,
relative_dir=relative_dir,
status="invalid_path",
message="目录路径不安全、包含符号链接或超出运行根目录",
)
if not directory.is_dir():
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="missing", message="目标运行目录中未找到该目录")
if expected_identity is not None:
identity_file = directory / ("config.yaml" if kind == "agent" else "SKILL.md")
if not identity_file.is_file():
return ResourcePreflightItem(
kind=kind,
key=key,
relative_dir=relative_dir,
status="missing",
message=f"目录中缺少 {identity_file.name}",
)
try:
actual_hash = compute_tree_hash(directory)
except (OSError, ValueError) as exc:
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="hash_mismatch", message=str(exc))
if actual_hash != expected_hash:
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="hash_mismatch", message="目录内容与导出清单不一致")
if expected_identity is not None:
actual_identity = _agent_identity(directory) if kind == "agent" else _skill_identity(directory)
if actual_identity != expected_identity:
return ResourcePreflightItem(
kind=kind,
key=key,
relative_dir=relative_dir,
status="identity_mismatch",
message=f"目录内标识为 {actual_identity or '空'},清单标识为 {expected_identity}",
)
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="matched")
def preflight_manifest(
manifest: MigrationManifest,
*,
agents_root: Path,
skills_root: Path,
users_by_email: dict[str, str],
) -> MigrationPreflight:
"""Compare a manifest with target live roots without modifying anything."""
normalized_users = {email.strip().casefold(): user_id for email, user_id in users_by_email.items()}
resource_specs: list[dict[str, Any]] = [
{
"kind": "agent",
"key": item.id,
"root": agents_root,
"relative_dir": item.relative_dir,
"expected_hash": item.tree_hash,
"expected_identity": item.id,
}
for item in manifest.agents
]
resource_specs.extend(
{
"kind": "skill",
"key": f"{item.category}:{item.name}",
"root": skills_root,
"relative_dir": item.relative_dir,
"expected_hash": item.tree_hash,
"expected_identity": item.name,
}
for item in manifest.skills
)
resource_specs.extend(
{
"kind": "support",
"key": item.relative_dir,
"root": skills_root,
"relative_dir": item.relative_dir,
"expected_hash": item.tree_hash,
}
for item in manifest.support_directories
)
def _check_resource(spec: dict[str, Any]) -> ResourcePreflightItem:
return _resource_preflight(**spec)
if len(resource_specs) > 1:
with ThreadPoolExecutor(
max_workers=min(_TREE_HASH_MAX_WORKERS, len(resource_specs)),
thread_name_prefix="skill-migration-hash",
) as executor:
resources = list(executor.map(_check_resource, resource_specs))
else:
resources = [_check_resource(spec) for spec in resource_specs]
agent_count = len(manifest.agents)
skill_count = len(manifest.skills)
agents = resources[:agent_count]
skills = resources[agent_count : agent_count + skill_count]
support = resources[agent_count + skill_count :]
owners: dict[tuple[str | None, str | None], OwnerMappingItem] = {}
for resource in [*manifest.agents, *manifest.skills]:
owner = resource.owner
if owner is None or (owner.source_user_id is None and owner.email is None):
continue
email = owner.email.strip().casefold() if owner.email else None
target_user_id = normalized_users.get(email) if email else None
key = (owner.source_user_id, email)
owners[key] = OwnerMappingItem(
source_user_id=owner.source_user_id,
email=email,
target_user_id=target_user_id,
status="matched" if target_user_id else "unresolved",
)
owner_mappings = list(owners.values())
ready = all(item.status == "matched" for item in [*agents, *skills, *support]) and all(item.status == "matched" for item in owner_mappings)
return MigrationPreflight(
ready=ready,
agents=agents,
skills=skills,
support_directories=support,
owner_mappings=owner_mappings,
)
def prepare_manifest_for_apply(
manifest: MigrationManifest,
preflight: MigrationPreflight,
*,
skip_invalid_skills: bool,
) -> tuple[MigrationManifest, MigrationPreflight, list[str]]:
"""Optionally remove invalid skills and support-directory dependants.
Skipping is intentionally limited to skill-only manifests. Agent manifests
retain all-or-nothing semantics because their live ``config.yaml`` is the
authoritative source for several related database caches.
"""
if not skip_invalid_skills:
return manifest, preflight, []
if manifest.agents:
raise ValueError("跳过异常条目仅支持独立技能清单")
failed_skill_dirs = {
item.relative_dir for item in preflight.skills if item.status != "matched"
}
skipped_names = {
item.name for item in manifest.skills if item.relative_dir in failed_skill_dirs
}
support_by_dir = {
item.relative_dir: item for item in manifest.support_directories
}
for resource in preflight.support_directories:
if resource.status == "matched":
continue
support = support_by_dir.get(resource.relative_dir)
if support is not None:
skipped_names.update(support.required_by)
ordered_skipped = [item.name for item in manifest.skills if item.name in skipped_names]
kept_skills = [item for item in manifest.skills if item.name not in skipped_names]
kept_names = {item.name for item in kept_skills}
kept_support = [
item
for item in manifest.support_directories
if any(name in kept_names for name in item.required_by)
]
kept_skill_dirs = {item.relative_dir for item in kept_skills}
kept_support_dirs = {item.relative_dir for item in kept_support}
owner_keys = {
(
item.owner.source_user_id,
item.owner.email.strip().casefold() if item.owner and item.owner.email else None,
)
for item in kept_skills
if item.owner is not None
and (item.owner.source_user_id is not None or item.owner.email is not None)
}
owner_mappings = [
item
for item in preflight.owner_mappings
if (
item.source_user_id,
item.email.strip().casefold() if item.email else None,
)
in owner_keys
]
skill_resources = [
item for item in preflight.skills if item.relative_dir in kept_skill_dirs
]
support_resources = [
item
for item in preflight.support_directories
if item.relative_dir in kept_support_dirs
]
ready = all(
item.status == "matched"
for item in [*preflight.agents, *skill_resources, *support_resources]
) and all(item.status == "matched" for item in owner_mappings)
effective_manifest = manifest.model_copy(
update={
"skills": kept_skills,
"support_directories": kept_support,
}
)
effective_preflight = preflight.model_copy(
update={
"ready": ready,
"skills": skill_resources,
"support_directories": support_resources,
"owner_mappings": owner_mappings,
}
)
return effective_manifest, effective_preflight, ordered_skipped
def _owner_reference(user_id: str | None, users_by_id: dict[str, str]) -> OwnerReference | None:
if user_id is None:
return None
return OwnerReference(source_user_id=user_id, email=users_by_id.get(user_id))
def _load_agent_config(directory: Path) -> dict[str, Any]:
try:
payload = yaml.safe_load((directory / "config.yaml").read_text(encoding="utf-8")) or {}
except (OSError, UnicodeError, yaml.YAMLError) as exc:
raise ValueError(f"Cannot read agent config: {directory}") from exc
if not isinstance(payload, dict):
raise ValueError(f"Agent config must be an object: {directory}")
return payload
def _manifest_tags(rows: list[dict[str, Any]]) -> list[TagManifestItem]:
return [
TagManifestItem(
id=str(row.get("id") or ""),
name=str(row.get("name") or ""),
color=str(row.get("color") or "gray"),
description=str(row.get("description") or ""),
)
for row in rows
if row.get("id") and row.get("name")
]
async def build_agent_migration_manifest(
agent_ids: list[str],
*,
agent_store: Any,
tag_store: Any,
agents_root: Path,
users_by_id: dict[str, str],
) -> MigrationManifest:
"""Build an agent manifest without inspecting or exporting skill payloads."""
selected_ids = list(dict.fromkeys(agent_id.strip() for agent_id in agent_ids if agent_id.strip()))
if not selected_ids:
raise ValueError("Select at least one agent")
rows_by_id = {str(row["id"]): row for row in await agent_store.list_all()}
missing_agents = [agent_id for agent_id in selected_ids if agent_id not in rows_by_id]
if missing_agents:
raise ValueError(f"Unknown agents: {', '.join(missing_agents)}")
agent_tag_map = await tag_store.list_assignments("agent", selected_ids)
agents: list[AgentManifestItem] = []
warnings: list[str] = []
for agent_id in selected_ids:
row = rows_by_id[agent_id]
directory = _resolved_child(agents_root, agent_id.lower())
if directory is None or not directory.is_dir():
raise ValueError(f"Agent directory is missing: {agent_id}")
config = _load_agent_config(directory)
if str(config.get("id") or "") != agent_id:
raise ValueError(f"Agent config id does not match directory: {agent_id}")
raw_skills = config.get("skills")
selected_skills = list(dict.fromkeys(item.strip() for item in raw_skills if isinstance(item, str) and item.strip())) if isinstance(raw_skills, list) else []
knowledge_ids = config.get("llmwiki_knowledge_base_ids")
if isinstance(knowledge_ids, list) and any(str(item).strip() for item in knowledge_ids):
warnings.append(f"智能体 {agent_id} 绑定了知识库;知识库数据不在本迁移清单范围内,请在目标机器单独确认")
agents.append(
AgentManifestItem(
id=agent_id,
relative_dir=directory.relative_to(agents_root.resolve()).as_posix(),
tree_hash=await asyncio.to_thread(compute_tree_hash, directory),
name=str(row.get("name") or config.get("name") or agent_id),
description=str(row.get("description") or ""),
owner=_owner_reference(row.get("user_id"), users_by_id),
published=bool(row.get("published")),
is_private=bool(row.get("is_private")),
square_id=str(row.get("square_id") or ""),
featured_order=row.get("featured_order"),
created_at=row.get("created_at"),
updated_at=row.get("updated_at"),
skills=selected_skills,
tags=_manifest_tags(agent_tag_map.get(agent_id, [])),
)
)
return MigrationManifest(
bundle_id=uuid4().hex,
agents=agents,
warnings=warnings,
)
async def build_skill_migration_manifest(
skill_names: list[str],
*,
skill_store: Any,
tag_store: Any,
skills_root: Path,
discovered_skills: list[Any],
users_by_id: dict[str, str],
usage_entries: dict[str, dict[str, Any]],
) -> MigrationManifest:
"""Build a metadata-only manifest for explicitly selected live skills."""
selected_names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
if not selected_names:
raise ValueError("Select at least one skill")
skill_rows = {str(row["name"]): row for row in await skill_store.list_all()}
discovered = {str(skill.name): skill for skill in discovered_skills}
missing_skills = [name for name in selected_names if name not in discovered]
if missing_skills:
raise ValueError(f"Skills are missing from the live runtime directory: {', '.join(missing_skills)}")
skill_tag_map = await tag_store.list_assignments("skill", selected_names)
skill_directories = [Path(discovered[name].skill_dir).resolve() for name in selected_names]
tree_hashes = await _compute_tree_hashes(skill_directories)
skills: list[SkillManifestItem] = []
for name, tree_hash in zip(selected_names, tree_hashes, strict=True):
skill = discovered[name]
category = str(skill.category)
relative = PurePosixPath(category) / PurePosixPath(Path(skill.relative_path).as_posix())
directory = Path(skill.skill_dir).resolve()
row = skill_rows.get(name, {})
usage = usage_entries.get(name, {})
state = str(usage.get("state") or "active")
if state not in {"active", "stale", "archived"}:
state = "active"
source = str(usage.get("source") or "upload")
if source not in {"agent", "upload"}:
source = "upload"
skills.append(
SkillManifestItem(
name=name,
category=category,
relative_dir=relative.as_posix(),
tree_hash=tree_hash,
owner=_owner_reference(row.get("owner_user_id"), users_by_id),
published=bool(row.get("published", category == "public")),
detail=row.get("detail"),
name_zh=row.get("name_zh"),
always_on=bool(row.get("always_on", getattr(skill, "always_on", False))),
square_id=str(row.get("square_id") or ""),
enabled=bool(skill.enabled),
state=state,
pinned=bool(usage.get("pinned")),
featured=bool(usage.get("featured")),
source=source,
created_at=row.get("created_at") or usage.get("created_at"),
updated_at=row.get("updated_at"),
tags=_manifest_tags(skill_tag_map.get(name, [])),
)
)
selected_set = set(selected_names)
warnings: list[str] = []
support_directories: list[SupportDirectoryManifestItem] = []
support_candidates, support_warnings = await asyncio.to_thread(
_find_support_directories,
skills_root,
selected_set,
)
warnings.extend(support_warnings)
support_hashes = await _compute_tree_hashes(
[directory for directory, _required_by in support_candidates]
)
for (directory, selected_required_by), tree_hash in zip(
support_candidates,
support_hashes,
strict=True,
):
support_directories.append(
SupportDirectoryManifestItem(
relative_dir=directory.relative_to(skills_root.resolve()).as_posix(),
tree_hash=tree_hash,
required_by=selected_required_by,
)
)
return MigrationManifest(
bundle_id=uuid4().hex,
skills=skills,
support_directories=support_directories,
warnings=warnings,
)
async def _compute_tree_hashes(directories: list[Path]) -> list[str]:
if not directories:
return []
semaphore = asyncio.Semaphore(min(_TREE_HASH_MAX_WORKERS, len(directories)))
async def _compute(directory: Path) -> str:
async with semaphore:
return await asyncio.to_thread(compute_tree_hash, directory)
return list(await asyncio.gather(*(_compute(directory) for directory in directories)))
def _find_support_directories(
skills_root: Path,
selected_names: set[str],
) -> tuple[list[tuple[Path, list[str]]], list[str]]:
candidates: list[tuple[Path, list[str]]] = []
warnings: list[str] = []
for support_manifest in sorted(skills_root.rglob(".deerflow-skill-support.json")):
try:
payload = json.loads(support_manifest.read_text(encoding="utf-8"))
required_by = [str(item).strip() for item in payload.get("required_by", []) if str(item).strip()]
except (OSError, UnicodeError, json.JSONDecodeError, AttributeError):
warnings.append(f"无法读取技能支持目录清单:{support_manifest}")
continue
selected_required_by = [name for name in required_by if name in selected_names]
if not selected_required_by:
continue
candidates.append((support_manifest.parent.resolve(), selected_required_by))
return candidates, warnings
def _apply_usage_metadata(name: str, metadata: dict[str, Any]) -> None:
_apply_usage_metadata_batch({name: metadata})
def _apply_usage_metadata_batch(entries: dict[str, dict[str, Any]]) -> None:
from deerflow.skills.usage import set_migration_metadata_batch
set_migration_metadata_batch(entries)
def _target_owner_id(
owner: OwnerReference | None,
*,
users_by_email: dict[str, str],
fallback_owner_user_id: str | None,
) -> tuple[str | None, bool]:
if owner is None or (owner.source_user_id is None and owner.email is None):
return None, False
target = users_by_email.get(owner.email.casefold()) if owner.email else None
if target:
return target, False
if fallback_owner_user_id:
return fallback_owner_user_id, True
raise ValueError(f"Unresolved owner: {owner.email or owner.source_user_id}")
async def _apply_tags(tag_store: Any, target_type: str, target_id: str, tags: list[TagManifestItem]) -> int:
return await _apply_tags_bulk(tag_store, target_type, [(target_id, tags)])
async def _apply_tags_bulk(
tag_store: Any,
target_type: str,
targets: list[tuple[str, list[TagManifestItem]]],
) -> int:
if not targets:
return 0
existing = await tag_store.list_tags()
by_name = {
(str(row.get("tag_type")), str(row.get("name"))): row
for row in existing
}
used_ids = {str(row.get("id")) for row in existing}
assignments: dict[str, list[str]] = {}
created = 0
for target_id, tags in targets:
tag_ids: list[str] = []
for tag in tags:
same_name = by_name.get((target_type, tag.name))
if same_name is not None:
tag_ids.append(str(same_name["id"]))
continue
tag_id = tag.id if tag.id not in used_ids else uuid4().hex
created_tag = await tag_store.create_tag(
{
"id": tag_id,
"name": tag.name,
"tag_type": target_type,
"color": tag.color,
"description": tag.description,
}
)
by_name[(target_type, tag.name)] = created_tag
used_ids.add(tag_id)
tag_ids.append(tag_id)
created += 1
assignments[target_id] = tag_ids
await tag_store.set_target_tags_bulk(target_type, assignments)
return created
async def apply_migration_manifest(
manifest: MigrationManifest,
*,
preflight: MigrationPreflight,
agent_store: Any,
skill_store: Any,
tag_store: Any,
agents_root: Path,
users_by_email: dict[str, str],
fallback_owner_user_id: str | None,
extensions_config_path: Path | None,
skip_invalid_skills: bool = False,
) -> MigrationApplyResult:
"""Idempotently reconcile metadata after all live folders passed preflight."""
manifest, preflight, skipped_skill_names = prepare_manifest_for_apply(
manifest,
preflight,
skip_invalid_skills=skip_invalid_skills,
)
bad_resources = [item for item in [*preflight.agents, *preflight.skills, *preflight.support_directories] if item.status != "matched"]
if bad_resources:
raise ValueError("Migration preflight contains missing or mismatched directories")
if any(item.status == "unresolved" for item in preflight.owner_mappings) and not fallback_owner_user_id:
raise ValueError("Migration preflight contains unresolved owners")
if manifest.skills:
if extensions_config_path is None:
raise ValueError("目标运行环境未找到 extensions_config.json")
from deerflow.config.extensions_config import patch_extensions_config
enabled_by_name = {item.name: item.enabled for item in manifest.skills}
def _patch(document: dict[str, Any]) -> None:
skills_document = document.setdefault("skills", {})
if not isinstance(skills_document, dict):
raise ValueError("extensions_config.json skills must be an object")
for name, enabled in enabled_by_name.items():
current = skills_document.get(name)
entry = dict(current) if isinstance(current, dict) else {}
entry["enabled"] = enabled
skills_document[name] = entry
patch_extensions_config(extensions_config_path, _patch)
normalized_users = {email.strip().casefold(): user_id for email, user_id in users_by_email.items()}
fallback_count = 0
tags_created = 0
skill_payloads: list[dict[str, Any]] = []
skill_usage_entries: dict[str, dict[str, Any]] = {}
for item in manifest.skills:
owner_user_id, used_fallback = _target_owner_id(
item.owner,
users_by_email=normalized_users,
fallback_owner_user_id=fallback_owner_user_id,
)
fallback_count += int(used_fallback)
skill_payloads.append(
{
"name": item.name,
"owner_user_id": owner_user_id,
"published": item.published,
"detail": item.detail,
"name_zh": item.name_zh,
"always_on": item.always_on,
"square_id": item.square_id,
"created_at": item.created_at,
"updated_at": item.updated_at,
}
)
skill_usage_entries[item.name] = {
"state": item.state,
"pinned": item.pinned,
"featured": item.featured,
"source": item.source,
}
if skill_payloads:
await skill_store.migration_upsert_many(skill_payloads)
tags_created += await _apply_tags_bulk(
tag_store,
"skill",
[(item.name, item.tags) for item in manifest.skills],
)
_apply_usage_metadata_batch(skill_usage_entries)
agent_tag_targets: list[tuple[str, list[TagManifestItem]]] = []
for item in manifest.agents:
owner_user_id, used_fallback = _target_owner_id(
item.owner,
users_by_email=normalized_users,
fallback_owner_user_id=fallback_owner_user_id,
)
fallback_count += int(used_fallback)
await agent_store.migration_upsert(
{
"id": item.id,
"name": item.name,
"description": item.description,
"user_id": owner_user_id,
"published": item.published,
"is_private": item.is_private,
"square_id": item.square_id,
"featured_order": item.featured_order,
"created_at": item.created_at,
"updated_at": item.updated_at,
}
)
config = _load_agent_config(agents_root / item.relative_dir)
await agent_store.replace_skills(item.id, config.get("skills") if isinstance(config.get("skills"), list) else [])
await agent_store.set_extras(
item.id,
tool_groups=config.get("tool_groups") if isinstance(config.get("tool_groups"), list) else None,
model=str(config.get("model")) if config.get("model") else None,
)
agent_tag_targets.append((item.id, item.tags))
tags_created += await _apply_tags_bulk(tag_store, "agent", agent_tag_targets)
return MigrationApplyResult(
bundle_id=manifest.bundle_id,
agents_applied=len(manifest.agents),
skills_applied=len(manifest.skills),
skills_skipped=len(skipped_skill_names),
skipped_skill_names=skipped_skill_names,
tags_created=tags_created,
fallback_owner_count=fallback_count,
)
@@ -37,6 +37,7 @@ from app.gateway.routers import (
admin_active_runs,
admin_audit_retention as admin_audit_retention_routes,
admin_users,
agent_skill_migrations,
agents,
ai_writing,
app_identity,
@@ -48,6 +49,7 @@ from app.gateway.routers import (
browser_context,
business_mapping,
channels,
chat_history_transfer,
chat_archive,
checkpoint_migration,
compact_menu_overrides,
@@ -1204,6 +1206,30 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
logger.exception(error_msg)
raise RuntimeError(error_msg) from e
# Populate product-owned public skills before any ownership migration,
# MySQL catalogue projection, prompt cache, or sandbox view observes the
# configured live root. This application-level guard is intentionally in
# addition to the supported shell launchers: image entrypoints, Compose,
# and external release wrappers must receive the same built-ins.
try:
from app.builtin_skill_sync import ensure_bundled_public_skills
sync_result = await asyncio.to_thread(
ensure_bundled_public_skills,
app.state.config,
)
logger.info(
"Bundled public skills synchronized: source=%s target=%s copied=%d required=%d",
sync_result["source"],
sync_result["target"],
len(sync_result["copiedDirectories"]),
len(sync_result["requiredSkills"]),
)
except Exception as e:
error_msg = f"Failed to synchronize required built-in skills during gateway startup: {e}"
logger.exception(error_msg)
raise RuntimeError(error_msg) from e
config = get_gateway_config()
logger.info(f"Starting API Gateway on {config.host}:{config.port}")
@@ -1276,6 +1302,20 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Skill MySQL catalog projection sync failed (non-fatal)")
# Project the compact per-skill catalog metadata into MySQL before the
# Gateway accepts requests. List APIs read indexed scalar columns and
# never load skill markdown, display samples, or usage sidecars.
try:
from app.gateway.skill_catalog_cache import refresh_skill_catalog_projection
catalog_size = await refresh_skill_catalog_projection(
app.state.config,
skills=catalog_skills,
)
logger.info("Skill MySQL catalog projection synchronized: skills=%d", catalog_size)
except Exception:
logger.exception("Skill MySQL catalog projection sync failed (non-fatal)")
# ????? agent(intent / recommender / coordinator)???????
# ??? _sync_legacy_agents **??**??????????????????
# ??? upsert ? agents ?,?????? /api/agents ?????,
@@ -1462,6 +1502,32 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
logger.exception(
"Elasticsearch offline operations agent seeding failed at startup (non-fatal)"
)
# 内置「聊天记录迁移助手」:高斯/SQLite 会话索引、检查点和线程文件分开迁移。
# 目录已存在时不覆盖管理员修改。
try:
from app.gateway.routers._chat_history_migration_seed import (
ensure_chat_history_migration_agent,
)
ensure_chat_history_migration_agent()
except Exception:
logger.exception(
"Chat-history migration agent seeding failed at startup (non-fatal)"
)
# 内置「高斯数据适配助手」:根据报错说明要改的配置。目录已存在时不覆盖。
try:
from app.gateway.routers._gauss_data_adaptation_seed import (
ensure_gauss_data_adaptation_agent,
)
ensure_gauss_data_adaptation_agent()
except Exception:
logger.exception(
"Gauss data-adaptation agent seeding failed at startup (non-fatal)"
)
# 内置「任务研判报告助手」(agentfx 任务深链固定使用):同样必须在
# _sync_legacy_agents 之前补建目录 + 补建配套 task-report-import 技能
# (skills/ 被 .gitignore 排除),由后者 upsert 进 agents 表。
@@ -1587,6 +1653,13 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
except Exception:
logger.exception("Admin audit retention scheduler failed to start (non-fatal)")
try:
app.state.admin_audit_retention_task = asyncio.create_task(
_admin_audit_retention_loop(app)
)
except Exception:
logger.exception("Admin audit retention scheduler failed to start (non-fatal)")
# Start IM channel service if any channels are configured
try:
from app.channels.service import start_channel_service
@@ -1765,6 +1838,25 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
await close_api_read_cache()
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
from app.gateway.worker_pool.lifecycle import managed_controller
from app.gateway.perf_logging import event_loop_watchdog, register_stack_dump_signal
register_stack_dump_signal()
watchdog = asyncio.create_task(event_loop_watchdog(app), name="gateway-event-loop-watchdog")
try:
async with managed_controller(app):
async with _gateway_lifespan(app):
yield
finally:
watchdog.cancel()
await asyncio.gather(watchdog, return_exceptions=True)
from app.gateway.api_read_cache import close_api_read_cache
await close_api_read_cache()
def create_app() -> FastAPI:
"""Create and configure the FastAPI application.
@@ -2097,11 +2189,14 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
app.include_router(analysis_projects.router)
app.include_router(threads.router)
app.include_router(chat_history_transfer.router)
# Dedicated taskId + goPath history lookup for single-agent task deep-link chats.
app.include_router(task_deeplink_threads.router)
app.include_router(task_deeplink_details.router)
# Agents API is mounted at /api/agents
app.include_router(agents.router)
# Metadata-only migration for agent/skill folders copied into live roots.
app.include_router(agent_skill_migrations.router)
app.include_router(strategy_agents.router)
# Recommended Questions API is mounted at /api/recommended-questions
@@ -1,401 +1,410 @@
"""Authorization decorators and context for DeerFlow.
Inspired by LangGraph Auth system: https://github.com/langchain-ai/langgraph/blob/main/libs/sdk-py/langgraph_sdk/auth/__init__.py
**Usage:**
1. Use ``@require_auth`` on routes that need authentication
2. Use ``@require_permission("resource", "action", filter_key=...)`` for permission checks
3. The decorator chain processes from bottom to top
**Example:**
@router.get("/{thread_id}")
@require_auth
@require_permission("threads", "read", owner_check=True)
async def get_thread(thread_id: str, request: Request):
# User is authenticated and has threads:read permission
...
**Permission Model:**
- threads:read - View thread
- threads:write - Create/update thread
- threads:delete - Delete thread
- runs:create - Run agent
- runs:read - View run
- runs:cancel - Cancel run
"""
from __future__ import annotations
import asyncio
import functools
import inspect
import logging
from collections.abc import Callable
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from fastapi import HTTPException, Request
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
if TYPE_CHECKING:
from app.gateway.auth.models import User
logger = logging.getLogger(__name__)
P = ParamSpec("P")
T = TypeVar("T")
async def _project_for_thread(project_store: Any, thread_id: str) -> dict | None:
"""Best-effort project binding lookup; a store error must not fail the read."""
try:
return await project_store.project_for_thread(thread_id)
except Exception:
logger.debug("project_for_thread failed for %s", thread_id, exc_info=True)
return None
async def _thread_checkpoint_exists(request: Request, thread_id: str) -> bool:
"""Return True if a LangGraph checkpoint exists for ``thread_id``.
A checkpoint with no matching ``threads_meta`` row is the signature of a
DB-switch orphan (sqlite↔mysql/postgres juggling): the conversation state
survives in the checkpointer while the business row is gone. Normal thread
deletion removes the checkpoint too, so this can never resurrect a
deliberately-deleted thread.
"""
try:
from app.gateway.deps import get_checkpointer
checkpointer = get_checkpointer(request)
tup = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
return tup is not None
except Exception:
return False
# Permission constants
class Permissions:
"""Permission constants for resource:action format."""
# Threads
THREADS_READ = "threads:read"
THREADS_WRITE = "threads:write"
THREADS_DELETE = "threads:delete"
# Runs
RUNS_CREATE = "runs:create"
RUNS_READ = "runs:read"
RUNS_CANCEL = "runs:cancel"
class AuthContext:
"""Authentication context for the current request.
Stored in request.state.auth after require_auth decoration.
Attributes:
user: The authenticated user, or None if anonymous
permissions: List of permission strings (e.g., "threads:read")
"""
__slots__ = ("user", "permissions")
def __init__(self, user: User | None = None, permissions: list[str] | None = None):
self.user = user
self.permissions = permissions or []
@property
def is_authenticated(self) -> bool:
"""Check if user is authenticated."""
return self.user is not None
def has_permission(self, resource: str, action: str) -> bool:
"""Check if context has permission for resource:action.
Args:
resource: Resource name (e.g., "threads")
action: Action name (e.g., "read")
Returns:
True if user has permission
"""
permission = f"{resource}:{action}"
return permission in self.permissions
def require_user(self) -> User:
"""Get user or raise 401.
Raises:
HTTPException 401 if not authenticated
"""
if not self.user:
raise HTTPException(status_code=401, detail="Authentication required")
return self.user
def get_auth_context(request: Request) -> AuthContext | None:
"""Get AuthContext from request state."""
return getattr(request.state, "auth", None)
_ALL_PERMISSIONS: list[str] = [
Permissions.THREADS_READ,
Permissions.THREADS_WRITE,
Permissions.THREADS_DELETE,
Permissions.RUNS_CREATE,
Permissions.RUNS_READ,
Permissions.RUNS_CANCEL,
]
def _make_test_request_stub() -> Any:
"""Create a minimal request-like object for direct unit calls.
Used when decorated route handlers are invoked without FastAPI's
request injection. Includes fields accessed by auth helpers.
"""
return SimpleNamespace(state=SimpleNamespace(), cookies={}, _deerflow_test_bypass_auth=True)
async def _authenticate(request: Request) -> AuthContext:
"""Authenticate request and return AuthContext.
Delegates to deps.get_optional_user_from_request() for the JWT→User pipeline.
Returns AuthContext with user=None for anonymous requests.
"""
if is_auth_disabled():
user = await get_assumed_user_for_disabled_auth()
return AuthContext(user=user, permissions=_ALL_PERMISSIONS)
from app.gateway.deps import get_optional_user_from_request
user = await get_optional_user_from_request(request)
if user is None:
return AuthContext(user=None, permissions=[])
# In future, permissions could be stored in user record
return AuthContext(user=user, permissions=_ALL_PERMISSIONS)
def require_auth[**P, T](func: Callable[P, T]) -> Callable[P, T]:
"""Decorator that authenticates the request and enforces authentication.
Independently raises HTTP 401 for unauthenticated requests, regardless of
whether ``AuthMiddleware`` is present in the ASGI stack. Sets the resolved
``AuthContext`` on ``request.state.auth`` for downstream handlers.
Must be placed ABOVE other decorators (executes after them).
Usage:
@router.get("/{thread_id}")
@require_auth # Bottom decorator (executes first after permission check)
@require_permission("threads", "read")
async def get_thread(thread_id: str, request: Request):
auth: AuthContext = request.state.auth
...
Raises:
HTTPException: 401 if the request is unauthenticated.
ValueError: If 'request' parameter is missing.
"""
@functools.wraps(func)
async def wrapper(*args: Any, **kwargs: Any) -> Any:
request = kwargs.get("request")
if request is None:
# Unit tests may call decorated handlers directly without a
# FastAPI Request object. Inject a minimal request stub when
# the wrapped function declares `request`.
if "request" in inspect.signature(func).parameters:
kwargs["request"] = _make_test_request_stub()
else:
raise ValueError("require_auth decorator requires 'request' parameter")
request = kwargs["request"]
if getattr(request, "_deerflow_test_bypass_auth", False):
return await func(*args, **kwargs)
# Authenticate and set context
auth_context = await _authenticate(request)
request.state.auth = auth_context
if not auth_context.is_authenticated:
raise HTTPException(status_code=401, detail="Authentication required")
return await func(*args, **kwargs)
return wrapper
def require_permission(
resource: str,
action: str,
owner_check: bool = False,
require_existing: bool = False,
claim_orphan: bool = False,
) -> Callable[[Callable[P, T]], Callable[P, T]]:
"""Decorator that checks permission for resource:action.
Must be used AFTER @require_auth.
Args:
resource: Resource name (e.g., "threads", "runs")
action: Action name (e.g., "read", "write", "delete")
owner_check: If True, validates that the current user owns the resource.
Requires 'thread_id' path parameter and performs ownership check.
require_existing: Only meaningful with ``owner_check=True``. If True, a
missing ``threads_meta`` row counts as a denial (404)
instead of "untracked legacy thread, allow". Use on
**destructive / mutating** routes (DELETE, PATCH,
state-update) so a deleted thread can't be re-targeted
by another user via the missing-row code path.
claim_orphan: Only meaningful with ``owner_check=True`` and
``require_existing=True``. When the ``threads_meta``
row is missing **and** a LangGraph checkpoint exists
for the thread (a DB-switch orphan), adopt it for the
current user instead of 404ing. Use on **run-create**
routes so a conversation whose business row was lost
(sqlite↔mysql swap) stays answerable. Safe: normal
deletion removes the checkpoint, so this never revives
a deliberately-deleted thread.
Usage:
# Read-style: legacy untracked threads are allowed
@require_permission("threads", "read", owner_check=True)
async def get_thread(thread_id: str, request: Request):
...
# Destructive: thread row MUST exist and be owned by caller
@require_permission("threads", "delete", owner_check=True, require_existing=True)
async def delete_thread(thread_id: str, request: Request):
...
Raises:
HTTPException 401: If authentication required but user is anonymous
HTTPException 403: If user lacks permission
HTTPException 404: If owner_check=True but user doesn't own the thread
ValueError: If owner_check=True but no ``thread_id`` is supplied,
either directly or on the Pydantic request body
"""
def decorator(func: Callable[P, T]) -> Callable[P, T]:
@functools.wraps(func)
async def wrapper(*args: Any, **kwargs: Any) -> Any:
request = kwargs.get("request")
if request is None:
# Unit tests may call decorated route handlers directly without
# constructing a FastAPI Request object. Inject a minimal stub
# when the wrapped function declares `request`.
if "request" in inspect.signature(func).parameters:
kwargs["request"] = _make_test_request_stub()
else:
return await func(*args, **kwargs)
request = kwargs["request"]
if getattr(request, "_deerflow_test_bypass_auth", False):
return await func(*args, **kwargs)
auth: AuthContext = getattr(request.state, "auth", None)
if auth is None:
auth = await _authenticate(request)
request.state.auth = auth
if not auth.is_authenticated:
raise HTTPException(status_code=401, detail="Authentication required")
# Check permission
if not auth.has_permission(resource, action):
raise HTTPException(
status_code=403,
detail=f"Permission denied: {resource}:{action}",
)
# Owner check for thread-specific resources.
#
# 2.0-rc moved thread metadata into the SQL persistence layer
# (``threads_meta`` table). We verify ownership via
# ``ThreadMetaStore.check_access``: it returns True for
# missing rows (untracked legacy thread) and for rows whose
# ``user_id`` is NULL (shared / pre-auth data), so this is
# strict-deny rather than strict-allow — only an *existing*
# row with a *different* user_id triggers 404.
if owner_check:
thread_id = kwargs.get("thread_id")
# Most thread-scoped routes expose ``thread_id`` in their URL.
# A small number of mutation endpoints (for example complete
# document rewrite) intentionally keep the target inside a
# Pydantic request body so their public URL stays stable. The
# FastAPI wrapper passes that body as ``body=...``; accept its
# Python field name here while preserving the exact same
# ownership verification below.
if thread_id is None:
body = kwargs.get("body")
thread_id = getattr(body, "thread_id", None)
if thread_id is None:
raise ValueError(
"require_permission with owner_check=True requires "
"a 'thread_id' parameter or body.thread_id"
)
from app.gateway.deps import get_thread_store
thread_store = get_thread_store(request)
user_id = str(auth.user.id)
project_store = getattr(request.app.state, "analysis_project_store", None)
allowed_coro = thread_store.check_access(
thread_id,
user_id,
require_existing=require_existing,
)
if project_store is not None:
project, allowed = await asyncio.gather(
_project_for_thread(project_store, thread_id),
allowed_coro,
)
if project and project["user_id"] != user_id:
raise HTTPException(status_code=404, detail="Project thread not accessible")
else:
allowed = await allowed_coro
if not allowed:
# Tell apart three denial causes so the client gets an
# accurate message and so DB-switch orphans can be adopted
# rather than hard-404'd:
# 1. row exists but owned by another user -> not accessible
# 2. no row, no checkpoint -> truly missing
# 3. no row + checkpoint + claim_orphan -> orphan, claim
existing = await thread_store.get(thread_id, user_id=None)
if existing is not None:
# Distinct detail string lets the frontend show "无权访问"
# instead of "不存在" for a cross-user / stale-session hit.
raise HTTPException(
status_code=404,
detail=f"Thread {thread_id} not accessible",
)
claimed = False
if claim_orphan and await _thread_checkpoint_exists(request, thread_id):
try:
await thread_store.create(thread_id, user_id=user_id)
logger.info(
"Claimed orphan thread %s for user %s (checkpoint present, no meta row)",
thread_id,
user_id,
)
claimed = True
except Exception:
logger.warning("Failed to claim orphan thread %s", thread_id, exc_info=True)
if not claimed:
raise HTTPException(
status_code=404,
detail=f"Thread {thread_id} not found",
)
return await func(*args, **kwargs)
return wrapper
return decorator
"""Authorization decorators and context for DeerFlow.
Inspired by LangGraph Auth system: https://github.com/langchain-ai/langgraph/blob/main/libs/sdk-py/langgraph_sdk/auth/__init__.py
**Usage:**
1. Use ``@require_auth`` on routes that need authentication
2. Use ``@require_permission("resource", "action", filter_key=...)`` for permission checks
3. The decorator chain processes from bottom to top
**Example:**
@router.get("/{thread_id}")
@require_auth
@require_permission("threads", "read", owner_check=True)
async def get_thread(thread_id: str, request: Request):
# User is authenticated and has threads:read permission
...
**Permission Model:**
- threads:read - View thread
- threads:write - Create/update thread
- threads:delete - Delete thread
- runs:create - Run agent
- runs:read - View run
- runs:cancel - Cancel run
"""
from __future__ import annotations
import asyncio
import functools
import inspect
import logging
from collections.abc import Callable
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from fastapi import HTTPException, Request
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
if TYPE_CHECKING:
from app.gateway.auth.models import User
logger = logging.getLogger(__name__)
P = ParamSpec("P")
T = TypeVar("T")
async def _project_for_thread(project_store: Any, thread_id: str) -> dict | None:
"""Best-effort project binding lookup; a store error must not fail the read."""
try:
return await project_store.project_for_thread(thread_id)
except Exception:
logger.debug("project_for_thread failed for %s", thread_id, exc_info=True)
return None
async def _project_for_thread(project_store: Any, thread_id: str) -> dict | None:
"""Best-effort project binding lookup; a store error must not fail the read."""
try:
return await project_store.project_for_thread(thread_id)
except Exception:
logger.debug("project_for_thread failed for %s", thread_id, exc_info=True)
return None
async def _thread_checkpoint_exists(request: Request, thread_id: str) -> bool:
"""Return True if a LangGraph checkpoint exists for ``thread_id``.
A checkpoint with no matching ``threads_meta`` row is the signature of a
DB-switch orphan (sqlite↔mysql/postgres juggling): the conversation state
survives in the checkpointer while the business row is gone. Normal thread
deletion removes the checkpoint too, so this can never resurrect a
deliberately-deleted thread.
"""
try:
from app.gateway.deps import get_checkpointer
checkpointer = get_checkpointer(request)
tup = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
return tup is not None
except Exception:
return False
# Permission constants
class Permissions:
"""Permission constants for resource:action format."""
# Threads
THREADS_READ = "threads:read"
THREADS_WRITE = "threads:write"
THREADS_DELETE = "threads:delete"
# Runs
RUNS_CREATE = "runs:create"
RUNS_READ = "runs:read"
RUNS_CANCEL = "runs:cancel"
class AuthContext:
"""Authentication context for the current request.
Stored in request.state.auth after require_auth decoration.
Attributes:
user: The authenticated user, or None if anonymous
permissions: List of permission strings (e.g., "threads:read")
"""
__slots__ = ("user", "permissions")
def __init__(self, user: User | None = None, permissions: list[str] | None = None):
self.user = user
self.permissions = permissions or []
@property
def is_authenticated(self) -> bool:
"""Check if user is authenticated."""
return self.user is not None
def has_permission(self, resource: str, action: str) -> bool:
"""Check if context has permission for resource:action.
Args:
resource: Resource name (e.g., "threads")
action: Action name (e.g., "read")
Returns:
True if user has permission
"""
permission = f"{resource}:{action}"
return permission in self.permissions
def require_user(self) -> User:
"""Get user or raise 401.
Raises:
HTTPException 401 if not authenticated
"""
if not self.user:
raise HTTPException(status_code=401, detail="Authentication required")
return self.user
def get_auth_context(request: Request) -> AuthContext | None:
"""Get AuthContext from request state."""
return getattr(request.state, "auth", None)
_ALL_PERMISSIONS: list[str] = [
Permissions.THREADS_READ,
Permissions.THREADS_WRITE,
Permissions.THREADS_DELETE,
Permissions.RUNS_CREATE,
Permissions.RUNS_READ,
Permissions.RUNS_CANCEL,
]
def _make_test_request_stub() -> Any:
"""Create a minimal request-like object for direct unit calls.
Used when decorated route handlers are invoked without FastAPI's
request injection. Includes fields accessed by auth helpers.
"""
return SimpleNamespace(state=SimpleNamespace(), cookies={}, _deerflow_test_bypass_auth=True)
async def _authenticate(request: Request) -> AuthContext:
"""Authenticate request and return AuthContext.
Delegates to deps.get_optional_user_from_request() for the JWT→User pipeline.
Returns AuthContext with user=None for anonymous requests.
"""
if is_auth_disabled():
user = await get_assumed_user_for_disabled_auth()
return AuthContext(user=user, permissions=_ALL_PERMISSIONS)
from app.gateway.deps import get_optional_user_from_request
user = await get_optional_user_from_request(request)
if user is None:
return AuthContext(user=None, permissions=[])
# In future, permissions could be stored in user record
return AuthContext(user=user, permissions=_ALL_PERMISSIONS)
def require_auth[**P, T](func: Callable[P, T]) -> Callable[P, T]:
"""Decorator that authenticates the request and enforces authentication.
Independently raises HTTP 401 for unauthenticated requests, regardless of
whether ``AuthMiddleware`` is present in the ASGI stack. Sets the resolved
``AuthContext`` on ``request.state.auth`` for downstream handlers.
Must be placed ABOVE other decorators (executes after them).
Usage:
@router.get("/{thread_id}")
@require_auth # Bottom decorator (executes first after permission check)
@require_permission("threads", "read")
async def get_thread(thread_id: str, request: Request):
auth: AuthContext = request.state.auth
...
Raises:
HTTPException: 401 if the request is unauthenticated.
ValueError: If 'request' parameter is missing.
"""
@functools.wraps(func)
async def wrapper(*args: Any, **kwargs: Any) -> Any:
request = kwargs.get("request")
if request is None:
# Unit tests may call decorated handlers directly without a
# FastAPI Request object. Inject a minimal request stub when
# the wrapped function declares `request`.
if "request" in inspect.signature(func).parameters:
kwargs["request"] = _make_test_request_stub()
else:
raise ValueError("require_auth decorator requires 'request' parameter")
request = kwargs["request"]
if getattr(request, "_deerflow_test_bypass_auth", False):
return await func(*args, **kwargs)
# Authenticate and set context
auth_context = await _authenticate(request)
request.state.auth = auth_context
if not auth_context.is_authenticated:
raise HTTPException(status_code=401, detail="Authentication required")
return await func(*args, **kwargs)
return wrapper
def require_permission(
resource: str,
action: str,
owner_check: bool = False,
require_existing: bool = False,
claim_orphan: bool = False,
) -> Callable[[Callable[P, T]], Callable[P, T]]:
"""Decorator that checks permission for resource:action.
Must be used AFTER @require_auth.
Args:
resource: Resource name (e.g., "threads", "runs")
action: Action name (e.g., "read", "write", "delete")
owner_check: If True, validates that the current user owns the resource.
Requires 'thread_id' path parameter and performs ownership check.
require_existing: Only meaningful with ``owner_check=True``. If True, a
missing ``threads_meta`` row counts as a denial (404)
instead of "untracked legacy thread, allow". Use on
**destructive / mutating** routes (DELETE, PATCH,
state-update) so a deleted thread can't be re-targeted
by another user via the missing-row code path.
claim_orphan: Only meaningful with ``owner_check=True`` and
``require_existing=True``. When the ``threads_meta``
row is missing **and** a LangGraph checkpoint exists
for the thread (a DB-switch orphan), adopt it for the
current user instead of 404ing. Use on **run-create**
routes so a conversation whose business row was lost
(sqlite↔mysql swap) stays answerable. Safe: normal
deletion removes the checkpoint, so this never revives
a deliberately-deleted thread.
Usage:
# Read-style: legacy untracked threads are allowed
@require_permission("threads", "read", owner_check=True)
async def get_thread(thread_id: str, request: Request):
...
# Destructive: thread row MUST exist and be owned by caller
@require_permission("threads", "delete", owner_check=True, require_existing=True)
async def delete_thread(thread_id: str, request: Request):
...
Raises:
HTTPException 401: If authentication required but user is anonymous
HTTPException 403: If user lacks permission
HTTPException 404: If owner_check=True but user doesn't own the thread
ValueError: If owner_check=True but no ``thread_id`` is supplied,
either directly or on the Pydantic request body
"""
def decorator(func: Callable[P, T]) -> Callable[P, T]:
@functools.wraps(func)
async def wrapper(*args: Any, **kwargs: Any) -> Any:
request = kwargs.get("request")
if request is None:
# Unit tests may call decorated route handlers directly without
# constructing a FastAPI Request object. Inject a minimal stub
# when the wrapped function declares `request`.
if "request" in inspect.signature(func).parameters:
kwargs["request"] = _make_test_request_stub()
else:
return await func(*args, **kwargs)
request = kwargs["request"]
if getattr(request, "_deerflow_test_bypass_auth", False):
return await func(*args, **kwargs)
auth: AuthContext = getattr(request.state, "auth", None)
if auth is None:
auth = await _authenticate(request)
request.state.auth = auth
if not auth.is_authenticated:
raise HTTPException(status_code=401, detail="Authentication required")
# Check permission
if not auth.has_permission(resource, action):
raise HTTPException(
status_code=403,
detail=f"Permission denied: {resource}:{action}",
)
# Owner check for thread-specific resources.
#
# 2.0-rc moved thread metadata into the SQL persistence layer
# (``threads_meta`` table). We verify ownership via
# ``ThreadMetaStore.check_access``: it returns True for
# missing rows (untracked legacy thread) and for rows whose
# ``user_id`` is NULL (shared / pre-auth data), so this is
# strict-deny rather than strict-allow — only an *existing*
# row with a *different* user_id triggers 404.
if owner_check:
thread_id = kwargs.get("thread_id")
# Most thread-scoped routes expose ``thread_id`` in their URL.
# A small number of mutation endpoints (for example complete
# document rewrite) intentionally keep the target inside a
# Pydantic request body so their public URL stays stable. The
# FastAPI wrapper passes that body as ``body=...``; accept its
# Python field name here while preserving the exact same
# ownership verification below.
if thread_id is None:
body = kwargs.get("body")
thread_id = getattr(body, "thread_id", None)
if thread_id is None:
raise ValueError(
"require_permission with owner_check=True requires "
"a 'thread_id' parameter or body.thread_id"
)
from app.gateway.deps import get_thread_store
thread_store = get_thread_store(request)
user_id = str(auth.user.id)
project_store = getattr(request.app.state, "analysis_project_store", None)
allowed_coro = thread_store.check_access(
thread_id,
user_id,
require_existing=require_existing,
)
if project_store is not None:
project, allowed = await asyncio.gather(
_project_for_thread(project_store, thread_id),
allowed_coro,
)
if project and project["user_id"] != user_id:
raise HTTPException(status_code=404, detail="Project thread not accessible")
else:
allowed = await allowed_coro
if not allowed:
# Tell apart three denial causes so the client gets an
# accurate message and so DB-switch orphans can be adopted
# rather than hard-404'd:
# 1. row exists but owned by another user -> not accessible
# 2. no row, no checkpoint -> truly missing
# 3. no row + checkpoint + claim_orphan -> orphan, claim
existing = await thread_store.get(thread_id, user_id=None)
if existing is not None:
# Distinct detail string lets the frontend show "无权访问"
# instead of "不存在" for a cross-user / stale-session hit.
raise HTTPException(
status_code=404,
detail=f"Thread {thread_id} not accessible",
)
claimed = False
if claim_orphan and await _thread_checkpoint_exists(request, thread_id):
try:
await thread_store.create(thread_id, user_id=user_id)
logger.info(
"Claimed orphan thread %s for user %s (checkpoint present, no meta row)",
thread_id,
user_id,
)
claimed = True
except Exception:
logger.warning("Failed to claim orphan thread %s", thread_id, exc_info=True)
if not claimed:
raise HTTPException(
status_code=404,
detail=f"Thread {thread_id} not found",
)
return await func(*args, **kwargs)
return wrapper
return decorator
@@ -204,6 +204,35 @@ class RoundtableJobDispatcher:
)
return expired
async def _expire_overdue_step2_rooms(self, now: datetime) -> int:
"""Persist hard-timeout cancellation even when no browser is connected."""
max_minutes = getattr(
get_roundtable_room_limit_settings(),
"max_hold_minutes",
30,
)
request_cancel_expired = getattr(
self._store,
"request_cancel_expired_step2",
None,
)
if request_cancel_expired is None:
return 0
try:
expired = await request_cancel_expired(
created_before=now - timedelta(minutes=max_minutes),
non_occupying_phases=tuple(NON_OCCUPYING_JOB_PHASES),
)
except Exception: # noqa: BLE001
logger.warning("[roundtable-jobs] hard-timeout scan failed", exc_info=True)
return 0
if expired:
logger.info(
"[roundtable-jobs] requested cancellation for %d overdue Step-2 room(s)",
expired,
)
return expired
async def _reap_cancel_requested(self, now: datetime) -> int:
"""收口持租 worker 崩溃后遗留的 ``cancel_requested`` 作业。
@@ -69,6 +69,26 @@ _DRAFT_WRITE_RETRIES = 3
ROUNDTABLE_BACKGROUND_REPORT_ENABLED = True
def _background_step2_room_expired(row: dict[str, Any] | None) -> bool:
"""Whether a background job exceeded the configurable Step-2 hard limit."""
if not row or str(row.get("phase") or "") in NON_OCCUPYING_JOB_PHASES:
return False
raw_created = row.get("created_at")
if isinstance(raw_created, datetime):
created_at = raw_created
elif isinstance(raw_created, str) and raw_created:
try:
created_at = datetime.fromisoformat(raw_created)
except ValueError:
return False
else:
return False
if created_at.tzinfo is None:
created_at = created_at.replace(tzinfo=UTC)
max_minutes = getattr(get_roundtable_room_limit_settings(), "max_hold_minutes", 30)
return datetime.now(UTC) >= created_at + timedelta(minutes=max_minutes)
def _background_step2_room_expired(row: dict[str, Any] | None) -> bool:
"""Whether a background job exceeded the configurable Step-2 hard limit."""
if not row or str(row.get("phase") or "") in NON_OCCUPYING_JOB_PHASES:
@@ -54,6 +54,8 @@ _AGENT_PURPOSE: dict[str, str] = {
# Workflow Studio(候选流程总控,和圆桌总控隔离)
"workflow-planner": "工作流编排",
"task-analysis-coordinator": "任务分析定时",
"chat-history-migrator": "其他内置功能",
"gauss-data-adapter": "其他内置功能",
}
_TARGET_TYPE = "agent"
@@ -0,0 +1,81 @@
"""Seed the built-in chat-history migration assistant.
The agent and its public skill are copied only when their target directories are
missing. Existing deployments are never overwritten, so administrator edits made
through agent/skill management survive restarts. This seeder must run before
``_sync_legacy_agents``.
"""
from __future__ import annotations
import logging
import shutil
from pathlib import Path
from deerflow.config.paths import get_paths
logger = logging.getLogger(__name__)
CHAT_HISTORY_MIGRATION_AGENT_ID = "chat-history-migrator"
CHAT_HISTORY_MIGRATION_SKILL_NAME = "chat-history-migration"
_ASSETS_DIR = Path(__file__).parent / "_chat_history_migration_seed_assets"
_AGENT_FILES: tuple[str, ...] = ("config.yaml", "SOUL.md")
def _skills_public_dir() -> Path:
from deerflow.skills.storage import get_or_new_skill_storage
return Path(get_or_new_skill_storage().get_skills_root_path()) / "public"
def _ensure_skill() -> bool:
try:
source = _ASSETS_DIR / "skills" / CHAT_HISTORY_MIGRATION_SKILL_NAME
target = _skills_public_dir() / CHAT_HISTORY_MIGRATION_SKILL_NAME
if target.exists():
return False
if not (source / "SKILL.md").is_file():
logger.error("Missing chat-history migration skill seed at %s", source)
return False
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(source, target)
logger.info("Seeded public skill '%s'", CHAT_HISTORY_MIGRATION_SKILL_NAME)
return True
except Exception:
logger.exception("Failed to seed chat-history migration skill")
return False
def _ensure_agent() -> bool:
try:
target = get_paths().agent_dir(CHAT_HISTORY_MIGRATION_AGENT_ID)
if target.exists():
return False
source = _ASSETS_DIR / CHAT_HISTORY_MIGRATION_AGENT_ID
if not source.is_dir():
logger.error("Missing chat-history migration agent seed at %s", source)
return False
target.mkdir(parents=True, exist_ok=True)
for filename in _AGENT_FILES:
asset = source / filename
if asset.is_file():
(target / filename).write_bytes(asset.read_bytes())
logger.info("Seeded built-in agent '%s'", CHAT_HISTORY_MIGRATION_AGENT_ID)
return True
except Exception:
logger.exception("Failed to seed chat-history migration agent")
return False
def ensure_chat_history_migration_agent() -> bool:
"""Install the chat-history migration skill and agent when missing."""
_ensure_skill()
return _ensure_agent()
__all__ = [
"CHAT_HISTORY_MIGRATION_AGENT_ID",
"CHAT_HISTORY_MIGRATION_SKILL_NAME",
"ensure_chat_history_migration_agent",
]
@@ -0,0 +1,51 @@
# 角色
你是 **聊天记录迁移助手**。你帮助管理员把一台 DeerFlow 服务器上的普通聊天记录迁到另一台服务器。源站的业务库可以是 openGauss/高斯,目标站可以是高斯、SQLite 或 MySQL。
你面对的操作者可能不熟悉数据库。先用一句通俗的话解释术语,再给可复制的命令。命令里的主机、端口、库名、路径一律用对方提供的值;没有提供时用占位符,禁止猜。
# 聊天记录实际放在哪
一次可见的聊天不是高斯库里的一张表。它有三层,缺任何一层都会表现为「列表空」或「有标题但点进去没有内容」:
1. **会话索引**:`threads_meta` 决定聊天列表里的标题、所属用户和时间。`runs`、`run_events`、`feedback` 是运行记录和评价。项目里的会话还依赖 `analysis_project_threads`。这一层可以在高斯库里。
2. **消息正文**:LangGraph 检查点。推荐放在 SQLite 文件,也可以是真正的 PostgreSQL。这里才是问答内容。
3. **线程文件**:`.deer-flow/users/<user_id>/threads/<thread_id>/user-data/` 下面的上传文件和生成结果。
不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上。高斯只承接业务索引。检查点文件原样进入目标的 SQLite 检查点库。
# 必须遵守的边界
1. **智能体沙箱不是源服务器,也不是目标服务器。** 除非用户明确说明当前环境就是那台机器,否则不要根据沙箱里的主机名、localhost 或某个 `.db` 文件推断现场。需要现场执行的命令要写明在哪一台机器、哪个目录运行。
2. **不要接收密码。** 不要让用户把高斯密码、token、私钥贴进对话。`gsql` / `mysql` 使用 `~/.pgpass` 或客户端自己的密码提示。脚本不会连接高斯,也没有密码参数。
3. **先盘点,再导出,再演练,最后才写入。** 导入命令默认是演练。用户明确说过目标库已有可恢复备份之前,不要在命令里加 `--apply`。
4. **不导出账号密码。** `users` 表和密码列不进入迁移包。目标站必须已经有对应账号。`user_id` 不同时只用 `--user-map 旧ID=新ID` 改归属,不改 `thread_id`。
5. **默认跳过已有会话。** 只有用户明确要求覆盖时才使用 `--on-conflict replace`。替换只影响迁移包里的那些 `thread_id`,不会清空目标库其他会话。已有项目的名称和配置保持目标站原样。
6. **系统线程默认不迁。** 调度器和内部工作线程会让列表变脏。用户点名要包含时才加 `--include-system`。
7. **源库只读。** 不在源库执行 INSERT、UPDATE、DELETE、DROP。导出 SQL 只有查询和 `\copy TO`。
# 工作方式
收到请求后先归入一种模式:
- **还不清楚数据在哪**:读取技能里的 `references/storage-map.md`,根据用户贴出的 `database.backend` 和 `checkpointer` 判断三层位置。一次只追问会改变命令的几项:业务库类型、检查点文件路径、`.deer-flow` 目录、要迁的用户或时间范围、目标库类型。
- **准备搬迁**:读取 `references/runbook.md`。按「停写或接受截止点 → 只读导出 → 打包 → 传到目标 → 演练 → 备份确认 → `--apply` → 验收」推进。每一步都写成功信号和失败时停在哪里。
- **迁完对不上**:先区分「列表没有这条会话」「有标题但正文是空的」「附件不见了」「进了别的账号」。分别对应索引、检查点、线程文件和 `user_id`。不要一次改三层。
脚本在技能目录 `scripts/migrate_chat_history.py`,只用 Python 标准库。用户要在源服务器上跑时,把脚本复制到输出目录并交付:
```bash
cp /mnt/skills/public/chat-history-migration/scripts/migrate_chat_history.py /mnt/user-data/outputs/migrate_chat_history.py
```
沙箱里没有 `bash` 时,用 `read_file` 读取脚本,再用 `write_file` 原样写到该路径。用户已经把 SQLite 或 JSONL 传到当前会话时,可以对 `/mnt/user-data/uploads/` 里的文件运行脚本,把迁移包写到 `/mnt/user-data/outputs/`。
# 交付格式
每次给操作步骤时包含这几项;没有的信息写「还不知道」:
1. **在哪台机器做**:源服务器、目标服务器,或当前会话里的上传文件。
2. **这一步会不会改数据**:只读、演练,或写入。写入前重复备份要求。
3. **命令**:可复制,占位符用 `<GAUSS_HOST>`、`<GAUSS_DB>`、`<CHECKPOINT_SQLITE>`、`<DEER_FLOW_ROOT>` 这种形式。
4. **成功信号**:应该看到的线程数、某个已知会话标题、检查点行数。
5. **停机点**:命令失败或数量对不上时不要继续下一步。
@@ -0,0 +1,35 @@
id: chat-history-migrator
name: 聊天记录迁移助手
description: 把一台 DeerFlow 服务器上的聊天记录迁到另一台。源站业务库可以是高斯/openGauss。会话索引、LangGraph 检查点和线程文件分开处理,不把 PostgreSQL 检查点迁移直接套到高斯上。
tool_groups:
- file:read
- file:write
- bash
skills:
- chat-history-migration
common_questions:
- title: 高斯库里的聊天记录怎么迁
prompt: >-
源服务器的业务库是高斯/openGauss,我要把聊天记录迁到另一台 DeerFlow。
请先用通俗的话说明聊天记录分哪几层存放,再给我一份只读盘点步骤。
不要让我把数据库密码贴进对话,也不要给出把 LangGraph PostgreSQL 检查点迁移直接执行到高斯库上的命令。
- title: 先做只读盘点
prompt: >-
请根据我提供的 database.backend、checkpointer 类型和目录,判断会话索引、消息正文和线程文件各在哪里。
缺哪一项就告诉我该在哪台机器上看哪个文件,不要猜路径,也不要开始拷贝。
- title: 导出迁移包
prompt: >-
请带我从源服务器导出聊天迁移包。业务库如果是高斯或 MySQL 协议的高斯,先生成只读导出 SQL;
检查点仍从 SQLite 文件提取。导出前说明会排除系统线程、不会导出用户密码。
- title: 导入前先演练
prompt: >-
迁移包已经放到目标服务器。请先做导入演练,告诉我会新增哪些会话、会跳过哪些已有会话、哪些文件会拷贝。
我明确确认已经有可恢复备份之前,不要加上 --apply。
- title: 导入后怎么验收
prompt: >-
导入已经执行。请给我验收步骤:会话列表能看到原标题,点进会话能看到原来的问答,附件还在原线程目录。
如果只有标题、打开是空的,请判断是检查点没迁过去,还是 user_id 对不上。
- title: 两边的用户 ID 不一样
prompt: >-
源服务器和目标服务器都有这些账号,但 user_id 不同,聊天记录迁过去后列表是空的。
请说明怎样用 --user-map 只改归属、不改 thread_id,以及哪些字段不会自动改写。
@@ -0,0 +1,45 @@
---
name: chat-history-migration
description: 把 DeerFlow 聊天记录从一台服务器迁到另一台。源业务库可以是 openGauss/高斯或 MySQL 协议高斯。分别导出会话索引、SQLite 检查点和线程文件,导入前默认只演练,不把 PostgreSQL 检查点迁移执行到高斯上。
---
# 聊天记录迁移
本技能的脚本和说明都在本地。先确认用户处于「还不知道数据在哪」「准备导出」「准备导入」还是「迁完对不上」,再只读需要的材料:
- 判断高斯库、检查点和线程目录各存什么:读取 `references/storage-map.md`。
- 已经知道路径,需要逐步导出、传输、演练和验收:读取 `references/runbook.md`。
- 两件事都还没确定时先读存储说明,不要把两份参考一次读完。
脚本路径:
```text
/mnt/skills/public/chat-history-migration/scripts/migrate_chat_history.py
```
它只用 Python 标准库,不连接高斯,也不接收密码。`import` 不加 `--apply` 时只演练,不创建、不修改目标文件。
## 现场和沙箱
`bash` 跑在 DeerFlow 沙箱里,通常不是要迁移的那台服务器。沙箱里的数据库文件和目录不能当成源站或目标站,除非用户明确说当前环境就是那台机器。
两种做法:
1. 把脚本复制到输出目录,让用户在源服务器和目标服务器上执行,再把 JSON 结果贴回来。
2. 用户已经把 SQLite、JSONL 或迁移包上传到当前会话时,直接对 `/mnt/user-data/uploads/` 运行脚本,产物写到 `/mnt/user-data/outputs/`。
交付脚本:
```bash
cp /mnt/skills/public/chat-history-migration/scripts/migrate_chat_history.py /mnt/user-data/outputs/migrate_chat_history.py
```
没有 `bash` 时,读取脚本原文,再用 `write_file` 原样写到上面的路径。
## 不允许的做法
- 不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上。
- 不要导出或覆盖 `users` 表,不要读写 `password_hash`、`login_password`。
- 不要在命令行参数或对话里出现数据库密码。
- 不要在用户确认目标库已有可恢复备份之前执行带 `--apply` 的导入。
- 不要因为列表能看到标题,就认为消息正文已经迁完。正文在检查点里。
@@ -0,0 +1,132 @@
# 迁移步骤
在源服务器导出,在目标服务器导入。迁移包里有对话正文,只通过可信通道拷贝。下面的命令在对应服务器的 shell 里执行,不在智能体沙箱里执行,除非用户确认当前环境就是那台机器。
开始前向用户要这些值,缺的用占位符,不要猜:
- 源、目标的 `database.backend`
- 高斯是 `opengauss` 还是 Dolphin(`gauss_mysql`)
- 检查点 SQLite 的实际路径
- `.deer-flow` 根目录
- 要迁的 `user_id` 或时间范围
- 目标账号 id 是否和源相同
建议先停止源 Gateway,或接受导出时刻之后的新消息不在包里。脚本不会锁高斯库,热导出不是一致快照。
## 1. 源站只读盘点
SQLite 索引或统一的 `deerflow.db`:
```bash
python3 migrate_chat_history.py inventory \
--app-sqlite <APP_SQLITE> \
--checkpoint-sqlite <CHECKPOINT_SQLITE> \
--threads-root <DEER_FLOW_ROOT>
```
只传存在的路径。业务库是高斯、检查点是 SQLite 时,不要传 `--app-sqlite`;索引改用下一步的 SQL 盘点。成功信号是 JSON 里的 `threads_meta`、检查点线程数和文件数。`system_threads` 是默认会排除的内部线程。
## 2. 从高斯导出索引
在源服务器上执行。密码放在 `~/.pgpass`(权限 600)或让客户端提示,不要写进命令,也不要贴进对话。
```bash
python3 migrate_chat_history.py emit-export-sql --dialect opengauss --out export_chat_index.sql
gsql -h <GAUSS_HOST> -p 26000 -d <GAUSS_DB> -U <GAUSS_USER> -f export_chat_index.sql
```
Dolphin / MySQL 协议把 `--dialect` 换成 `mysql`,然后用 `mysql -h <HOST> -P 3306 -u <USER> -p -D <DB> -N` 逐条执行,每条查询的输出存成对应的 `.jsonl`。`-p` 会提示输入密码。
这份 SQL 只有查询。它不包含检查点表。不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上。
## 3. 打迁移包
索引也在 SQLite 时:
```bash
python3 migrate_chat_history.py export \
--out <BUNDLE_DIR> \
--app-sqlite <APP_SQLITE> \
--checkpoint-sqlite <CHECKPOINT_SQLITE> \
--threads-root <DEER_FLOW_ROOT> \
--user-id <USER_ID>
```
索引来自刚才的 JSONL 时:
```bash
python3 migrate_chat_history.py pack-jsonl \
--out <BUNDLE_DIR> \
--threads-meta threads_meta.jsonl \
--runs runs.jsonl \
--run-events run_events.jsonl \
--feedback feedback.jsonl \
--project-threads analysis_project_threads.jsonl \
--projects analysis_projects.jsonl \
--checkpoint-sqlite <CHECKPOINT_SQLITE> \
--threads-root <DEER_FLOW_ROOT>
```
`<BUNDLE_DIR>` 必须是空目录,并且不能放在源数据库或 `.deer-flow` 目录里面。成功信号是目录里有 `manifest.json`。`excluded_system_thread_ids` 里的会话是故意不迁的。需要它们时两边都加 `--include-system`。
只迁某个时间之后的会话时加 `--since 2026-10-01T00:00:00`。
## 4. 目标站先演练
先复制备份:目标检查点 SQLite、目标业务库(高斯做库级备份)。没有备份就不加 `--apply`。
SQLite 业务库:
```bash
python3 migrate_chat_history.py import \
--bundle <BUNDLE_DIR> \
--app-sqlite <TARGET_APP_SQLITE> \
--checkpoint-sqlite <TARGET_CHECKPOINT_SQLITE> \
--threads-root <TARGET_DEER_FLOW_ROOT>
```
目标业务库是高斯时,索引和检查点分开:
```bash
python3 migrate_chat_history.py emit-gauss-sql --bundle <BUNDLE_DIR> --out load_chat_index.sql
python3 migrate_chat_history.py import \
--bundle <BUNDLE_DIR> \
--checkpoint-sqlite <TARGET_CHECKPOINT_SQLITE> \
--threads-root <TARGET_DEER_FLOW_ROOT>
```
演练的成功信号是 `"dry_run": true`,并且目标文件的修改时间没有变。看 `inserted`、`skipped_threads` 和 `files_copied`。已有的 `thread_id` 默认跳过;如果索引已在而检查点缺失,演练会显示还要补检查点。
账号 id 不同:
```bash
python3 migrate_chat_history.py import --bundle <BUNDLE_DIR> --user-map <OLD_USER_ID>=<NEW_USER_ID>
```
可以重复写多个 `--user-map`。
## 5. 确认备份后再写入
SQLite 目标把同一条 `import` 加上 `--apply`。高斯索引另执行:
```bash
gsql -h <GAUSS_HOST> -p 26000 -d <GAUSS_DB> -U <GAUSS_USER> -f load_chat_index.sql
```
`load_chat_index.sql` 默认 `INSERT ... WHERE NOT EXISTS`,不覆盖已有主键。用户明确要求覆盖这些会话时,生成 SQL 才加 `--on-conflict replace`。替换会删除目标上这些 `thread_id` 的索引行再写入,不会删除同 id 的已有项目定义,也不会动迁移包以外的会话。
目标 SQLite 文件必须已经存在(先启动一次目标 Gateway,让它建好表)。检查点文件不存在时,`--apply` 可以按迁移包里的表结构创建。
## 6. 验收
```bash
python3 migrate_chat_history.py verify \
--bundle <BUNDLE_DIR> \
--app-sqlite <TARGET_APP_SQLITE> \
--checkpoint-sqlite <TARGET_CHECKPOINT_SQLITE> \
--threads-root <TARGET_DEER_FLOW_ROOT>
```
退出码 0 表示会话、检查点行数和文件都在。高斯索引不经过这个 SQLite 校验;在目标高斯里对一个已知 `thread_id` 查询 `threads_meta`,再在网页上打开该会话,确认能看到迁前的问答和附件。
数量不对就停。跳过模式下重新执行 `--apply` 只会补缺失的检查点或文件,不会覆盖已有会话。要用备份恢复时,恢复第 4 步复制的目标库和检查点文件,并删掉本次新建的 `users/<user_id>/threads/<thread_id>` 目录。源服务器始终没有被修改。
@@ -0,0 +1,60 @@
# 聊天记录存在哪
DeerFlow 的一条聊天要三层都在,打开后才能看到原来的问答。
## 1. 会话索引(可以在高斯里)
`config.yaml` 的 `database.backend` 决定这一层:
| backend | 索引在哪 | 导出方式 |
| --- | --- | --- |
| `gauss` | `database.gauss_url` 指向的 openGauss/GaussDB | `emit-export-sql --dialect opengauss`,再用 `gsql` 执行 |
| `mysql`,且部署设置了 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql` | 高斯的 MySQL 协议(Dolphin) | `emit-export-sql --dialect mysql`,再用 `mysql` 客户端执行 |
| `sqlite` | `{sqlite_dir}/deerflow.db`,默认 `.deer-flow/data/deerflow.db` | 脚本直接读这个文件 |
| `postgres` | `database.postgres_url` | 先把索引导出成 JSONL,再 `pack-jsonl`。不要改检查点迁移方式 |
和聊天直接相关的表:
- `threads_meta`:聊天列表。`thread_id`、`user_id`、`display_name`、`metadata_json`、时间。
- `runs` / `run_events`:一轮运行的状态、首句预览和事件。
- `feedback`:回答评价。
- `analysis_projects` / `analysis_project_threads`:会话属于哪个分析项目。目标上已有的同 id 项目不会被覆盖;项目下的知识库文件不在这个迁移里。
`users` 表不迁。密码列不迁。目标服务器要先有账号,并且 `threads_meta.user_id` 等于目标账号的 id。两边 id 不同时,导入用 `--user-map 旧ID=新ID`。这个映射只改 `user_id` 以及 `users/<旧ID>/threads/...` 目录,不改 `thread_id`,也不改消息正文里的文字。
`metadata_json.system = true` 的是调度器或内部工作线程,默认排除。
## 2. 消息正文(不要放进高斯迁移)
问答正文在 LangGraph 检查点,不在 `threads_meta`。
`config.yaml` 的 `checkpointer`:
- `type: sqlite`:`connection_string` 指向的文件。业务库是 SQLite 且没有单独的 checkpointer 文件时,检查点表和 `threads_meta` 可能在同一个 `deerflow.db` 里。脚本遇到这种文件会把两层拆开。
- `type: postgres`:只适用于真正的 PostgreSQL。它的建表迁移不能对高斯执行。
- 业务库是 `gauss` 时,检查点应继续用 SQLite。如果现场误把检查点建在了高斯里,先停下来说明这个部署和当前脚本的支持范围不一致,不要自动改跑 PostgreSQL 迁移。
检查点表通常叫 `checkpoints`、`writes`、`checkpoint_blobs`、`checkpoint_writes`。迁移包里的 `checkpoints.sqlite` 只含选中会话的这些表。
只迁了 `threads_meta` 时,列表有标题,点进去没有历史消息。
## 3. 线程文件
默认在后端运行目录:
```text
<DEER_FLOW_ROOT>/users/<user_id>/threads/<thread_id>/user-data/
```
旧数据可能在 `<DEER_FLOW_ROOT>/threads/<thread_id>/`。`--threads-root` 传 `<DEER_FLOW_ROOT>`,也就是同时能看到 `users/` 或旧 `threads/` 的那一层,不要只传某一个会话目录。
这里有上传文件和生成的 HTML、Markdown。知识库原文、记忆库和分享链接不在这份迁移里。
## 迁完后的对照
| 现象 | 先看 |
| --- | --- |
| 目标列表完全没有这些会话 | 索引没进目标库,或 `user_id` 对不上 |
| 有标题,打开是空的 | 检查点没进目标 SQLite |
| 文字在,附件不见 | `--threads-root` 没指到同一个 `.deer-flow` 根 |
| 普通聊天在,项目里看不到 | 目标没有同 id 的 `analysis_projects`,或绑定表没导入 |
@@ -0,0 +1,81 @@
"""Seed the built-in GaussDB data-adaptation assistant.
The agent and its public skill are copied only when their target directories are
missing. Existing deployments are never overwritten, so administrator edits made
through agent/skill management survive restarts. This seeder must run before
``_sync_legacy_agents``.
"""
from __future__ import annotations
import logging
import shutil
from pathlib import Path
from deerflow.config.paths import get_paths
logger = logging.getLogger(__name__)
GAUSS_DATA_ADAPTATION_AGENT_ID = "gauss-data-adapter"
GAUSS_DATA_ADAPTATION_SKILL_NAME = "gauss-data-adaptation"
_ASSETS_DIR = Path(__file__).parent / "_gauss_data_adaptation_seed_assets"
_AGENT_FILES: tuple[str, ...] = ("config.yaml", "SOUL.md")
def _skills_public_dir() -> Path:
from deerflow.skills.storage import get_or_new_skill_storage
return Path(get_or_new_skill_storage().get_skills_root_path()) / "public"
def _ensure_skill() -> bool:
try:
source = _ASSETS_DIR / "skills" / GAUSS_DATA_ADAPTATION_SKILL_NAME
target = _skills_public_dir() / GAUSS_DATA_ADAPTATION_SKILL_NAME
if target.exists():
return False
if not (source / "SKILL.md").is_file():
logger.error("Missing Gauss data-adaptation skill seed at %s", source)
return False
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copytree(source, target)
logger.info("Seeded public skill '%s'", GAUSS_DATA_ADAPTATION_SKILL_NAME)
return True
except Exception:
logger.exception("Failed to seed Gauss data-adaptation skill")
return False
def _ensure_agent() -> bool:
try:
target = get_paths().agent_dir(GAUSS_DATA_ADAPTATION_AGENT_ID)
if target.exists():
return False
source = _ASSETS_DIR / GAUSS_DATA_ADAPTATION_AGENT_ID
if not source.is_dir():
logger.error("Missing Gauss data-adaptation agent seed at %s", source)
return False
target.mkdir(parents=True, exist_ok=True)
for filename in _AGENT_FILES:
asset = source / filename
if asset.is_file():
(target / filename).write_bytes(asset.read_bytes())
logger.info("Seeded built-in agent '%s'", GAUSS_DATA_ADAPTATION_AGENT_ID)
return True
except Exception:
logger.exception("Failed to seed Gauss data-adaptation agent")
return False
def ensure_gauss_data_adaptation_agent() -> bool:
"""Install the Gauss data-adaptation skill and agent when missing."""
_ensure_skill()
return _ensure_agent()
__all__ = [
"GAUSS_DATA_ADAPTATION_AGENT_ID",
"GAUSS_DATA_ADAPTATION_SKILL_NAME",
"ensure_gauss_data_adaptation_agent",
]
@@ -0,0 +1,21 @@
你是高斯数据适配助手。用户会把内网高斯适配时的报错或现象发给你,你告诉他改哪里。
不要先给安装教程。不要连接高斯,不要让用户把密码、完整连接串或令牌发到对话里。沙箱里的配置只代表跑脚本的那台机器;用户说的是另一台内网服务器时,不要用沙箱里的 `config.yaml` 给那台机器下改法。
先读技能里的 `references/adaptation-map.md`,只命中一条。
1. 报错原文已经能对应地图里唯一一条时,直接写改法。例如缺少 `asyncpg` / `opengauss_sqlalchemy`、错误 322、`in-memory fallback`、`current_schema()` 不一致、`VARCHAR(n)` 被拒绝、`standard_conforming_strings` 冲突。
2. 改法取决于「原生 openGauss」还是「Dolphin(MySQL 协议)」时,先确认模式,再给改法。能在出问题的那台 Gateway 上运行技能脚本时就运行;不能运行时,只追问 `database.backend` 和进程环境里有没有 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql`。
3. 对不上地图时,要改的地方写「先不要改」。指出下一段必须贴来的日志,不要编造 `ALTER`、不要把检查点迁到高斯、不要让内网机器访问公网安装依赖。
4. 用户只贴了 `Automatic ORM schema creation failed` 这一句时,改法还不能确定。向他要这句下面的异常类型和第一条 Gauss/MySQL/openGauss 报错,不要根据这句摘要改配置。
5. 通用问答和智能体问答共用同一套业务表。列表不见,改业务库;列表还在、正文没有,改检查点。不要为这两种聊天各做一套适配。
每次回答用这五段,没有内容就写「无」:
现象
判定模式
要改的地方
不要改的地方
改完如何确认
要改的地方必须是用户能执行的一项:改哪一个配置字段、哪一个环境变量、换哪一个已在现场的镜像或离线包、或明确「先不要改」。改完如何确认要写出重启后日志里应出现或应消失的那一句。
@@ -0,0 +1,19 @@
id: gauss-data-adapter
name: 高斯数据适配助手
description: 内网把业务库适配到高斯后出现报错或异常现象时,根据报错说明要改的配置、环境变量或部署方式。不连接数据库,不收集密码。
tool_groups:
- file:read
- bash
skills:
- gauss-data-adaptation
common_questions:
- title: 启动报缺少高斯驱动
prompt: Gateway 启动报错,说 database.backend 是 gauss,但没有安装 asyncpg 或 opengauss 的 SQLAlchemy 方言。这是内网,不能访问公网。请告诉我要改哪里。
- title: 进程起来了但建表失败
prompt: Gateway 进程已经起来,日志里有 Automatic ORM schema creation failed。请根据这个现象告诉我要改什么,不要先给一整套安装步骤。
- title: Dolphin 报错 322
prompt: 高斯走 MySQL 协议(Dolphin)。启动或建表时报错 322。请告诉我要改哪一项配置或环境变量。
- title: 列表还在,重启后正文没了
prompt: 通用问答和智能体的会话列表还在,但重启 Gateway 后对话正文没了。日志里有 in-memory fallback。请告诉我该改检查点还是高斯业务库。
- title: 时间差了 8 小时
prompt: 业务库改到高斯之后,页面上的时间比北京时间差了 8 小时。请告诉我要改配置的哪一处,不要让我在 SQL 里手工加减 8 小时。
@@ -0,0 +1,22 @@
---
name: gauss-data-adaptation
description: 根据内网高斯适配报错判断要改的配置或环境变量。区分原生 openGauss 与 Dolphin,不连接数据库,不读取密码。
---
# 高斯数据适配
用户贴来报错或现象后,先读 `references/adaptation-map.md`,只命中一条,按其中的改法回答。
报错已经能唯一对应时,直接给改法。只有改法取决于原生 openGauss 还是 Dolphin 时,才在**出问题的那台 Gateway** 上运行:
```bash
python /mnt/skills/public/gauss-data-adaptation/scripts/print_gauss_adaptation.py
```
沙箱里找不到该路径时:
```bash
python scripts/print_gauss_adaptation.py
```
脚本只读本机 `config.yaml` 和当前进程是否设置了 `DEERFLOW_DATABASE_PRODUCT`,并用 `find_spec` 查看驱动是否可导入。它不连接高斯,不读取 `$GAUSS_DATABASE_URL` 或 `$MYSQL_DATABASE_URL` 的值,输出里不保留账号和密码。沙箱不是出问题的那台机器时,把命令交给那台机器,不要把沙箱的判定当成那台机器的改法。
@@ -0,0 +1,153 @@
# 高斯适配报错对照
只命中一条。改完都要重启出问题的那个 Gateway 进程。环境变量写在启动该进程的服务配置里;在另一个 shell 里临时 export 不会影响已经启动的进程。
两种模式不能同时打开:
- 原生 openGauss:`database.backend: gauss`,`database.gauss_url: $GAUSS_DATABASE_URL`。不要设置 `DEERFLOW_DATABASE_PRODUCT`。
- Dolphin(高斯的 MySQL 协议):`database.backend: mysql`,`database.mysql_url: $MYSQL_DATABASE_URL`,并且进程环境为 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql`。不要把 `backend` 改成 `gauss`。
口令只放在服务器的环境变量或客户端口令文件里。下面的改法都不包含密码。
固定约束:不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上。
## 缺驱动
现象:启动日志含 `asyncpg is not installed` 或 `openGauss SQLAlchemy dialect is not installed`。
模式:原生 openGauss。不需要再确认。
要改的地方:不要在内网执行会访问公网的 `uv sync --extra gauss`。换成构建时带 `--build-arg UV_EXTRAS=gauss` 的镜像,或使用现场离线 wheelhouse 里的 `asyncpg` 和 `opengauss-sqlalchemy`。装进当前环境后重启。
不要改的地方:不要为了绕过缺包把 `backend` 改成 `postgres` 或 `mysql`。
改完如何确认:这两句 ImportError 消失,日志出现 `Persistence engine initialized` 且 `dialect=opengauss`。
Dolphin 缺的是 MySQL 驱动。日志含 `MySQL async driver is not installed` 时,用离线 wheelhouse 把 `asyncmy` 装进 `.deer-flow/python-vendor`,不要从公网安装,也不要安装 `opengauss-sqlalchemy`。
## 误用通用 PostgreSQL 方言
现象:TCP 能连上高斯,但方言初始化或建表失败;或者脚本显示 `backend` 不是 `gauss`,连接串却指向高斯端口。
模式:应为原生 openGauss。
要改的地方:`database.backend` 改为 `gauss`,`gauss_url` 使用 `opengauss://`、`opengauss+asyncpg://`、`gaussdb://` 或 PostgreSQL 风格 URL。启动时会归一化成 `opengauss+asyncpg`。取消 `DEERFLOW_DATABASE_PRODUCT`。
不要改的地方:不要手改 SQLAlchemy 的 PostgreSQL 方言,不要把这条 URL 同时填进 `checkpointer.connection_string`。
改完如何确认:`dialect=opengauss`。
## VARCHAR(n) 被拒绝
现象:建表报错指向 `VARCHAR(数字)`。
模式:原生 openGauss。不需要再确认。
要改的地方:确认进程确实以 `backend: gauss` 启动,从而装上把 `VARCHAR(n)` 编译成不带长度的 `VARCHAR` 的兼容层。驱动按上面的「缺驱动」处理。不要在库里把列改成某种固定长度来迁就这条 SQL。
不要改的地方:不要改成 Dolphin,也不要删业务列。
改完如何确认:同一条建表错误消失,`dialect=opengauss`。
## 错误 322,或 SHOW CREATE TABLE 解析失败
现象:日志含 `322`,或 MySQL 反射、`SHOW CREATE TABLE`、重复 `ALTER` 导致事务中止。
模式:Dolphin。不需要再确认。
要改的地方:保证**正在运行的进程**有 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql`,`database.backend` 保持 `mysql`。设置后重启该进程。不要再执行 MySQL 的 `SHOW CREATE TABLE` 反射、MySQL JSON 回填、MySQL 会话超时变量,也不要让应用执行 `CREATE DATABASE`。
不要改的地方:不要把 `backend` 改成 `gauss` 来消掉 322。库、用户和 schema 仍由离线部署层创建。
改完如何确认:日志出现 `GaussDB MySQL compatibility enabled` 和 `Skipping MySQL reflection and JSON backfill for openGauss Dolphin`,不再出现 322。
## init_command 冲突
现象:日志含 `gauss_mysql requires init_command` 或 `standard_conforming_strings`。
模式:Dolphin。
要改的地方:从 `mysql_url` 里去掉冲突的 `init_command`。兼容层会在连接时设置 `SET standard_conforming_strings TO off`。
不要改的地方:不要在 URL 里写另一条 `init_command`,不要把口令贴进对话。
改完如何确认:这句 ValueError 消失,并出现 `GaussDB MySQL compatibility enabled`。
## current_schema() 对不上
现象:日志含 `URL schema` 与 `current_schema()` 不一致,或 `current_schema() is empty`。
模式:Dolphin 的启动补列。
要改的地方:把 `mysql_url` 里的库名改成 `SELECT current_schema();` 的结果,或让部署层把该登录用户的当前 schema 改成 URL 里的库名。两边必须是同一个名字。
不要改的地方:不要用 `SHOW CREATE TABLE` 补列,不要由应用 `CREATE DATABASE`。
改完如何确认:这句 schema integrity check 消失。若随后出现 `openGauss schema is incomplete`,按下一条处理,不要提前改别的配置。
## 缺列但启动拒绝补列
现象:日志含 `openGauss schema is incomplete`,以及 `NOT NULL column has no server_default`、主键、identity、computed 或约束需要 explicit migration。
模式:Dolphin。
要改的地方:先不要改。把日志里点名的 `schema.表.列` 原样留下。这种列不能靠启动自动补。需要一份针对该列的显式迁移,而不是在对话里手写 `ALTER TABLE`。
不要改的地方:不要删表、不要把列改成可空来让启动继续。
改完如何确认:同一句 `schema is incomplete` 消失。只为日志点名的那一列做迁移。
## 只看到建表失败摘要
现象:只有 `Automatic ORM schema creation failed; continuing Gateway startup`。
模式:还不能判定。
要改的地方:先不要改。把这句下面的异常类型和第一条数据库报错贴出来,再按本表重新命中。
不要改的地方:不要因为进程还活着就认为适配已经成功。缺表的接口仍会失败。
改完如何确认:重启后这句消失;若仍在,新贴出的第一条数据库报错必须变了,才能进入下一条。
## 列表还在,重启后正文没了
现象:通用问答或智能体的会话列表还在,重启后正文没有;日志含 `LangGraph checkpointer initialization failed` 和 `in-memory fallback`。
模式:与原生或 Dolphin 无关。列表来自业务库,正文来自检查点。
要改的地方:修复日志里检查点初始化的那条异常。检查点使用现有 SQLite 文件,或一台真正的 PostgreSQL。不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上。若 `checkpointer.type` 已是 `postgres` 且 `use_postgres: false`,实际文件是 `sqlite_connection_string`,改这个路径的权限或位置,不要改高斯 URL。
不要改的地方:不要重导 `threads_meta`,不要改 `assistant_id`。两种聊天的正文都在同一套检查点里。
改完如何确认:`in-memory fallback` 消失。重启后再打开刚才那条通用问答和那条智能体问答,正文仍在。
## 时间差 8 小时
现象:库里或页面上的时间比北京时间差 8 小时,业务库刚刚改到高斯。
先确认模式。
- 原生 openGauss 却把 `backend` 配成 `mysql` 或 `postgres`:改回上一节的原生配置。`opengauss` 与 `postgresql` 走带时区的时间类型,不在写入时改成北京墙钟。
- Dolphin 却没设 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql`,或误设成 `backend: gauss`:改回 Dolphin 配置。Dolphin 按 MySQL 方言把 UTC 存成北京墙钟。
- 模式已经配对,只是客户端用另一套时区显示:先不要改 DeerFlow 配置,也不要在 SQL 里加减 8 小时。
改完如何确认:新写入的一行与页面显示一致。不要批量改历史时间列。
## JSON 读失败
现象:日志含 `GaussDB MySQL JSON value is neither valid JSON`,或普通 JSON 解析失败。
- 这句完整报错只出现在 Dolphin 兼容层已经装上、但库里的值不是严格 JSON、也不是已支持的旧字面量时:先不要改配置。把报错里的预览留给库侧,不要改用另一种解码把坏值读成对象。
- 同样的数据在未设置 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql` 时解析失败:按 Dolphin 配置补上该环境变量并重启。普通 MySQL 必须继续拒绝坏 JSON,不要为了读旧数据去设置这个变量。
改完如何确认:Dolphin 应出现 `GaussDB MySQL compatibility enabled`。坏值那一行仍失败时,失败原因应仍是这句 JSON 报错,而不是驱动或方言报错。
## 两种模式同时打开
现象:`backend: gauss`,同时进程环境是 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql`。
要改的地方:只留一种。原生就删掉该环境变量;Dolphin 就把 `backend` 改回 `mysql` 并保留该环境变量。
不要改的地方:不要两套 URL 都填上指望自动选择。
改完如何确认:脚本的「判定模式」只剩一种,启动日志里的 `dialect` 与之相符:原生是 `opengauss`,Dolphin 是 `mysql`。
@@ -0,0 +1,226 @@
"""Classify the local Gauss adaptation mode without connecting or reading secrets."""
from __future__ import annotations
import os
from importlib.util import find_spec
from pathlib import Path
PROHIBITION = "不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上"
DOLPHIN_PRODUCT = "gauss_mysql"
GAUSS_URL_PREFIXES = (
"opengauss+asyncpg://",
"opengauss://",
"gaussdb://",
"postgres://",
"postgresql://",
"postgresql+asyncpg://",
)
def config_candidates() -> list[Path]:
found: list[Path] = []
for key in ("DEER_FLOW_CONFIG_PATH", "DEER_FLOW_CONFIG", "DEERFLOW_CONFIG"):
value = os.environ.get(key, "").strip()
if value:
found.append(Path(value))
found.extend(
[
Path("/app/backend/config.yaml"),
Path.cwd() / "config.yaml",
Path.cwd().parent / "config.yaml",
]
)
return found
def parse_sections(text: str, names: set[str]) -> dict[str, dict[str, str]]:
"""Read one level of scalar keys from the named top-level YAML blocks."""
found = {name: {} for name in names}
current = ""
for raw in text.splitlines():
line = raw.split("#", 1)[0].rstrip()
if not line.strip():
continue
if not line.startswith((" ", "\t")):
header = line.strip()
current = header[:-1] if header.endswith(":") else ""
if current not in names:
current = ""
continue
if not current or ":" not in line:
continue
key, _, value = line.strip().partition(":")
value = value.strip().strip("\"'")
if value:
found[current][key.strip()] = value
return found
def describe_value(value: str) -> str:
"""Describe a config value without usernames, passwords, or env-var contents."""
text = value.strip().strip("\"'")
if not text:
return "未设置"
if text.startswith("$"):
return f"环境变量 {text[1:]}(不读取变量值)"
if "://" not in text:
return text
scheme, _, rest = text.partition("://")
target = rest.split("@")[-1].split("?")[0]
return f"{scheme}://{target}(已去掉账号和密码)"
def driver_status() -> dict[str, bool]:
names = ("asyncpg", "opengauss_sqlalchemy", "asyncmy", "aiomysql")
return {name: find_spec(name) is not None for name in names}
def _enabled(value: str) -> bool:
return value.strip().lower() not in {"", "false", "0", "no", "off"}
def classify(
database: dict[str, str],
checkpointer: dict[str, str],
product: str,
drivers: dict[str, bool],
) -> list[str]:
"""Return the mode and the concrete config change, if one is already known."""
backend = database.get("backend", "").strip().lower()
product_name = product.strip().lower()
checkpoint_type = checkpointer.get("type", "").strip().lower()
use_postgres = checkpointer.get("use_postgres", "")
lines = [
f"database.backend={backend or '未解析'}",
f"DEERFLOW_DATABASE_PRODUCT={product_name or '未设置'}",
f"checkpointer.type={checkpoint_type or '未解析'}",
]
if "gauss_url" in database:
lines.append(f"database.gauss_url={describe_value(database['gauss_url'])}")
if "mysql_url" in database:
lines.append(f"database.mysql_url={describe_value(database['mysql_url'])}")
if "connection_string" in checkpointer:
lines.append(
"checkpointer.connection_string="
+ describe_value(checkpointer["connection_string"])
)
if use_postgres:
lines.append(f"checkpointer.use_postgres={use_postgres}")
for name in ("asyncpg", "opengauss_sqlalchemy", "asyncmy", "aiomysql"):
lines.append(f"{name}={'可导入' if drivers.get(name) else '不可导入'}")
changes: list[str] = []
constraints: list[str] = []
if backend == "gauss" and product_name == DOLPHIN_PRODUCT:
mode = "冲突:原生 openGauss 与 Dolphin 同时打开"
changes.append(
"只保留一种。原生就取消 DEERFLOW_DATABASE_PRODUCT;"
"Dolphin 就把 database.backend 改回 mysql,并保留该环境变量。"
)
elif backend == "gauss":
mode = "原生 openGauss"
url = database.get("gauss_url", "")
if not url:
changes.append("补上 database.gauss_url: $GAUSS_DATABASE_URL。口令只放在该环境变量里。")
elif not url.startswith("$") and not url.startswith(GAUSS_URL_PREFIXES):
changes.append(
"把 gauss_url 改成 opengauss://、opengauss+asyncpg://、gaussdb:// 或 PostgreSQL 风格 URL。"
)
elif not url.startswith("$") and not url.startswith("opengauss+asyncpg://"):
constraints.append("这份 gauss_url 启动时会归一化为 opengauss+asyncpg,不会走通用 PostgreSQL 方言。")
if not drivers.get("asyncpg") or not drivers.get("opengauss_sqlalchemy"):
changes.append(
"当前进程缺少 asyncpg 或 opengauss_sqlalchemy。"
"不要在内网访问公网安装。改用带 UV_EXTRAS=gauss 的镜像,或现场离线 wheelhouse。"
)
constraints.append("不要设置 DEERFLOW_DATABASE_PRODUCT。")
elif backend == "mysql" and product_name == DOLPHIN_PRODUCT:
mode = "Dolphin(高斯 MySQL 协议)"
if not database.get("mysql_url"):
changes.append("补上 database.mysql_url: $MYSQL_DATABASE_URL。口令只放在该环境变量里。")
if not drivers.get("asyncmy") and not drivers.get("aiomysql"):
changes.append(
"当前进程缺少 asyncmy 和 aiomysql。用离线 wheelhouse 把 asyncmy 装进 "
".deer-flow/python-vendor,不要从公网安装。"
)
constraints.append(
"不要执行 SHOW CREATE TABLE、MySQL JSON 回填或应用侧 CREATE DATABASE。"
"URL 里的库名必须等于 SELECT current_schema()。"
)
elif backend == "mysql":
mode = "普通 MySQL"
constraints.append(
"现在不会安装 Dolphin 补丁。只有报错已经表现为 322、current_schema 或 "
"SHOW CREATE TABLE 解析失败时,才给运行中的进程加上 "
"DEERFLOW_DATABASE_PRODUCT=gauss_mysql 并重启。普通 MySQL 不要加。"
)
elif backend == "postgres":
mode = "PostgreSQL"
changes.append("这不是高斯适配路径。连高斯业务库时不要使用 backend: postgres。")
elif backend in {"sqlite", "memory"}:
mode = backend
constraints.append("当前业务库配置不是高斯。这只代表运行脚本的机器。")
else:
mode = "未识别"
changes.append("先不要改。确认 database.backend 是 gauss、mysql、postgres、sqlite 还是 memory。")
postgres_checkpoint = checkpoint_type == "postgres" and _enabled(use_postgres or "true")
if backend in {"gauss", "mysql"} and postgres_checkpoint and product_name == DOLPHIN_PRODUCT:
changes.append(
"先确认 checkpointer.connection_string 是不是高斯。"
+ PROHIBITION
+ " 是高斯就把 checkpointer.type 改为 sqlite,并沿用现有 SQLite 文件;是独立的 PostgreSQL 就不要改。"
)
elif backend == "gauss" and postgres_checkpoint:
changes.append(
"先确认 checkpointer.connection_string 是不是高斯。"
+ PROHIBITION
+ " 是高斯就把 checkpointer.type 改为 sqlite,并沿用现有 SQLite 文件;是独立的 PostgreSQL 就不要改。"
)
elif checkpoint_type == "memory" and backend in {"gauss", "mysql"}:
changes.append("把 checkpointer.type 从 memory 改为 sqlite,否则重启后问答正文不会保留。")
lines.append(f"判定模式:{mode}")
if constraints:
lines.append("先不要改:")
lines.extend(f"- {item}" for item in constraints)
lines.append("要改的地方:")
if changes:
lines.extend(f"- {item}" for item in changes)
else:
lines.append("- 配置层面没有必须修改的项。把报错原文发来后再决定。")
lines.append(f"固定约束:{PROHIBITION}")
return lines
def main() -> None:
print("这个结果只代表运行脚本的机器,不连接高斯,也不读取连接串环境变量的值。")
chosen = next((item for item in config_candidates() if item.is_file()), None)
if chosen is None:
database: dict[str, str] = {}
checkpointer: dict[str, str] = {}
print("配置:未找到 config.yaml。")
else:
print(f"配置文件:{chosen}")
try:
parsed = parse_sections(
chosen.read_text(encoding="utf-8"),
{"database", "checkpointer"},
)
except OSError as exc:
print(f"配置:读取失败:{exc}")
parsed = {"database": {}, "checkpointer": {}}
database = parsed["database"]
checkpointer = parsed["checkpointer"]
for line in classify(
database,
checkpointer,
os.environ.get("DEERFLOW_DATABASE_PRODUCT", ""),
driver_status(),
):
print(line)
if __name__ == "__main__":
main()
@@ -21,7 +21,7 @@
## 页面生成流程
1. 先读取 `page-report-generation` 技能,并将材料归为:事实数据、可推导结论、待补充项、图表数据、长文本、表格行、行动/风险项。优先保留用户原文的关键判断和数据口径。
2. 默认从零直接创建 `/mnt/user-data/outputs/<主题>.html`,第一项文件操作就是写最终路径。只有用户明确要求使用模板或点名模板时,才读取 `page-template-library` 并调用 `page_report_template(action="prepare")`;主题优先使用界面传入的 `presentation_theme`,其次遵从用户明示。
2. 默认从零直接创建 `/mnt/user-data/outputs/<主题>.html`,第一项文件操作就是写最终路径。只有用户明确要求使用模板或点名模板时,才读取 `page-template-library` 并调用 `page_report_template(action="prepare")`;用户明示的页面主题优先,否则只跟随界面传入的 `presentation_theme` 明暗模式。
3. 页面文字通常要多于图表:至少包含结论摘要、关键发现、事项/风险说明、数据口径或来源、行动建议中的多个展示样式。默认生成一个可纵向滚动的响应式页面;只有用户明确要求多页或给出页数时才生成多页。
4. 使用模板时,工具返回的 outputs 工作副本是唯一成品路径。保留已有图表渲染和本地运行时契约,用真实数据替换占位内容,并删除无数据页面和面板;绝不修改模板源文件。
5. 所有可见正文、表头、标签、说明文字最小字号均为 16px;长文本面板要增高、提高行高,不能把段落挤成一团。
@@ -29,7 +29,7 @@
## 设计要求
- 跟随 `presentation_theme`:`dark` 选深蓝骨架,`light` 选浅色骨架;不要把产品现有聊天页面风格照搬进汇报页面。
- 用户未指定页面主题时跟随 `presentation_theme`:`dark` 选深色骨架,`light` 选浅色骨架;不要复制系统的红、蓝、绿等主题主色,页面自己的强调色根据内容、信息层级和可读性独立设计,也不要把产品现有聊天页面风格照搬进汇报页面。
- 深色使用深蓝科技感,选中状态可有细腻的扫描光、描边或流光;浅色使用清晰留白与高对比文字。
- 动效服务于阅读:数字滚动、图表入场、进度条、轻量背景粒子即可;支持 `prefers-reduced-motion`,不使用刺眼或持续大幅运动。
- 图表必须与材料一致。数据不够时宁可删掉图表或明确标为示意,不伪造事实;避免为了“丰富”而堆满图。
@@ -15,7 +15,7 @@ allowed-tools:
## 使用流程
1. 先盘点材料和输出约束,确定:
- `theme`:优先使用界面传入的 `presentation_theme`,否则遵从用户明示;仍未指定时使用 `auto`。
- `theme`:用户明示的页面主题优先;否则使用界面传入的 `presentation_theme`。`auto` 会由工具按当前界面明暗模式解析,界面信息缺失时才回退浅色。
- `content_density`:`text-heavy`、`balanced` 或 `data-heavy`。
- `required_features`:如 `flowchart`、`chart`、`map`、`table`、`text-panels`;源 Markdown 中的 Mermaid 必须记为 `flowchart`。
- `preferred_page_count`:默认 `1`;只有用户明确要求多页或指定页数时才大于 `1`。
@@ -318,6 +318,20 @@ class AdminLeaderboardResponse(BaseModel):
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
class AdminUserAnalyticsResponse(BaseModel):
range_days: int
since: str
until: str
user: AdminLeaderboardUser
overview: AdminLeaderboardOverview
skills: list[AdminLeaderboardSkill] = Field(default_factory=list)
tools: list[AdminLeaderboardSkill] = Field(default_factory=list)
models: list[AdminLeaderboardDimension] = Field(default_factory=list)
agents: list[AdminLeaderboardDimension] = Field(default_factory=list)
run_statuses: list[AdminLeaderboardDimension] = Field(default_factory=list)
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
class AdminLeaderboardReportResponse(BaseModel):
report: str
generated_at: str
@@ -2224,6 +2238,67 @@ async def stream_user_leaderboard_report(
)
@router.get("/{user_id}/analytics", response_model=AdminUserAnalyticsResponse)
async def get_user_analytics(
user_id: str,
request: Request,
days: int = Query(default=30, ge=1, le=ADMIN_QUERY_WINDOW_DAYS),
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
limit: int = Query(default=20, ge=5, le=100),
) -> AdminUserAnalyticsResponse:
"""Return one user's admin analytics profile."""
await _require_admin(request)
session_factory = get_session_factory()
if session_factory is None:
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
settings = load_system_settings().leaderboard
common = {
"include_scheduled": settings.include_scheduled,
"include_admins": settings.include_admins,
"include_failed": settings.include_failed,
}
tool_common = {
"include_scheduled": settings.include_scheduled,
"include_admins": settings.include_admins,
}
since_dt, until_dt, range_days = resolve_admin_query_window(
days=days,
since=since,
until=until,
)
async with session_factory() as session:
await _set_analytics_read_timeout(session)
users = await _query_user_leaderboard(session, since_dt, until_dt, user_id=user_id, **common)
if not users:
raise HTTPException(status_code=404, detail="User not found")
overview = await _query_overview(session, since_dt, until_dt, user_id=user_id, **common)
skills = await _query_call_rankings(session, since_dt, until_dt, user_id=user_id, field="skill", limit=limit, **tool_common)
tools = await _query_call_rankings(session, since_dt, until_dt, user_id=user_id, field="tool", limit=limit, **tool_common)
models = await _query_model_rankings(session, since_dt, until_dt, user_id=user_id, limit=limit, **common)
if not models:
models = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="model", limit=limit, **common)
agents = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="agent", limit=limit, **common)
run_statuses = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="status", limit=limit, **common)
trends = await _query_trends(session, since_dt, until_dt, user_id=user_id, **common)
return AdminUserAnalyticsResponse(
range_days=range_days,
since=_iso(since_dt),
until=_iso(until_dt),
user=users[0],
overview=overview,
skills=skills,
tools=tools,
models=models,
agents=agents,
run_statuses=run_statuses,
trends=trends,
)
# ── Q&A audit list ─────────────────────────────────────────────────────────
@@ -0,0 +1,245 @@
"""Admin-only metadata migration for agent/skill folders copied in place."""
from __future__ import annotations
import asyncio
import logging
from pathlib import Path
from fastapi import APIRouter, Depends, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.agent_skill_migration import (
MigrationApplyResult,
MigrationManifest,
MigrationPreflight,
apply_migration_manifest,
build_agent_migration_manifest,
build_skill_migration_manifest,
preflight_manifest,
prepare_manifest_for_apply,
)
from app.gateway.deps import (
get_agent_store,
get_local_provider,
get_optional_user_from_request,
get_skill_store,
get_tag_store,
)
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
from deerflow.config.extensions_config import ExtensionsConfig
from deerflow.config.paths import get_paths
from deerflow.config.system_settings import load_system_settings
from deerflow.persistence.agents import AgentStore
from deerflow.persistence.skills import SkillStore
from deerflow.persistence.tags import TagStore
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.usage import all_entries as get_skill_usage_entries
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/admin/agent-skill-migrations", tags=["agent-skill-migrations"])
class MigrationExportRequest(BaseModel):
agent_ids: list[str] = Field(min_length=1, max_length=1000)
class SkillMigrationExportRequest(BaseModel):
skill_names: list[str] = Field(min_length=1, max_length=5000)
class MigrationApplyRequest(BaseModel):
manifest: MigrationManifest
use_current_admin_for_unresolved_owners: bool = False
skip_invalid_skills: bool = False
class MigrationRootsResponse(BaseModel):
agents_root: str
skills_root: str
extensions_config_path: str | None
async def _require_admin(request: Request):
user = await get_optional_user_from_request(request)
if user is not None and getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="智能体与技能迁移仅限管理员")
return user
def _require_migration_enabled() -> None:
if not load_system_settings().agent_skill_migration.enabled:
raise HTTPException(status_code=403, detail="智能体与技能迁移功能未开启")
async def _users() -> tuple[dict[str, str], dict[str, str]]:
"""Return ``(id -> email, normalized email -> id)`` without auth secrets."""
try:
users = await get_local_provider().list_users()
except Exception: # noqa: BLE001 - metadata migration degrades to manual owner fallback
logger.warning("Could not load users for migration owner mapping", exc_info=True)
return {}, {}
by_id = {str(user.id): str(user.email) for user in users}
by_email = {str(user.email).strip().casefold(): str(user.id) for user in users}
return by_id, by_email
def _roots() -> tuple[Path, Path]:
storage = get_or_new_skill_storage()
return get_paths().agents_dir.resolve(), Path(storage.get_skills_root_path()).resolve()
@router.get("/roots", response_model=MigrationRootsResponse)
async def migration_roots(request: Request) -> MigrationRootsResponse:
await _require_admin(request)
_require_migration_enabled()
agents_root, skills_root = _roots()
extensions_path = ExtensionsConfig.resolve_config_path()
return MigrationRootsResponse(
agents_root=str(agents_root),
skills_root=str(skills_root),
extensions_config_path=str(extensions_path.resolve()) if extensions_path else None,
)
@router.post("/export", response_model=MigrationManifest)
async def export_migration_manifest(
body: MigrationExportRequest,
request: Request,
agent_store: AgentStore = Depends(get_agent_store),
tag_store: TagStore = Depends(get_tag_store),
) -> MigrationManifest:
await _require_admin(request)
_require_migration_enabled()
agents_root, _ = _roots()
users_by_id, _ = await _users()
try:
return await build_agent_migration_manifest(
body.agent_ids,
agent_store=agent_store,
tag_store=tag_store,
agents_root=agents_root,
users_by_id=users_by_id,
)
except (FileNotFoundError, ValueError) as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
@router.post("/export-skills", response_model=MigrationManifest)
async def export_skill_migration_manifest(
body: SkillMigrationExportRequest,
request: Request,
skill_store: SkillStore = Depends(get_skill_store),
tag_store: TagStore = Depends(get_tag_store),
) -> MigrationManifest:
await _require_admin(request)
_require_migration_enabled()
_, skills_root = _roots()
storage = get_or_new_skill_storage()
users_by_id, _ = await _users()
discovered_skills = await asyncio.to_thread(storage.load_skills, enabled_only=False)
try:
return await build_skill_migration_manifest(
body.skill_names,
skill_store=skill_store,
tag_store=tag_store,
skills_root=skills_root,
discovered_skills=discovered_skills,
users_by_id=users_by_id,
usage_entries=get_skill_usage_entries(),
)
except (FileNotFoundError, ValueError) as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
@router.post("/preflight", response_model=MigrationPreflight)
async def preflight_migration_manifest(
manifest: MigrationManifest,
request: Request,
) -> MigrationPreflight:
await _require_admin(request)
_require_migration_enabled()
agents_root, skills_root = _roots()
_, users_by_email = await _users()
return await asyncio.to_thread(
preflight_manifest,
manifest,
agents_root=agents_root,
skills_root=skills_root,
users_by_email=users_by_email,
)
@router.post("/apply", response_model=MigrationApplyResult)
async def apply_migration(
body: MigrationApplyRequest,
request: Request,
agent_store: AgentStore = Depends(get_agent_store),
skill_store: SkillStore = Depends(get_skill_store),
tag_store: TagStore = Depends(get_tag_store),
) -> MigrationApplyResult:
user = await _require_admin(request)
_require_migration_enabled()
agents_root, skills_root = _roots()
_, users_by_email = await _users()
# Re-scan at apply time. The browser's earlier preflight is informative,
# never an authorization or integrity token.
preflight = await asyncio.to_thread(
preflight_manifest,
body.manifest,
agents_root=agents_root,
skills_root=skills_root,
users_by_email=users_by_email,
)
try:
effective_manifest, effective_preflight, _ = prepare_manifest_for_apply(
body.manifest,
preflight,
skip_invalid_skills=body.skip_invalid_skills,
)
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
if body.manifest.skills and not effective_manifest.skills:
raise HTTPException(status_code=409, detail="所有技能均未通过预检,没有可导入的技能")
resource_failures = [item.model_dump(mode="json") for item in [*effective_preflight.agents, *effective_preflight.skills, *effective_preflight.support_directories] if item.status != "matched"]
unresolved = [item for item in effective_preflight.owner_mappings if item.status == "unresolved"]
if resource_failures or (unresolved and not body.use_current_admin_for_unresolved_owners):
raise HTTPException(
status_code=409,
detail={
"message": "迁移预检未通过,未写入任何元数据",
"preflight": effective_preflight.model_dump(mode="json"),
},
)
extensions_path = ExtensionsConfig.resolve_config_path() if effective_manifest.skills else None
if effective_manifest.skills and extensions_path is None:
raise HTTPException(status_code=503, detail="目标运行环境未找到 extensions_config.json")
fallback_owner_user_id = None
if unresolved and body.use_current_admin_for_unresolved_owners:
fallback_owner_user_id = str(user.id) if user is not None else get_effective_user_id()
try:
result = await apply_migration_manifest(
body.manifest,
preflight=preflight,
agent_store=agent_store,
skill_store=skill_store,
tag_store=tag_store,
agents_root=agents_root,
users_by_email=users_by_email,
fallback_owner_user_id=fallback_owner_user_id,
extensions_config_path=extensions_path,
skip_invalid_skills=body.skip_invalid_skills,
)
if result.skills_applied:
await refresh_skills_system_prompt_cache_async()
storage = get_or_new_skill_storage()
await asyncio.to_thread(refresh_skill_sandbox_view, storage=storage, force=True)
return result
except ValueError as exc:
raise HTTPException(status_code=409, detail=str(exc)) from exc
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,700 @@
"""Account-scoped chat history transfer for the conversation list page.
Exports and imports ordinary / agent chats as a JSON bundle that includes
the latest checkpoint (messages) and the thread ``user-data`` files
(uploads + outputs). One oversized or broken thread is skipped so the
rest of the batch can still succeed.
"""
from __future__ import annotations
import asyncio
import base64
import json
import logging
import re
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import FileResponse
from langgraph.checkpoint.base import empty_checkpoint
from pydantic import BaseModel, Field
from app.gateway.authz import require_permission
from app.gateway.deps import get_checkpointer, get_thread_store
from app.gateway.routers.threads import _strip_reserved_metadata
from app.gateway.utils import sanitize_log_param
from deerflow.config.paths import get_paths
from deerflow.config.system_settings import load_system_settings
from deerflow.persistence.thread_meta.base import is_excluded_system_thread
from deerflow.runtime import serialize_channel_values
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.utils.time import coerce_iso, now_iso
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/chat-history", tags=["chat-history-transfer"])
FORMAT = "deerflow-chat-history-ui"
FORMAT_VERSION = 1
BATCH_LIMIT = 1
MAX_THREAD_BYTES = 50 * 1024 * 1024
MAX_FILES_PER_THREAD = 200
SAFE_REL_PATH = re.compile(r"^(?:[A-Za-z0-9._-]+/)*[A-Za-z0-9._-]+$")
_FILE_JSON_OVERHEAD = 64
_DROP_META_KEYS = frozenset(
{
"owner_id",
"user_id",
"analysis_project_id",
"analysis_project_mode",
"analysis_project_name",
"shared_from_code",
"shared_origin",
"shared_by",
"taskId",
"actionId",
}
)
class SkippedThread(BaseModel):
thread_id: str
title: str = ""
reason: str
class TransferFile(BaseModel):
path: str
content_b64: str
size: int = 0
class TransferThread(BaseModel):
thread_id: str
title: str = ""
assistant_id: str | None = None
metadata: dict[str, Any] = Field(default_factory=dict)
created_at: str = ""
updated_at: str = ""
values: dict[str, Any] = Field(default_factory=dict)
files: list[TransferFile] = Field(default_factory=list)
skipped_files: list[str] = Field(default_factory=list)
class ExportRequest(BaseModel):
thread_ids: list[str] = Field(..., min_length=1, max_length=BATCH_LIMIT)
class ExportResponse(BaseModel):
format: str = FORMAT
version: int = FORMAT_VERSION
threads: list[TransferThread] = Field(default_factory=list)
skipped: list[SkippedThread] = Field(default_factory=list)
class ImportRequest(BaseModel):
threads: list[TransferThread] = Field(..., min_length=1, max_length=BATCH_LIMIT)
class ImportResponse(BaseModel):
imported: int = 0
skipped: list[SkippedThread] = Field(default_factory=list)
class ExportAllStartResponse(BaseModel):
job_id: str
class ExportAllStatusResponse(BaseModel):
job_id: str
status: str
total: int = 0
processed: int = 0
succeeded: int = 0
skipped: list[SkippedThread] = Field(default_factory=list)
skipped_files: int = 0
error: str | None = None
@dataclass
class ExportAllJob:
job_id: str
user_id: str
status: str = "running"
total: int = 0
processed: int = 0
succeeded: int = 0
skipped: list[SkippedThread] = field(default_factory=list)
skipped_files: int = 0
error: str | None = None
path: Path | None = None
task: asyncio.Task[None] | None = None
_EXPORT_JOBS: dict[str, ExportAllJob] = {}
_SAFE_USER_DIR = re.compile(r"[^A-Za-z0-9._-]+")
def require_transfer_enabled() -> None:
if not load_system_settings().agent_skill_migration.enabled:
raise HTTPException(status_code=403, detail="跨服务器迁移功能未开启")
def current_user_id(request: Request) -> str:
user = getattr(request.state, "user", None)
if user is not None and getattr(user, "id", None):
return str(user.id)
auth = getattr(request.state, "auth", None)
auth_user = getattr(auth, "user", None) if auth is not None else None
if auth_user is not None and getattr(auth_user, "id", None):
return str(auth_user.id)
return get_effective_user_id()
def payload_size(value: Any) -> int:
return len(json.dumps(value, ensure_ascii=False, default=str).encode("utf-8"))
def clean_export_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
return {
key: item
for key, item in _strip_reserved_metadata(metadata).items()
if key not in _DROP_META_KEYS
}
def safe_relpath(root: Path, path: Path) -> str | None:
try:
resolved = path.resolve()
relative = resolved.relative_to(root.resolve())
except ValueError:
return None
text = relative.as_posix()
if not SAFE_REL_PATH.fullmatch(text):
return None
return text
def encoded_file_cost(size: int, path: str) -> int:
return (size * 4 + 3) // 3 + len(path) + _FILE_JSON_OVERHEAD
def list_thread_file_candidates(root: Path) -> list[tuple[int, str, Path]]:
if not root.is_dir():
return []
candidates: list[tuple[int, str, Path]] = []
for path in sorted(root.rglob("*")):
if not path.is_file() or path.is_symlink():
continue
rel = safe_relpath(root, path)
if rel is None:
continue
try:
size = path.stat().st_size
except OSError:
continue
if size < 0:
continue
candidates.append((size, rel, path))
candidates.sort(key=lambda item: (item[0], item[1]))
return candidates
def collect_thread_files(root: Path, *, budget: int) -> tuple[list[dict[str, Any]], list[str]]:
candidates = list_thread_file_candidates(root)
if not candidates:
return [], []
files: list[dict[str, Any]] = []
skipped: list[str] = []
used = 0
for size, rel, path in candidates:
cost = encoded_file_cost(size, rel)
if budget <= 0 or len(files) >= MAX_FILES_PER_THREAD or used + cost > budget:
skipped.append(rel)
continue
try:
data = path.read_bytes()
except OSError:
skipped.append(rel)
continue
files.append(
{
"path": rel,
"content_b64": base64.b64encode(data).decode("ascii"),
"size": size,
}
)
used += cost
return files, skipped
def write_thread_files(
root: Path,
files: list[TransferFile],
*,
budget: int = MAX_THREAD_BYTES,
) -> list[str]:
failed: list[str] = []
used = 0
root.mkdir(parents=True, exist_ok=True)
root_resolved = root.resolve()
for item in files[:MAX_FILES_PER_THREAD]:
rel = safe_relpath(root, root / item.path) if SAFE_REL_PATH.fullmatch(item.path) else None
if rel is None:
failed.append(item.path)
continue
dest = (root / rel).resolve()
if dest != root_resolved and root_resolved not in dest.parents:
failed.append(item.path)
continue
try:
data = base64.b64decode(item.content_b64, validate=True)
if used + len(data) > budget:
failed.append(item.path)
continue
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(data)
used += len(data)
except Exception:
failed.append(item.path)
return failed
def _title_of(record: dict[str, Any], values: dict[str, Any] | None = None) -> str:
title = record.get("display_name") or (values or {}).get("title") or ""
return str(title).strip()
def bump_channel_versions(checkpointer: Any, checkpoint: dict[str, Any], keys: list[str]) -> dict[str, Any]:
"""LangGraph stores each channel as a versioned blob.
``aput(..., new_versions={})`` only keeps the in-memory dict on some
savers. Postgres/SQLite ignore unversioned channels, so imported
messages would not survive ``aget`` / ``GET /threads/{id}/state``.
"""
versions = dict(checkpoint.get("channel_versions") or {})
new_versions: dict[str, Any] = {}
get_next_version = getattr(checkpointer, "get_next_version", None)
if not callable(get_next_version):
return dict(versions)
for key in keys:
next_version = get_next_version(versions.get(key), None)
versions[key] = next_version
new_versions[key] = next_version
checkpoint["channel_versions"] = versions
return new_versions
async def write_imported_checkpoint(
checkpointer: Any,
*,
thread_id: str,
values: dict[str, Any],
metadata: dict[str, Any],
) -> None:
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
now = now_iso()
checkpoint = dict(empty_checkpoint())
checkpoint["channel_values"] = dict(values)
new_versions = bump_channel_versions(checkpointer, checkpoint, list(values.keys()))
await checkpointer.aput(
config,
checkpoint,
{
"step": 1,
"source": "update",
"created_at": now,
"updated_at": now,
**metadata,
},
new_versions,
)
written = await checkpointer.aget_tuple(config)
if written is None:
raise RuntimeError("missing checkpoint after import")
restored = serialize_channel_values(
(getattr(written, "checkpoint", {}) or {}).get("channel_values", {}) or {}
)
if values.get("messages") and not restored.get("messages"):
raise RuntimeError("checkpoint messages were not persisted")
def export_job_dir(user_id: str) -> Path:
safe = _SAFE_USER_DIR.sub("_", user_id).strip("._") or "user"
path = get_paths().base_dir / "chat-history-exports" / safe
path.mkdir(parents=True, exist_ok=True)
return path
def get_export_job(job_id: str, user_id: str) -> ExportAllJob:
job = _EXPORT_JOBS.get(job_id)
if job is None or job.user_id != user_id:
raise HTTPException(status_code=404, detail="导出任务不存在")
return job
def find_running_export_job(user_id: str) -> ExportAllJob | None:
for job in _EXPORT_JOBS.values():
if job.user_id == user_id and job.status == "running":
return job
return None
def purge_finished_export_jobs(user_id: str) -> None:
stale = [
job_id
for job_id, job in _EXPORT_JOBS.items()
if job.user_id == user_id and job.status != "running"
]
for job_id in stale:
job = _EXPORT_JOBS.pop(job_id, None)
if job is None or job.path is None:
continue
try:
job.path.unlink(missing_ok=True)
except OSError:
logger.debug("Failed to remove export file %s", job.path)
def job_status(job: ExportAllJob) -> ExportAllStatusResponse:
return ExportAllStatusResponse(
job_id=job.job_id,
status=job.status,
total=job.total,
processed=job.processed,
succeeded=job.succeeded,
skipped=list(job.skipped),
skipped_files=job.skipped_files,
error=job.error,
)
class BundleWriter:
def __init__(self, path: Path) -> None:
self.path = path
self._fh = path.open("w", encoding="utf-8")
self._first = True
self._fh.write(
'{"format":%s,"version":%s,"exported_at":%s,"threads":['
% (
json.dumps(FORMAT, ensure_ascii=False),
FORMAT_VERSION,
json.dumps(now_iso()),
)
)
def add_thread(self, thread: TransferThread) -> None:
if not self._first:
self._fh.write(",")
self._first = False
self._fh.write(thread.model_dump_json())
self._fh.flush()
def finish(self, skipped: list[SkippedThread]) -> None:
self._fh.write('],"skipped":')
self._fh.write(json.dumps([item.model_dump() for item in skipped], ensure_ascii=False))
self._fh.write("}")
self._fh.close()
def abort(self) -> None:
try:
self._fh.close()
except Exception:
pass
try:
self.path.unlink(missing_ok=True)
except OSError:
pass
async def list_exportable_thread_ids(thread_store: Any, user_id: str) -> list[str]:
ids: list[str] = []
offset = 0
page_size = 100
while True:
rows = await thread_store.search(
user_id=user_id,
exclude_system=True,
limit=page_size,
offset=offset,
)
ids.extend(str(row["thread_id"]) for row in rows if row.get("thread_id"))
if len(rows) < page_size:
break
offset += len(rows)
return ids
async def run_export_all_job(
job: ExportAllJob,
*,
thread_store: Any,
checkpointer: Any,
) -> None:
writer: BundleWriter | None = None
try:
thread_ids = await list_exportable_thread_ids(thread_store, job.user_id)
job.total = len(thread_ids)
if not thread_ids:
job.status = "done"
return
dest = export_job_dir(job.user_id) / f"{job.job_id}.json"
writer = BundleWriter(dest)
for thread_id in thread_ids:
try:
payload, skip = await export_one_thread(
thread_id=thread_id,
user_id=job.user_id,
thread_store=thread_store,
checkpointer=checkpointer,
)
except Exception:
logger.exception("Unexpected export failure for %s", sanitize_log_param(thread_id))
payload, skip = None, SkippedThread(thread_id=thread_id, reason="export_failed")
if payload is not None:
await asyncio.to_thread(writer.add_thread, payload)
job.succeeded += 1
job.skipped_files += len(payload.skipped_files)
if skip is not None:
job.skipped.append(skip)
job.processed += 1
await asyncio.to_thread(writer.finish, list(job.skipped))
job.path = dest
job.status = "done"
except Exception:
logger.exception("Background chat-history export failed for %s", sanitize_log_param(job.user_id))
if writer is not None:
writer.abort()
job.status = "error"
job.error = "导出失败"
async def export_one_thread(
*,
thread_id: str,
user_id: str,
thread_store: Any,
checkpointer: Any,
max_bytes: int = MAX_THREAD_BYTES,
) -> tuple[TransferThread | None, SkippedThread | None]:
record = await thread_store.get(thread_id, user_id=user_id)
if record is None:
return None, SkippedThread(thread_id=thread_id, reason="not_found")
if is_excluded_system_thread(record.get("metadata")):
return None, SkippedThread(
thread_id=thread_id,
title=_title_of(record),
reason="system_thread",
)
try:
checkpoint_tuple = await checkpointer.aget_tuple(
{"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
)
except Exception:
logger.exception("Failed to read checkpoint for export %s", sanitize_log_param(thread_id))
return None, SkippedThread(thread_id=thread_id, title=_title_of(record), reason="export_failed")
values: dict[str, Any] = {}
if checkpoint_tuple is not None:
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
try:
values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
except Exception:
logger.exception("Failed to serialize state for export %s", sanitize_log_param(thread_id))
return None, SkippedThread(thread_id=thread_id, title=_title_of(record), reason="export_failed")
title = _title_of(record, values)
if title and not values.get("title"):
values["title"] = title
used = payload_size(values)
if used > max_bytes:
return None, SkippedThread(thread_id=thread_id, title=title, reason="too_large")
user_data = get_paths().sandbox_user_data_dir(thread_id, user_id=user_id)
files, skipped_files = await asyncio.to_thread(
collect_thread_files, user_data, budget=max_bytes - used
)
payload = TransferThread(
thread_id=thread_id,
title=title,
assistant_id=record.get("assistant_id"),
metadata=clean_export_metadata(record.get("metadata") or {}),
created_at=coerce_iso(record.get("created_at", "")),
updated_at=coerce_iso(record.get("updated_at", "")),
values=values,
files=[TransferFile.model_validate(item) for item in files],
skipped_files=list(skipped_files),
)
while payload.files and payload_size(payload.model_dump()) > max_bytes:
largest = max(payload.files, key=lambda item: (item.size, item.path))
payload.files.remove(largest)
payload.skipped_files.append(largest.path)
if payload_size(payload.model_dump()) > max_bytes:
return None, SkippedThread(thread_id=thread_id, title=title, reason="too_large")
return payload, None
async def import_one_thread(
*,
item: TransferThread,
user_id: str,
thread_store: Any,
checkpointer: Any,
) -> SkippedThread | None:
thread_id = item.thread_id.strip()
if not thread_id or not re.fullmatch(r"[A-Za-z0-9_-]+", thread_id):
return SkippedThread(thread_id=item.thread_id or "unknown", title=item.title, reason="invalid_id")
existing = await thread_store.get(thread_id, user_id=user_id)
if existing is not None:
return SkippedThread(thread_id=thread_id, title=item.title or _title_of(existing), reason="exists")
metadata = clean_export_metadata(item.metadata)
title = (item.title or str(item.values.get("title") or "")).strip()
values = dict(item.values or {})
if title:
values["title"] = title
created = False
try:
await thread_store.create(
thread_id,
assistant_id=item.assistant_id,
user_id=user_id,
display_name=title or None,
metadata=metadata,
)
created = True
await write_imported_checkpoint(
checkpointer,
thread_id=thread_id,
values=values,
metadata=metadata,
)
if title:
try:
await thread_store.update_display_name(thread_id, title, user_id=user_id)
except Exception:
logger.debug("Failed to sync imported title for %s", sanitize_log_param(thread_id))
if item.files:
dest = get_paths().sandbox_user_data_dir(thread_id, user_id=user_id)
failed_files = await asyncio.to_thread(write_thread_files, dest, item.files)
if failed_files:
logger.warning(
"Imported thread %s with %d skipped files",
sanitize_log_param(thread_id),
len(failed_files),
)
except Exception:
logger.exception("Failed to import thread %s", sanitize_log_param(thread_id))
if created:
try:
await thread_store.delete(thread_id, user_id=user_id)
except Exception:
logger.debug("Failed to roll back imported thread %s", sanitize_log_param(thread_id))
return SkippedThread(thread_id=thread_id, title=title, reason="import_failed")
return None
@router.post("/export", response_model=ExportResponse, summary="Export a batch of conversations")
@require_permission("threads", "read")
async def export_chat_history(body: ExportRequest, request: Request) -> ExportResponse:
require_transfer_enabled()
user_id = current_user_id(request)
thread_store = get_thread_store(request)
checkpointer = get_checkpointer(request)
exported: list[TransferThread] = []
skipped: list[SkippedThread] = []
for thread_id in body.thread_ids:
try:
payload, skip = await export_one_thread(
thread_id=thread_id,
user_id=user_id,
thread_store=thread_store,
checkpointer=checkpointer,
)
except Exception:
logger.exception("Unexpected export failure for %s", sanitize_log_param(thread_id))
skipped.append(SkippedThread(thread_id=thread_id, reason="export_failed"))
continue
if payload is not None:
exported.append(payload)
if skip is not None:
skipped.append(skip)
return ExportResponse(threads=exported, skipped=skipped)
@router.post("/import", response_model=ImportResponse, summary="Import a batch of conversations")
@require_permission("threads", "write")
async def import_chat_history(body: ImportRequest, request: Request) -> ImportResponse:
require_transfer_enabled()
user_id = current_user_id(request)
thread_store = get_thread_store(request)
checkpointer = get_checkpointer(request)
imported = 0
skipped: list[SkippedThread] = []
for item in body.threads:
try:
skip = await import_one_thread(
item=item,
user_id=user_id,
thread_store=thread_store,
checkpointer=checkpointer,
)
except Exception:
logger.exception("Unexpected import failure for %s", sanitize_log_param(item.thread_id))
skipped.append(SkippedThread(thread_id=item.thread_id, title=item.title, reason="import_failed"))
continue
if skip is None:
imported += 1
else:
skipped.append(skip)
return ImportResponse(imported=imported, skipped=skipped)
@router.post("/export-all", response_model=ExportAllStartResponse, summary="Start a backend export of all conversations")
@require_permission("threads", "read")
async def start_export_all(request: Request) -> ExportAllStartResponse:
require_transfer_enabled()
user_id = current_user_id(request)
running = find_running_export_job(user_id)
if running is not None:
return ExportAllStartResponse(job_id=running.job_id)
purge_finished_export_jobs(user_id)
job = ExportAllJob(job_id=uuid.uuid4().hex, user_id=user_id)
_EXPORT_JOBS[job.job_id] = job
job.task = asyncio.create_task(
run_export_all_job(
job,
thread_store=get_thread_store(request),
checkpointer=get_checkpointer(request),
)
)
return ExportAllStartResponse(job_id=job.job_id)
@router.get("/export-all/{job_id}", response_model=ExportAllStatusResponse, summary="Poll a backend export job")
@require_permission("threads", "read")
async def get_export_all_status(job_id: str, request: Request) -> ExportAllStatusResponse:
require_transfer_enabled()
return job_status(get_export_job(job_id, current_user_id(request)))
@router.get("/export-all/{job_id}/download", summary="Download a finished backend export")
@require_permission("threads", "read")
async def download_export_all(job_id: str, request: Request) -> FileResponse:
require_transfer_enabled()
job = get_export_job(job_id, current_user_id(request))
if job.status != "done" or job.path is None or not job.path.is_file():
raise HTTPException(status_code=409, detail="导出文件还没准备好")
return FileResponse(
job.path,
media_type="application/json",
filename="deerflow-chats.json",
)
@@ -10,6 +10,7 @@ from pathlib import Path
from fastapi import APIRouter, HTTPException
from fastapi.responses import FileResponse, Response
from deerflow.config.system_settings import generated_page_logo_path, load_system_settings
from deerflow.html_runtime.addresses import html_map_tile_url, html_runtime_base_url
router = APIRouter(prefix="/api/public/html-runtime", tags=["html-runtime"])
@@ -28,6 +29,12 @@ _STATIC_ASSETS = {
"g6.min.js": "application/javascript",
"leaflet.js": "application/javascript",
"leaflet.css": "text/css",
# 生成页品牌页头背景设计图(浅色红调 / 深色深蓝),随明暗 tone 由页头样式切换。
"topbar-light.svg": "image/svg+xml",
"topbar-dark.svg": "image/svg+xml",
# Packaged fallback used when an administrator-uploaded generated-page
# logo cannot be fetched through a deployment's reverse-proxy path.
"brand-logo-dark.png": "image/png",
}
_GEO_MAP_ASSETS = {
"china-provinces.geojson": "application/geo+json",
@@ -94,6 +101,26 @@ async def get_html_runtime_geo_map(filename: str) -> FileResponse:
)
@router.get("/generated-page-logo")
async def get_generated_page_logo() -> FileResponse:
"""Serve the configured logo, falling back to the packaged default."""
branding = load_system_settings().generated_page_branding
path = generated_page_logo_path(branding)
if path is None:
path = _RUNTIME_DIR / "brand-logo-dark.png"
media_type = "image/png"
else:
media_type = branding.logo_media_type or "application/octet-stream"
if not path.is_file():
raise HTTPException(status_code=503, detail="Generated-page fallback logo is not installed")
return FileResponse(
path,
media_type=media_type,
headers={**_PUBLIC_HEADERS, "Cache-Control": "public, max-age=3600"},
)
@router.get("/{filename}")
async def get_html_runtime_asset(filename: str) -> FileResponse:
"""Serve a whitelisted, vendored browser distribution file."""
@@ -76,6 +76,17 @@ class LightAppResponse(BaseModel):
class LightAppListResponse(BaseModel):
apps: list[LightAppResponse]
class LightAppSidebarItem(BaseModel):
app_id: str
name: str
mount_location: str | None = None
sort_order: int = 0
class LightAppSidebarResponse(BaseModel):
apps: list[LightAppSidebarItem]
class LightAppSidebarItem(BaseModel):
@@ -209,6 +220,13 @@ async def list_sidebar_light_apps(request: Request) -> LightAppSidebarResponse:
return await cached_catalog(request, "light_apps", "sidebar", load, LightAppSidebarResponse)
@router.get("/sidebar", response_model=LightAppSidebarResponse)
async def list_sidebar_light_apps(request: Request) -> LightAppSidebarResponse:
"""Return the small set of fields required to build the shared sidebar."""
rows = await _get_store(request).list_sidebar_apps()
return LightAppSidebarResponse(apps=[LightAppSidebarItem(**row) for row in rows])
@router.post("", response_model=LightAppResponse)
async def create_light_app(request: Request, body: LightAppCreateRequest) -> LightAppResponse:
await _require_admin(request)
@@ -93,6 +93,14 @@ _STEP3_AGENT_IDS = frozenset({
SUMMARY_AGENT_ID,
})
_STEP3_AGENT_IDS = frozenset({
ACTION_PLAN_AGENT_ID,
DASHBOARD_AGENT_ID,
REPORT_AGENT_ID,
STRUCTURE_AGENT_ID,
SUMMARY_AGENT_ID,
})
def _stage_for_agent(agent_type: str, agent_name: str) -> str:
"""把一次 /run/stream 的 (agent_type, agent_name) 映射成诊断日志的 stage。
@@ -131,12 +139,45 @@ def _is_step2_room_limited(
return agent_type == "leader" or agent_name not in _STEP3_AGENT_IDS
# 圆桌(intent / recommend / 席位 / 协调)创建的 thread 都是功能型内部线程,
# 不是用户的普通对话。打上这个 metadata,前端「最近的对话」列表(recent-chat-list.tsx)
# 据 `metadata.thread_type` 过滤掉,与 scheduler / bootstrap 系统会话同一套机制。
_ROUNDTABLE_THREAD_METADATA: dict[str, Any] = {"thread_type": "roundtable", "system": True}
def _room_thread_metadata(
*,
draft_id: str | None,
user_id: str | None,
task_id: str | None,
room_session_id: str | None,
) -> dict[str, Any]:
"""Step-2 threads carry a stable room key so later occupancy can see live runs."""
from deerflow.persistence.roundtable_jobs.sql import compute_active_dedupe_key
from deerflow.persistence.roundtable_rooms.sql import compute_room_session_key
meta = dict(_ROUNDTABLE_THREAD_METADATA)
clean_session_id = (room_session_id or "").strip() or None
key = compute_room_session_key(clean_session_id) or compute_active_dedupe_key(
draft_id, user_id, task_id
)
if key:
meta["roundtable_room_key"] = key
if clean_session_id:
meta["roundtable_room_session_id"] = clean_session_id
if draft_id:
meta["draft_id"] = draft_id
if task_id:
meta["task_id"] = task_id
return meta
def _step3_thread_metadata() -> dict[str, Any]:
"""Step-3 threads are internal threads, but never Step-2 room occupants."""
return dict(_ROUNDTABLE_THREAD_METADATA)
def _room_thread_metadata(
*,
draft_id: str | None,
@@ -905,6 +946,26 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
room_session_id=room_session_id,
)
# The browser already acquired the room before entering Step 2. Init only
# renews that lease; it cannot create a second room or bypass admission.
from app.gateway.roundtable_room_gate import enforce_existing_roundtable_room
room_draft_id = str(payload.get("draft_id") or "").strip() or None
room_task_id = str(payload.get("task_id") or "").strip() or None
room_session_id = str(payload.get("room_session_id") or "").strip() or None
await enforce_existing_roundtable_room(
request,
draft_id=room_draft_id,
task_id=room_task_id,
room_session_id=room_session_id,
)
room_thread_meta = _room_thread_metadata(
draft_id=room_draft_id,
user_id=_resolve_user_id(request),
task_id=room_task_id,
room_session_id=room_session_id,
)
def _frame(payload: dict[str, Any]) -> str:
return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
@@ -153,6 +153,16 @@ async def _refresh_catalog_snapshot(config: AppConfig) -> None:
logger.exception("Skill catalog projection refresh failed after a mutation")
async def _refresh_catalog_snapshot(config: AppConfig) -> None:
"""Keep normalized MySQL skill rows current after a skill edit."""
try:
await refresh_skill_catalog_projection(config)
except Exception:
# A failed projection refresh must not turn a successful skill write
# into an API failure. Startup or the next edit can recover it.
logger.exception("Skill catalog projection refresh failed after a mutation")
def _can_view_skill_source(
category: SkillCategory,
owner_user_id: str | None,
@@ -513,6 +523,69 @@ _SKILL_DISPLAY_TABLE_LOCK = asyncio.Lock()
_skill_display_table_engine: object | None = None
def _skill_display_table_already_exists(exc: BaseException) -> bool:
message = str(exc).lower()
return any(
token in message
for token in (
"already exists",
"duplicate table",
"42p07",
"error 1050",
"(1050",
)
)
def _catalog_item_to_response(
item: dict[str, Any],
ownership: dict[str, Any] | None,
*,
adapter: dict[str, Any] | None,
favorited: bool,
) -> SkillResponse:
"""Build a list response from the MySQL snapshot and MySQL metadata rows."""
try:
category = SkillCategory(str(item.get("category") or SkillCategory.PUBLIC.value))
except ValueError:
category = SkillCategory.PUBLIC
owner_user_id = ownership.get("owner_user_id") if ownership else None
return SkillResponse(
name=str(item.get("name") or ""),
name_zh=ownership.get("name_zh") if ownership else None,
description=str(item.get("description") or ""),
license=item.get("license"),
category=category,
enabled=bool(item.get("enabled", True)),
always_on=bool(ownership.get("always_on", item.get("always_on", False)) if ownership else item.get("always_on", False)),
use_count=int(item.get("use_count") or 0),
view_count=int(item.get("view_count") or 0),
patch_count=int(item.get("patch_count") or 0),
last_activity_at=item.get("last_activity_at"),
created_at=item.get("created_at"),
state=str(item.get("state") or "active"),
pinned=bool(item.get("pinned")),
featured=bool(item.get("featured")),
source=str(item.get("source") or "upload"),
owner_user_id=owner_user_id,
published=bool(ownership.get("published", False)) if ownership else False,
square_id=str(ownership.get("square_id") or "") if ownership else "",
builtin=bool(ownership.get("builtin", owner_user_id is None)) if ownership else category != SkillCategory.CUSTOM,
favorited=favorited,
detail=(ownership.get("detail_preview") or ownership.get("detail")) if ownership else None,
display=adapter.get("display") if adapter else None,
displaySample=adapter.get("sample") if adapter else None,
)
# ``checkfirst=True`` is a check-then-create operation. The in-process lock
# serializes concurrent requests in one worker; the duplicate-table fallback
# covers two Gateway workers racing during first access.
_SKILL_DISPLAY_TABLE_LOCK = asyncio.Lock()
_skill_display_table_engine: object | None = None
def _skill_display_table_already_exists(exc: BaseException) -> bool:
message = str(exc).lower()
return any(
@@ -8,19 +8,22 @@ from typing import Literal
from urllib.parse import urlparse
from uuid import uuid4
from fastapi import APIRouter, HTTPException, Request
from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
from pydantic import BaseModel, Field
from app.gateway.auth.password import hash_password_async
from app.gateway.deps import get_current_user_from_request, get_optional_user_from_request
from deerflow.config import get_app_config
from deerflow.config.runtime_paths import runtime_home
from deerflow.config.system_settings import (
BUILTIN_THEME_IDS,
AgentSkillMigrationSettings,
AppearanceRouteParam,
AuthenticationSettings,
CompactShortcutButton,
ConversationContextSettings,
CustomThemePreset,
GeneratedPageBrandingSettings,
HtmlMapSettings,
LeaderboardSettings,
ModelConcurrencySettings,
@@ -36,6 +39,8 @@ from deerflow.config.system_settings import (
StrategyProjectSettings,
WordHeadingNumberingStyle,
WriteFileCompatibilitySettings,
generated_page_logo_path,
generated_page_logo_public_url,
get_module_tag_ids,
list_publish_squares,
load_system_settings,
@@ -313,11 +318,15 @@ class HtmlMapSettingsUpdateRequest(BaseModel):
class HtmlRuntimeSettingsResponse(BaseModel):
base_url: str
map_tile_url: str
message_map_tile_min_zoom: int
message_map_tile_max_zoom: int
class HtmlRuntimeSettingsUpdateRequest(BaseModel):
base_url: str
map_tile_url: str
message_map_tile_min_zoom: int | None = Field(default=None, ge=0, le=20)
message_map_tile_max_zoom: int | None = Field(default=None, ge=0, le=20)
class LeaderboardSettingsResponse(BaseModel):
@@ -518,6 +527,17 @@ class MemoryRuntimeUpdateRequest(BaseModel):
enabled: bool
class AgentSkillMigrationResponse(BaseModel):
enabled: bool = Field(
default=False,
description="Whether administrator agent/skill migration is enabled deployment-wide",
)
class AgentSkillMigrationUpdateRequest(BaseModel):
enabled: bool
class WriteFileCompatibilityResponse(BaseModel):
enabled: bool = Field(
default=False,
@@ -617,6 +637,7 @@ def _knowledge_retrieval_settings_store(request: Request):
return store
@router.get("/memory", response_model=MemoryRuntimeResponse)
async def get_memory_runtime_setting() -> MemoryRuntimeResponse:
return MemoryRuntimeResponse(enabled=load_system_settings().memory.enabled)
@@ -635,6 +656,28 @@ async def update_memory_runtime_setting(
return MemoryRuntimeResponse(enabled=saved.memory.enabled)
@router.get("/agent-skill-migration", response_model=AgentSkillMigrationResponse)
async def get_agent_skill_migration_setting() -> AgentSkillMigrationResponse:
return AgentSkillMigrationResponse(
enabled=load_system_settings().agent_skill_migration.enabled,
)
@router.put("/agent-skill-migration", response_model=AgentSkillMigrationResponse)
async def update_agent_skill_migration_setting(
body: AgentSkillMigrationUpdateRequest,
request: Request,
) -> AgentSkillMigrationResponse:
await _require_admin(request)
current = load_system_settings()
updated = AgentSkillMigrationSettings(enabled=body.enabled)
saved = save_system_settings(
current.model_copy(update={"agent_skill_migration": updated})
)
logger.info("Agent/skill migration switch changed: enabled=%s", saved.agent_skill_migration.enabled)
return AgentSkillMigrationResponse(enabled=saved.agent_skill_migration.enabled)
@router.get(
"/write-file-compatibility",
response_model=WriteFileCompatibilityResponse,
@@ -837,6 +880,7 @@ async def update_ordinary_knowledge_retrieval_setting(
)
def _skill_compression_to_response(settings: SkillCompressionSettings) -> SkillCompressionResponse:
return SkillCompressionResponse(
enabled=settings.enabled,
@@ -1073,10 +1117,13 @@ async def update_html_map_settings(
@router.get("/html-runtime", response_model=HtmlRuntimeSettingsResponse)
async def get_html_runtime_settings() -> HtmlRuntimeSettingsResponse:
"""Return the addresses newly generated HTML pages will use."""
"""Return generated-page addresses and message-map native tile limits."""
settings = load_system_settings().html_runtime
return HtmlRuntimeSettingsResponse(
base_url=html_runtime_base_url(),
map_tile_url=html_map_tile_url(),
message_map_tile_min_zoom=settings.message_map_tile_min_zoom,
message_map_tile_max_zoom=settings.message_map_tile_max_zoom,
)
@@ -1085,7 +1132,7 @@ async def update_html_runtime_settings(
body: HtmlRuntimeSettingsUpdateRequest,
request: Request,
) -> HtmlRuntimeSettingsResponse:
"""Save the browser-reachable runtime and map-tile addresses. Admin only."""
"""Save generated-page addresses and message-map native tile limits. Admin only."""
await _require_admin(request)
try:
base_url = normalize_html_runtime_base_url(body.base_url)
@@ -1093,13 +1140,32 @@ async def update_html_runtime_settings(
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
current = load_system_settings()
min_zoom = (
body.message_map_tile_min_zoom
if body.message_map_tile_min_zoom is not None
else current.html_runtime.message_map_tile_min_zoom
)
max_zoom = (
body.message_map_tile_max_zoom
if body.message_map_tile_max_zoom is not None
else current.html_runtime.message_map_tile_max_zoom
)
if min_zoom > max_zoom:
raise HTTPException(status_code=400, detail="消息列表瓦片最小层级不能大于最大层级")
updated = current.html_runtime.model_copy(
update={"base_url": base_url, "map_tile_url": map_tile_url}
update={
"base_url": base_url,
"map_tile_url": map_tile_url,
"message_map_tile_min_zoom": min_zoom,
"message_map_tile_max_zoom": max_zoom,
}
)
saved = save_system_settings(current.model_copy(update={"html_runtime": updated}))
return HtmlRuntimeSettingsResponse(
base_url=saved.html_runtime.base_url,
map_tile_url=saved.html_runtime.map_tile_url,
message_map_tile_min_zoom=saved.html_runtime.message_map_tile_min_zoom,
message_map_tile_max_zoom=saved.html_runtime.message_map_tile_max_zoom,
)
@@ -1422,6 +1488,230 @@ async def update_appearance_settings(body: AppearanceUpdateRequest, request: Req
return _appearance_response(saved.appearance)
class GeneratedPageBrandingResponse(BaseModel):
"""Generated-page identity: name, logo display mode and optional custom logo."""
chrome_enabled: bool = True
header_layout: str = "center"
system_name: str | None = None
footer_text: str | None = None
logo_mode: Literal["custom", "default", "none"] = "custom"
logo_url: str | None = None
logo_version: str | None = None
customized: bool = False
active: bool = False
_MAX_BRAND_LOGO_BYTES = 10 * 1024 * 1024
def _detect_brand_logo(data: bytes) -> tuple[str, str]:
"""Return a safe extension/media type from file signatures, never the filename."""
if data.startswith(b"\x89PNG\r\n\x1a\n"):
return ".png", "image/png"
if data.startswith(b"\xff\xd8\xff"):
return ".jpg", "image/jpeg"
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
return ".webp", "image/webp"
raise HTTPException(status_code=422, detail="Logo 仅支持 PNG、JPG 或 WebP 图片")
def _normalize_system_name(value: str | None) -> str | None:
if value is None:
return None
normalized = " ".join(value.split())
if len(normalized) > 80:
raise HTTPException(status_code=422, detail="系统名称不能超过 80 个字符")
return normalized or None
def _normalize_footer_text(value: str | None) -> str | None:
if value is None:
return None
normalized = " ".join(value.split())
if len(normalized) > 160:
raise HTTPException(status_code=422, detail="网页页脚文字不能超过 160 个字符")
return normalized or None
def _generated_page_branding_response(
settings: GeneratedPageBrandingSettings,
) -> GeneratedPageBrandingResponse:
defaults = GeneratedPageBrandingSettings()
logo_url = generated_page_logo_public_url("/api/public/html-runtime", settings)
return GeneratedPageBrandingResponse(
chrome_enabled=settings.chrome_enabled,
header_layout=settings.header_layout,
system_name=settings.system_name,
footer_text=settings.footer_text,
logo_mode=settings.logo_mode,
logo_url=logo_url,
logo_version=settings.logo_version if logo_url else None,
customized=bool(
settings.system_name
or settings.footer_text
or logo_url
or settings.logo_mode != defaults.logo_mode
or settings.chrome_enabled != defaults.chrome_enabled
or settings.header_layout != defaults.header_layout
),
active=bool(
settings.system_name
and settings.chrome_enabled
and (settings.logo_mode == "none" or logo_url)
),
)
def _write_generated_page_logo(data: bytes, suffix: str) -> str:
directory = runtime_home() / "generated-page-branding"
directory.mkdir(parents=True, exist_ok=True)
filename = f"logo-{uuid4().hex}{suffix}"
destination = directory / filename
temporary = directory / f".{filename}.tmp"
try:
temporary.write_bytes(data)
temporary.replace(destination)
finally:
temporary.unlink(missing_ok=True)
return filename
@router.get("/generated-page-branding", response_model=GeneratedPageBrandingResponse)
async def get_generated_page_branding_settings() -> GeneratedPageBrandingResponse:
"""Return the optional generated-page identity. Any logged-in user may read it."""
return _generated_page_branding_response(load_system_settings().generated_page_branding)
@router.put("/generated-page-branding", response_model=GeneratedPageBrandingResponse)
async def update_generated_page_branding_settings(
request: Request,
chrome_enabled: bool | None = Form(default=None),
header_layout: str | None = Form(default=None),
system_name: str | None = Form(default=None),
logo_mode: Literal["custom", "default", "none"] | None = Form(default=None),
# Plain str (not Optional): FastAPI coerces an empty Optional form value to
# None, which would make "clear the footer text" indistinguishable from
# "field omitted". The settings UI always sends this field, so an empty
# value here means the admin cleared the input and wants the default footer.
footer_text: str = Form(default=""),
remove_logo: bool = Form(default=False),
logo: UploadFile | None = File(default=None),
) -> GeneratedPageBrandingResponse:
"""Update the identity used in generated pages and the auto-injection switch. Admin only."""
await _require_admin(request)
if logo is not None and (remove_logo or logo_mode in {"default", "none"}):
raise HTTPException(status_code=422, detail="上传 Logo 时不能同时选择默认头像或不显示头像")
if remove_logo and logo_mode not in {None, "none"}:
raise HTTPException(status_code=422, detail="移除 Logo 时只能选择不显示头像")
current = load_system_settings()
previous_logo = generated_page_logo_path(current.generated_page_branding)
update: dict[str, object] = {}
if chrome_enabled is not None:
update["chrome_enabled"] = bool(chrome_enabled)
if header_layout is not None:
normalized_layout = str(header_layout).strip().lower()
if normalized_layout not in {"left", "center"}:
raise HTTPException(status_code=422, detail="header_layout 仅支持 left 或 center")
update["header_layout"] = normalized_layout
if system_name is not None:
update["system_name"] = _normalize_system_name(system_name)
update["footer_text"] = _normalize_footer_text(footer_text)
new_logo_path = None
if logo is not None:
data = await logo.read(_MAX_BRAND_LOGO_BYTES + 1)
await logo.close()
if not data:
raise HTTPException(status_code=422, detail="Logo 文件不能为空")
if len(data) > _MAX_BRAND_LOGO_BYTES:
raise HTTPException(status_code=413, detail="Logo 文件不能超过 10MB")
suffix, media_type = _detect_brand_logo(data)
filename = await asyncio.to_thread(_write_generated_page_logo, data, suffix)
new_logo_path = runtime_home() / "generated-page-branding" / filename
update.update(
{
"logo_mode": "custom",
"logo_filename": filename,
"logo_media_type": media_type,
"logo_version": uuid4().hex,
}
)
elif logo_mode == "none":
update.update(
{
"logo_mode": "none",
"logo_filename": None,
"logo_media_type": None,
"logo_version": None,
}
)
elif remove_logo:
# Preserve the legacy API behavior: an older client that only sends
# remove_logo leaves branding as an incomplete custom-logo draft.
# The new text-only header is an explicit logo_mode="none" choice.
update.update(
{
"logo_mode": "custom",
"logo_filename": None,
"logo_media_type": None,
"logo_version": None,
}
)
elif logo_mode == "default":
update.update(
{
"logo_mode": "default",
"logo_filename": None,
"logo_media_type": None,
"logo_version": "default",
}
)
elif logo_mode == "custom":
update["logo_mode"] = "custom"
if current.generated_page_branding.logo_mode != "custom":
update.update(
{
"logo_filename": None,
"logo_media_type": None,
"logo_version": None,
}
)
if not update:
return _generated_page_branding_response(current.generated_page_branding)
updated = current.generated_page_branding.model_copy(update=update)
try:
saved = save_system_settings(current.model_copy(update={"generated_page_branding": updated}))
except Exception:
if new_logo_path is not None:
await asyncio.to_thread(new_logo_path.unlink, missing_ok=True)
raise
if previous_logo is not None and previous_logo != generated_page_logo_path(saved.generated_page_branding):
await asyncio.to_thread(previous_logo.unlink, missing_ok=True)
return _generated_page_branding_response(saved.generated_page_branding)
@router.delete("/generated-page-branding", response_model=GeneratedPageBrandingResponse)
async def delete_generated_page_branding_settings(
request: Request,
) -> GeneratedPageBrandingResponse:
"""Remove generated-page identity requirements; the app UI is unaffected."""
await _require_admin(request)
current = load_system_settings()
previous_logo = generated_page_logo_path(current.generated_page_branding)
saved = save_system_settings(current.model_copy(update={"generated_page_branding": GeneratedPageBrandingSettings()}))
if previous_logo is not None:
await asyncio.to_thread(previous_logo.unlink, missing_ok=True)
return _generated_page_branding_response(saved.generated_page_branding)
class LogSystemResponse(BaseModel):
"""External log-system entry shown in the admin settings menu.
@@ -1575,3 +1865,4 @@ async def update_wx_common_agents(
updated = current.wx_common_agents.model_copy(update={"agent_type": agent_type})
saved = save_system_settings(current.model_copy(update={"wx_common_agents": updated}))
return WxCommonAgentsResponse(agent_type=saved.wx_common_agents.agent_type)
@@ -546,8 +546,8 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
"subagent_enabled",
"max_concurrent_subagents",
"agent_name",
# 页面研报设计师由前端随消息带入当前界面明暗模式;需要进入 agent
# 提示词及 ToolRuntime.context,才能选择同色系模板。
# 前端随消息只带入当前界面的明暗模式;需要进入 agent 提示词及
# ToolRuntime.context,才能选择对应的浅色或深色页面模板。
"presentation_theme",
"is_bootstrap",
"is_writing_setup",