sync snapshot from LAN repo (source master-zg @ 40658c74)
This commit is contained in:
1 parent
ea9dcff9c0
commit
754b61572c
179 files changed
+26085
-8788
No files matched your search
@@ -0,0 +1,232 @@
|
||||
"""Publish product-owned public skills into the configured live skill root.
|
||||
|
||||
The Gateway may be launched through the supported shell scripts, an image
|
||||
entrypoint, Compose, or an external release repository. Startup therefore
|
||||
cannot rely on one shell wrapper to populate a persistent ``/app/skills``
|
||||
mount. This module is deliberately standard-library-only so both the launcher
|
||||
CLI and the Gateway lifespan can use the exact same synchronization contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import shutil
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
ROUTED_PAGE_SKILLS = (
|
||||
"facebook-page-generator",
|
||||
"x-page-generator",
|
||||
"tiktok-page-generator",
|
||||
"youtube-page-generator",
|
||||
"threads-page-generator",
|
||||
"cnn-page-generator",
|
||||
"bbc-page-generator",
|
||||
"xinhua-page-generator",
|
||||
"white-house-page-generator",
|
||||
"manila-times-page-generator",
|
||||
"gma-news-page-generator",
|
||||
"yahoo-japan-page-generator",
|
||||
"yomiuri-news-page-generator",
|
||||
)
|
||||
|
||||
_BUNDLED_SKILLS_ENV = "DEER_FLOW_BUNDLED_SKILLS_PATH"
|
||||
_SYNC_LOCK_FILE = ".builtin-public-skills.lock"
|
||||
_COPY_IGNORE = shutil.ignore_patterns("__pycache__", "*.pyc", "*.pyo")
|
||||
|
||||
|
||||
class SkillSyncError(RuntimeError):
|
||||
"""Raised when bundled skills cannot be published safely."""
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _exclusive_target_lock(target_root: Path) -> Iterator[None]:
|
||||
"""Serialize startup copies across Uvicorn worker processes."""
|
||||
|
||||
target_root.mkdir(parents=True, exist_ok=True)
|
||||
lock_path = target_root / _SYNC_LOCK_FILE
|
||||
with lock_path.open("a+b") as handle:
|
||||
handle.seek(0, os.SEEK_END)
|
||||
if handle.tell() == 0:
|
||||
handle.write(b"\0")
|
||||
handle.flush()
|
||||
handle.seek(0)
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_LOCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_EX)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
handle.seek(0)
|
||||
if os.name == "nt":
|
||||
import msvcrt
|
||||
|
||||
msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
|
||||
else:
|
||||
import fcntl
|
||||
|
||||
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
|
||||
|
||||
|
||||
def sync_builtin_public_skills(
|
||||
source_root: Path,
|
||||
target_root: Path,
|
||||
*,
|
||||
required: tuple[str, ...] = (),
|
||||
) -> dict[str, object]:
|
||||
"""Overlay bundled ``public`` skills while preserving live custom skills."""
|
||||
|
||||
source_public = source_root.resolve() / "public"
|
||||
target_root = target_root.resolve()
|
||||
target_public = target_root / "public"
|
||||
target_custom = target_root / "custom"
|
||||
if not source_public.is_dir():
|
||||
raise SkillSyncError(f"Bundled public skills directory not found: {source_public}")
|
||||
|
||||
target_public.mkdir(parents=True, exist_ok=True)
|
||||
target_custom.mkdir(parents=True, exist_ok=True)
|
||||
copied: list[str] = []
|
||||
discovered: list[str] = []
|
||||
source_is_live = source_public == target_public.resolve()
|
||||
for source in sorted(source_public.iterdir(), key=lambda path: path.name):
|
||||
if not source.is_dir() or source.name.startswith("."):
|
||||
continue
|
||||
if not source_is_live:
|
||||
destination = target_public / source.name
|
||||
shutil.copytree(
|
||||
source,
|
||||
destination,
|
||||
dirs_exist_ok=True,
|
||||
symlinks=True,
|
||||
ignore=_COPY_IGNORE,
|
||||
)
|
||||
copied.append(source.name)
|
||||
if (source / "SKILL.md").is_file():
|
||||
discovered.append(source.name)
|
||||
|
||||
missing = sorted(name for name in required if not (target_public / name / "SKILL.md").is_file())
|
||||
if missing:
|
||||
raise SkillSyncError("Required built-in skills are unavailable after sync: " + ", ".join(missing))
|
||||
return {
|
||||
"status": "ok",
|
||||
"source": str(source_public),
|
||||
"target": str(target_public),
|
||||
"sourceIsLive": source_is_live,
|
||||
"copiedDirectories": copied,
|
||||
"discoveredSkills": discovered,
|
||||
"requiredSkills": list(required),
|
||||
}
|
||||
|
||||
|
||||
def _required_packages_digest(public_root: Path, required: tuple[str, ...]) -> str | None:
|
||||
"""Hash the required packages, ignoring interpreter cache artifacts."""
|
||||
|
||||
digest = hashlib.sha256()
|
||||
for name in required:
|
||||
skill_root = public_root / name
|
||||
if not (skill_root / "SKILL.md").is_file():
|
||||
return None
|
||||
files = sorted(
|
||||
path
|
||||
for path in skill_root.rglob("*")
|
||||
if path.is_file()
|
||||
and "__pycache__" not in path.parts
|
||||
and path.suffix not in {".pyc", ".pyo"}
|
||||
)
|
||||
for path in files:
|
||||
relative = path.relative_to(public_root).as_posix().encode("utf-8")
|
||||
digest.update(len(relative).to_bytes(4, "big"))
|
||||
digest.update(relative)
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def publish_bundled_public_skills(
|
||||
source_root: Path,
|
||||
target_root: Path,
|
||||
*,
|
||||
required: tuple[str, ...] = ROUTED_PAGE_SKILLS,
|
||||
) -> dict[str, object]:
|
||||
"""Lock, compare and publish the current bundled public skill tree."""
|
||||
|
||||
source_root = source_root.resolve()
|
||||
target_root = target_root.resolve()
|
||||
if source_root == target_root:
|
||||
result = sync_builtin_public_skills(source_root, target_root, required=required)
|
||||
result["skipped"] = True
|
||||
return result
|
||||
with _exclusive_target_lock(target_root):
|
||||
source_digest = _required_packages_digest(source_root / "public", required)
|
||||
if source_digest is None:
|
||||
missing = sorted(
|
||||
name for name in required if not (source_root / "public" / name / "SKILL.md").is_file()
|
||||
)
|
||||
raise SkillSyncError("Required bundled skills are unavailable: " + ", ".join(missing))
|
||||
target_digest = _required_packages_digest(target_root / "public", required)
|
||||
if source_digest == target_digest:
|
||||
return {
|
||||
"status": "ok",
|
||||
"source": str(source_root / "public"),
|
||||
"target": str(target_root / "public"),
|
||||
"sourceIsLive": (source_root / "public") == (target_root / "public"),
|
||||
"copiedDirectories": [],
|
||||
"discoveredSkills": list(required),
|
||||
"requiredSkills": list(required),
|
||||
"skipped": True,
|
||||
}
|
||||
result = sync_builtin_public_skills(source_root, target_root, required=required)
|
||||
result["skipped"] = False
|
||||
return result
|
||||
|
||||
|
||||
def resolve_bundled_skills_root() -> Path:
|
||||
"""Return the authoritative bundled skill root for this deployment."""
|
||||
|
||||
explicit = os.getenv(_BUNDLED_SKILLS_ENV, "").strip()
|
||||
if explicit:
|
||||
candidate = Path(explicit).expanduser().resolve()
|
||||
if not (candidate / "public").is_dir():
|
||||
raise SkillSyncError(f"Configured bundled skills directory not found: {candidate / 'public'}")
|
||||
return candidate
|
||||
|
||||
packaged = Path(__file__).resolve().parents[1] / "skills"
|
||||
candidates = (Path("/app/bundled-skills"), packaged)
|
||||
for candidate in candidates:
|
||||
resolved = candidate.resolve()
|
||||
if (resolved / "public").is_dir():
|
||||
return resolved
|
||||
raise SkillSyncError(
|
||||
"Bundled public skills directory not found; checked: "
|
||||
+ ", ".join(str(candidate.resolve() / "public") for candidate in candidates)
|
||||
)
|
||||
|
||||
|
||||
def ensure_bundled_public_skills(
|
||||
app_config,
|
||||
*,
|
||||
required: tuple[str, ...] = ROUTED_PAGE_SKILLS,
|
||||
) -> dict[str, object]:
|
||||
"""Synchronize bundled skills into the configured root during startup."""
|
||||
|
||||
source_root = resolve_bundled_skills_root()
|
||||
target_root = Path(app_config.skills.get_skills_path()).resolve()
|
||||
return publish_bundled_public_skills(source_root, target_root, required=required)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ROUTED_PAGE_SKILLS",
|
||||
"SkillSyncError",
|
||||
"ensure_bundled_public_skills",
|
||||
"publish_bundled_public_skills",
|
||||
"resolve_bundled_skills_root",
|
||||
"sync_builtin_public_skills",
|
||||
]
|
||||
@@ -0,0 +1,924 @@
|
||||
"""Agent/skill metadata migration manifests for already-copied live folders.
|
||||
|
||||
The manifest deliberately contains no agent or skill payload files. Operators
|
||||
copy those directories with their normal offline transfer process; these
|
||||
helpers only prove that the target runtime folders match the exported inventory
|
||||
before database/configuration metadata is reconciled.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
ResourceStatus = Literal["matched", "missing", "hash_mismatch", "identity_mismatch", "invalid_path"]
|
||||
OwnerStatus = Literal["matched", "unresolved"]
|
||||
|
||||
_TREE_HASH_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
_TREE_HASH_MAX_WORKERS = 4
|
||||
|
||||
|
||||
def _safe_relative_path(value: str) -> str:
|
||||
normalized = value.strip()
|
||||
if not normalized or "\\" in normalized or ":" in normalized:
|
||||
raise ValueError("relative_dir must be a non-empty portable relative path")
|
||||
path = PurePosixPath(normalized)
|
||||
if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
|
||||
raise ValueError("relative_dir must stay inside the configured runtime root")
|
||||
return path.as_posix()
|
||||
|
||||
|
||||
def compute_tree_hash(root: Path) -> str:
|
||||
"""Return a deterministic SHA-256 for a directory tree.
|
||||
|
||||
Relative paths and bytes are both signed. Symlinks are rejected so a
|
||||
manifest cannot attest content outside the selected runtime directory.
|
||||
"""
|
||||
|
||||
root = root.resolve()
|
||||
if not root.is_dir():
|
||||
raise FileNotFoundError(root)
|
||||
digest = hashlib.sha256()
|
||||
files = sorted((path for path in root.rglob("*") if path.is_file() or path.is_symlink()), key=lambda p: p.relative_to(root).as_posix())
|
||||
for path in files:
|
||||
if path.is_symlink():
|
||||
raise ValueError(f"Symlinks are not supported in migration trees: {path}")
|
||||
relative = path.relative_to(root).as_posix().encode("utf-8")
|
||||
digest.update(len(relative).to_bytes(4, "big"))
|
||||
digest.update(relative)
|
||||
digest.update(path.stat().st_size.to_bytes(8, "big"))
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
class OwnerReference(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
source_user_id: str | None = None
|
||||
email: str | None = None
|
||||
|
||||
@field_validator("source_user_id", "email")
|
||||
@classmethod
|
||||
def _strip_optional(cls, value: str | None) -> str | None:
|
||||
return value.strip() if value and value.strip() else None
|
||||
|
||||
|
||||
class TagManifestItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str = Field(min_length=1, max_length=64)
|
||||
name: str = Field(min_length=1, max_length=128)
|
||||
color: str = Field(default="gray", max_length=32)
|
||||
description: str = Field(default="", max_length=1024)
|
||||
|
||||
|
||||
class AgentManifestItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
relative_dir: str
|
||||
tree_hash: str
|
||||
name: str
|
||||
description: str = ""
|
||||
owner: OwnerReference | None = None
|
||||
published: bool = False
|
||||
is_private: bool = False
|
||||
square_id: str = ""
|
||||
featured_order: int | None = None
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
skills: list[str] = Field(default_factory=list, max_length=5000)
|
||||
tags: list[TagManifestItem] = Field(default_factory=list, max_length=200)
|
||||
|
||||
@field_validator("relative_dir")
|
||||
@classmethod
|
||||
def _relative_dir(cls, value: str) -> str:
|
||||
return _safe_relative_path(value)
|
||||
|
||||
@field_validator("tree_hash")
|
||||
@classmethod
|
||||
def _tree_hash(cls, value: str) -> str:
|
||||
normalized = value.lower()
|
||||
if not _TREE_HASH_RE.fullmatch(normalized):
|
||||
raise ValueError("tree_hash must be a SHA-256 hex digest")
|
||||
return normalized
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _directory_matches_id(self):
|
||||
if self.relative_dir != self.id.lower():
|
||||
raise ValueError("agent relative_dir must equal the stable agent id")
|
||||
return self
|
||||
|
||||
|
||||
class SkillManifestItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str
|
||||
category: Literal["public", "custom"]
|
||||
relative_dir: str
|
||||
tree_hash: str
|
||||
owner: OwnerReference | None = None
|
||||
published: bool = False
|
||||
detail: str | None = None
|
||||
name_zh: str | None = None
|
||||
always_on: bool = False
|
||||
square_id: str = ""
|
||||
enabled: bool = True
|
||||
state: Literal["active", "stale", "archived"] = "active"
|
||||
pinned: bool = False
|
||||
featured: bool = False
|
||||
source: Literal["agent", "upload"] = "upload"
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
tags: list[TagManifestItem] = Field(default_factory=list, max_length=200)
|
||||
|
||||
@field_validator("relative_dir")
|
||||
@classmethod
|
||||
def _relative_dir(cls, value: str) -> str:
|
||||
normalized = _safe_relative_path(value)
|
||||
if PurePosixPath(normalized).parts[0] not in {"public", "custom"}:
|
||||
raise ValueError("skill relative_dir must begin with public/ or custom/")
|
||||
return normalized
|
||||
|
||||
@field_validator("tree_hash")
|
||||
@classmethod
|
||||
def _tree_hash(cls, value: str) -> str:
|
||||
normalized = value.lower()
|
||||
if not _TREE_HASH_RE.fullmatch(normalized):
|
||||
raise ValueError("tree_hash must be a SHA-256 hex digest")
|
||||
return normalized
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _category_matches_directory(self):
|
||||
if PurePosixPath(self.relative_dir).parts[0] != self.category:
|
||||
raise ValueError("skill category must match relative_dir")
|
||||
return self
|
||||
|
||||
|
||||
class SupportDirectoryManifestItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
relative_dir: str
|
||||
tree_hash: str
|
||||
required_by: list[str] = Field(default_factory=list, max_length=5000)
|
||||
|
||||
@field_validator("relative_dir")
|
||||
@classmethod
|
||||
def _relative_dir(cls, value: str) -> str:
|
||||
normalized = _safe_relative_path(value)
|
||||
if PurePosixPath(normalized).parts[0] not in {"public", "custom"}:
|
||||
raise ValueError("support relative_dir must begin with public/ or custom/")
|
||||
return normalized
|
||||
|
||||
@field_validator("tree_hash")
|
||||
@classmethod
|
||||
def _tree_hash(cls, value: str) -> str:
|
||||
normalized = value.lower()
|
||||
if not _TREE_HASH_RE.fullmatch(normalized):
|
||||
raise ValueError("tree_hash must be a SHA-256 hex digest")
|
||||
return normalized
|
||||
|
||||
|
||||
class MigrationManifest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
schema_version: Literal[1] = 1
|
||||
bundle_id: str = Field(min_length=1, max_length=128)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
agents: list[AgentManifestItem] = Field(default_factory=list, max_length=1000)
|
||||
skills: list[SkillManifestItem] = Field(default_factory=list, max_length=5000)
|
||||
support_directories: list[SupportDirectoryManifestItem] = Field(default_factory=list, max_length=1000)
|
||||
warnings: list[str] = Field(default_factory=list, max_length=1000)
|
||||
|
||||
|
||||
class ResourcePreflightItem(BaseModel):
|
||||
kind: Literal["agent", "skill", "support"]
|
||||
key: str
|
||||
relative_dir: str
|
||||
status: ResourceStatus
|
||||
message: str = ""
|
||||
|
||||
|
||||
class OwnerMappingItem(BaseModel):
|
||||
source_user_id: str | None = None
|
||||
email: str | None = None
|
||||
target_user_id: str | None = None
|
||||
status: OwnerStatus
|
||||
|
||||
|
||||
class MigrationPreflight(BaseModel):
|
||||
ready: bool
|
||||
agents: list[ResourcePreflightItem]
|
||||
skills: list[ResourcePreflightItem]
|
||||
support_directories: list[ResourcePreflightItem]
|
||||
owner_mappings: list[OwnerMappingItem]
|
||||
|
||||
|
||||
class MigrationApplyResult(BaseModel):
|
||||
bundle_id: str
|
||||
agents_applied: int
|
||||
skills_applied: int
|
||||
skills_skipped: int = 0
|
||||
skipped_skill_names: list[str] = Field(default_factory=list)
|
||||
tags_created: int
|
||||
fallback_owner_count: int
|
||||
|
||||
|
||||
def _resolved_child(root: Path, relative_dir: str) -> Path | None:
|
||||
root = root.resolve()
|
||||
child = root
|
||||
for part in PurePosixPath(relative_dir).parts:
|
||||
child /= part
|
||||
if child.is_symlink():
|
||||
return None
|
||||
child = child.resolve()
|
||||
try:
|
||||
child.relative_to(root)
|
||||
except ValueError:
|
||||
return None
|
||||
return child
|
||||
|
||||
|
||||
def _agent_identity(directory: Path) -> str | None:
|
||||
config_path = directory / "config.yaml"
|
||||
if not config_path.is_file():
|
||||
return None
|
||||
try:
|
||||
payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
|
||||
except (OSError, UnicodeError, yaml.YAMLError):
|
||||
return None
|
||||
return str(payload.get("id") or "").strip() or None if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
def _skill_identity(directory: Path) -> str | None:
|
||||
skill_path = directory / "SKILL.md"
|
||||
if not skill_path.is_file():
|
||||
return None
|
||||
try:
|
||||
text = skill_path.read_text(encoding="utf-8")
|
||||
if not text.startswith("---"):
|
||||
return None
|
||||
parts = text.split("---", 2)
|
||||
payload = yaml.safe_load(parts[1]) or {}
|
||||
except (OSError, UnicodeError, yaml.YAMLError, IndexError):
|
||||
return None
|
||||
return str(payload.get("name") or "").strip() or None if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
def _resource_preflight(
|
||||
*,
|
||||
kind: Literal["agent", "skill", "support"],
|
||||
key: str,
|
||||
root: Path,
|
||||
relative_dir: str,
|
||||
expected_hash: str,
|
||||
expected_identity: str | None = None,
|
||||
) -> ResourcePreflightItem:
|
||||
directory = _resolved_child(root, relative_dir)
|
||||
if directory is None:
|
||||
return ResourcePreflightItem(
|
||||
kind=kind,
|
||||
key=key,
|
||||
relative_dir=relative_dir,
|
||||
status="invalid_path",
|
||||
message="目录路径不安全、包含符号链接或超出运行根目录",
|
||||
)
|
||||
if not directory.is_dir():
|
||||
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="missing", message="目标运行目录中未找到该目录")
|
||||
if expected_identity is not None:
|
||||
identity_file = directory / ("config.yaml" if kind == "agent" else "SKILL.md")
|
||||
if not identity_file.is_file():
|
||||
return ResourcePreflightItem(
|
||||
kind=kind,
|
||||
key=key,
|
||||
relative_dir=relative_dir,
|
||||
status="missing",
|
||||
message=f"目录中缺少 {identity_file.name}",
|
||||
)
|
||||
try:
|
||||
actual_hash = compute_tree_hash(directory)
|
||||
except (OSError, ValueError) as exc:
|
||||
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="hash_mismatch", message=str(exc))
|
||||
if actual_hash != expected_hash:
|
||||
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="hash_mismatch", message="目录内容与导出清单不一致")
|
||||
if expected_identity is not None:
|
||||
actual_identity = _agent_identity(directory) if kind == "agent" else _skill_identity(directory)
|
||||
if actual_identity != expected_identity:
|
||||
return ResourcePreflightItem(
|
||||
kind=kind,
|
||||
key=key,
|
||||
relative_dir=relative_dir,
|
||||
status="identity_mismatch",
|
||||
message=f"目录内标识为 {actual_identity or '空'},清单标识为 {expected_identity}",
|
||||
)
|
||||
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="matched")
|
||||
|
||||
|
||||
def preflight_manifest(
|
||||
manifest: MigrationManifest,
|
||||
*,
|
||||
agents_root: Path,
|
||||
skills_root: Path,
|
||||
users_by_email: dict[str, str],
|
||||
) -> MigrationPreflight:
|
||||
"""Compare a manifest with target live roots without modifying anything."""
|
||||
|
||||
normalized_users = {email.strip().casefold(): user_id for email, user_id in users_by_email.items()}
|
||||
resource_specs: list[dict[str, Any]] = [
|
||||
{
|
||||
"kind": "agent",
|
||||
"key": item.id,
|
||||
"root": agents_root,
|
||||
"relative_dir": item.relative_dir,
|
||||
"expected_hash": item.tree_hash,
|
||||
"expected_identity": item.id,
|
||||
}
|
||||
for item in manifest.agents
|
||||
]
|
||||
resource_specs.extend(
|
||||
{
|
||||
"kind": "skill",
|
||||
"key": f"{item.category}:{item.name}",
|
||||
"root": skills_root,
|
||||
"relative_dir": item.relative_dir,
|
||||
"expected_hash": item.tree_hash,
|
||||
"expected_identity": item.name,
|
||||
}
|
||||
for item in manifest.skills
|
||||
)
|
||||
resource_specs.extend(
|
||||
{
|
||||
"kind": "support",
|
||||
"key": item.relative_dir,
|
||||
"root": skills_root,
|
||||
"relative_dir": item.relative_dir,
|
||||
"expected_hash": item.tree_hash,
|
||||
}
|
||||
for item in manifest.support_directories
|
||||
)
|
||||
|
||||
def _check_resource(spec: dict[str, Any]) -> ResourcePreflightItem:
|
||||
return _resource_preflight(**spec)
|
||||
|
||||
if len(resource_specs) > 1:
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=min(_TREE_HASH_MAX_WORKERS, len(resource_specs)),
|
||||
thread_name_prefix="skill-migration-hash",
|
||||
) as executor:
|
||||
resources = list(executor.map(_check_resource, resource_specs))
|
||||
else:
|
||||
resources = [_check_resource(spec) for spec in resource_specs]
|
||||
|
||||
agent_count = len(manifest.agents)
|
||||
skill_count = len(manifest.skills)
|
||||
agents = resources[:agent_count]
|
||||
skills = resources[agent_count : agent_count + skill_count]
|
||||
support = resources[agent_count + skill_count :]
|
||||
|
||||
owners: dict[tuple[str | None, str | None], OwnerMappingItem] = {}
|
||||
for resource in [*manifest.agents, *manifest.skills]:
|
||||
owner = resource.owner
|
||||
if owner is None or (owner.source_user_id is None and owner.email is None):
|
||||
continue
|
||||
email = owner.email.strip().casefold() if owner.email else None
|
||||
target_user_id = normalized_users.get(email) if email else None
|
||||
key = (owner.source_user_id, email)
|
||||
owners[key] = OwnerMappingItem(
|
||||
source_user_id=owner.source_user_id,
|
||||
email=email,
|
||||
target_user_id=target_user_id,
|
||||
status="matched" if target_user_id else "unresolved",
|
||||
)
|
||||
|
||||
owner_mappings = list(owners.values())
|
||||
ready = all(item.status == "matched" for item in [*agents, *skills, *support]) and all(item.status == "matched" for item in owner_mappings)
|
||||
return MigrationPreflight(
|
||||
ready=ready,
|
||||
agents=agents,
|
||||
skills=skills,
|
||||
support_directories=support,
|
||||
owner_mappings=owner_mappings,
|
||||
)
|
||||
|
||||
|
||||
def prepare_manifest_for_apply(
|
||||
manifest: MigrationManifest,
|
||||
preflight: MigrationPreflight,
|
||||
*,
|
||||
skip_invalid_skills: bool,
|
||||
) -> tuple[MigrationManifest, MigrationPreflight, list[str]]:
|
||||
"""Optionally remove invalid skills and support-directory dependants.
|
||||
|
||||
Skipping is intentionally limited to skill-only manifests. Agent manifests
|
||||
retain all-or-nothing semantics because their live ``config.yaml`` is the
|
||||
authoritative source for several related database caches.
|
||||
"""
|
||||
|
||||
if not skip_invalid_skills:
|
||||
return manifest, preflight, []
|
||||
if manifest.agents:
|
||||
raise ValueError("跳过异常条目仅支持独立技能清单")
|
||||
|
||||
failed_skill_dirs = {
|
||||
item.relative_dir for item in preflight.skills if item.status != "matched"
|
||||
}
|
||||
skipped_names = {
|
||||
item.name for item in manifest.skills if item.relative_dir in failed_skill_dirs
|
||||
}
|
||||
support_by_dir = {
|
||||
item.relative_dir: item for item in manifest.support_directories
|
||||
}
|
||||
for resource in preflight.support_directories:
|
||||
if resource.status == "matched":
|
||||
continue
|
||||
support = support_by_dir.get(resource.relative_dir)
|
||||
if support is not None:
|
||||
skipped_names.update(support.required_by)
|
||||
|
||||
ordered_skipped = [item.name for item in manifest.skills if item.name in skipped_names]
|
||||
kept_skills = [item for item in manifest.skills if item.name not in skipped_names]
|
||||
kept_names = {item.name for item in kept_skills}
|
||||
kept_support = [
|
||||
item
|
||||
for item in manifest.support_directories
|
||||
if any(name in kept_names for name in item.required_by)
|
||||
]
|
||||
kept_skill_dirs = {item.relative_dir for item in kept_skills}
|
||||
kept_support_dirs = {item.relative_dir for item in kept_support}
|
||||
|
||||
owner_keys = {
|
||||
(
|
||||
item.owner.source_user_id,
|
||||
item.owner.email.strip().casefold() if item.owner and item.owner.email else None,
|
||||
)
|
||||
for item in kept_skills
|
||||
if item.owner is not None
|
||||
and (item.owner.source_user_id is not None or item.owner.email is not None)
|
||||
}
|
||||
owner_mappings = [
|
||||
item
|
||||
for item in preflight.owner_mappings
|
||||
if (
|
||||
item.source_user_id,
|
||||
item.email.strip().casefold() if item.email else None,
|
||||
)
|
||||
in owner_keys
|
||||
]
|
||||
skill_resources = [
|
||||
item for item in preflight.skills if item.relative_dir in kept_skill_dirs
|
||||
]
|
||||
support_resources = [
|
||||
item
|
||||
for item in preflight.support_directories
|
||||
if item.relative_dir in kept_support_dirs
|
||||
]
|
||||
ready = all(
|
||||
item.status == "matched"
|
||||
for item in [*preflight.agents, *skill_resources, *support_resources]
|
||||
) and all(item.status == "matched" for item in owner_mappings)
|
||||
|
||||
effective_manifest = manifest.model_copy(
|
||||
update={
|
||||
"skills": kept_skills,
|
||||
"support_directories": kept_support,
|
||||
}
|
||||
)
|
||||
effective_preflight = preflight.model_copy(
|
||||
update={
|
||||
"ready": ready,
|
||||
"skills": skill_resources,
|
||||
"support_directories": support_resources,
|
||||
"owner_mappings": owner_mappings,
|
||||
}
|
||||
)
|
||||
return effective_manifest, effective_preflight, ordered_skipped
|
||||
|
||||
|
||||
def _owner_reference(user_id: str | None, users_by_id: dict[str, str]) -> OwnerReference | None:
|
||||
if user_id is None:
|
||||
return None
|
||||
return OwnerReference(source_user_id=user_id, email=users_by_id.get(user_id))
|
||||
|
||||
|
||||
def _load_agent_config(directory: Path) -> dict[str, Any]:
|
||||
try:
|
||||
payload = yaml.safe_load((directory / "config.yaml").read_text(encoding="utf-8")) or {}
|
||||
except (OSError, UnicodeError, yaml.YAMLError) as exc:
|
||||
raise ValueError(f"Cannot read agent config: {directory}") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"Agent config must be an object: {directory}")
|
||||
return payload
|
||||
|
||||
|
||||
def _manifest_tags(rows: list[dict[str, Any]]) -> list[TagManifestItem]:
|
||||
return [
|
||||
TagManifestItem(
|
||||
id=str(row.get("id") or ""),
|
||||
name=str(row.get("name") or ""),
|
||||
color=str(row.get("color") or "gray"),
|
||||
description=str(row.get("description") or ""),
|
||||
)
|
||||
for row in rows
|
||||
if row.get("id") and row.get("name")
|
||||
]
|
||||
|
||||
|
||||
async def build_agent_migration_manifest(
|
||||
agent_ids: list[str],
|
||||
*,
|
||||
agent_store: Any,
|
||||
tag_store: Any,
|
||||
agents_root: Path,
|
||||
users_by_id: dict[str, str],
|
||||
) -> MigrationManifest:
|
||||
"""Build an agent manifest without inspecting or exporting skill payloads."""
|
||||
|
||||
selected_ids = list(dict.fromkeys(agent_id.strip() for agent_id in agent_ids if agent_id.strip()))
|
||||
if not selected_ids:
|
||||
raise ValueError("Select at least one agent")
|
||||
rows_by_id = {str(row["id"]): row for row in await agent_store.list_all()}
|
||||
missing_agents = [agent_id for agent_id in selected_ids if agent_id not in rows_by_id]
|
||||
if missing_agents:
|
||||
raise ValueError(f"Unknown agents: {', '.join(missing_agents)}")
|
||||
|
||||
agent_tag_map = await tag_store.list_assignments("agent", selected_ids)
|
||||
agents: list[AgentManifestItem] = []
|
||||
warnings: list[str] = []
|
||||
for agent_id in selected_ids:
|
||||
row = rows_by_id[agent_id]
|
||||
directory = _resolved_child(agents_root, agent_id.lower())
|
||||
if directory is None or not directory.is_dir():
|
||||
raise ValueError(f"Agent directory is missing: {agent_id}")
|
||||
config = _load_agent_config(directory)
|
||||
if str(config.get("id") or "") != agent_id:
|
||||
raise ValueError(f"Agent config id does not match directory: {agent_id}")
|
||||
raw_skills = config.get("skills")
|
||||
selected_skills = list(dict.fromkeys(item.strip() for item in raw_skills if isinstance(item, str) and item.strip())) if isinstance(raw_skills, list) else []
|
||||
knowledge_ids = config.get("llmwiki_knowledge_base_ids")
|
||||
if isinstance(knowledge_ids, list) and any(str(item).strip() for item in knowledge_ids):
|
||||
warnings.append(f"智能体 {agent_id} 绑定了知识库;知识库数据不在本迁移清单范围内,请在目标机器单独确认")
|
||||
agents.append(
|
||||
AgentManifestItem(
|
||||
id=agent_id,
|
||||
relative_dir=directory.relative_to(agents_root.resolve()).as_posix(),
|
||||
tree_hash=await asyncio.to_thread(compute_tree_hash, directory),
|
||||
name=str(row.get("name") or config.get("name") or agent_id),
|
||||
description=str(row.get("description") or ""),
|
||||
owner=_owner_reference(row.get("user_id"), users_by_id),
|
||||
published=bool(row.get("published")),
|
||||
is_private=bool(row.get("is_private")),
|
||||
square_id=str(row.get("square_id") or ""),
|
||||
featured_order=row.get("featured_order"),
|
||||
created_at=row.get("created_at"),
|
||||
updated_at=row.get("updated_at"),
|
||||
skills=selected_skills,
|
||||
tags=_manifest_tags(agent_tag_map.get(agent_id, [])),
|
||||
)
|
||||
)
|
||||
|
||||
return MigrationManifest(
|
||||
bundle_id=uuid4().hex,
|
||||
agents=agents,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
async def build_skill_migration_manifest(
|
||||
skill_names: list[str],
|
||||
*,
|
||||
skill_store: Any,
|
||||
tag_store: Any,
|
||||
skills_root: Path,
|
||||
discovered_skills: list[Any],
|
||||
users_by_id: dict[str, str],
|
||||
usage_entries: dict[str, dict[str, Any]],
|
||||
) -> MigrationManifest:
|
||||
"""Build a metadata-only manifest for explicitly selected live skills."""
|
||||
|
||||
selected_names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
|
||||
if not selected_names:
|
||||
raise ValueError("Select at least one skill")
|
||||
|
||||
skill_rows = {str(row["name"]): row for row in await skill_store.list_all()}
|
||||
discovered = {str(skill.name): skill for skill in discovered_skills}
|
||||
missing_skills = [name for name in selected_names if name not in discovered]
|
||||
if missing_skills:
|
||||
raise ValueError(f"Skills are missing from the live runtime directory: {', '.join(missing_skills)}")
|
||||
|
||||
skill_tag_map = await tag_store.list_assignments("skill", selected_names)
|
||||
skill_directories = [Path(discovered[name].skill_dir).resolve() for name in selected_names]
|
||||
tree_hashes = await _compute_tree_hashes(skill_directories)
|
||||
skills: list[SkillManifestItem] = []
|
||||
for name, tree_hash in zip(selected_names, tree_hashes, strict=True):
|
||||
skill = discovered[name]
|
||||
category = str(skill.category)
|
||||
relative = PurePosixPath(category) / PurePosixPath(Path(skill.relative_path).as_posix())
|
||||
directory = Path(skill.skill_dir).resolve()
|
||||
row = skill_rows.get(name, {})
|
||||
usage = usage_entries.get(name, {})
|
||||
state = str(usage.get("state") or "active")
|
||||
if state not in {"active", "stale", "archived"}:
|
||||
state = "active"
|
||||
source = str(usage.get("source") or "upload")
|
||||
if source not in {"agent", "upload"}:
|
||||
source = "upload"
|
||||
skills.append(
|
||||
SkillManifestItem(
|
||||
name=name,
|
||||
category=category,
|
||||
relative_dir=relative.as_posix(),
|
||||
tree_hash=tree_hash,
|
||||
owner=_owner_reference(row.get("owner_user_id"), users_by_id),
|
||||
published=bool(row.get("published", category == "public")),
|
||||
detail=row.get("detail"),
|
||||
name_zh=row.get("name_zh"),
|
||||
always_on=bool(row.get("always_on", getattr(skill, "always_on", False))),
|
||||
square_id=str(row.get("square_id") or ""),
|
||||
enabled=bool(skill.enabled),
|
||||
state=state,
|
||||
pinned=bool(usage.get("pinned")),
|
||||
featured=bool(usage.get("featured")),
|
||||
source=source,
|
||||
created_at=row.get("created_at") or usage.get("created_at"),
|
||||
updated_at=row.get("updated_at"),
|
||||
tags=_manifest_tags(skill_tag_map.get(name, [])),
|
||||
)
|
||||
)
|
||||
|
||||
selected_set = set(selected_names)
|
||||
warnings: list[str] = []
|
||||
support_directories: list[SupportDirectoryManifestItem] = []
|
||||
support_candidates, support_warnings = await asyncio.to_thread(
|
||||
_find_support_directories,
|
||||
skills_root,
|
||||
selected_set,
|
||||
)
|
||||
warnings.extend(support_warnings)
|
||||
support_hashes = await _compute_tree_hashes(
|
||||
[directory for directory, _required_by in support_candidates]
|
||||
)
|
||||
for (directory, selected_required_by), tree_hash in zip(
|
||||
support_candidates,
|
||||
support_hashes,
|
||||
strict=True,
|
||||
):
|
||||
support_directories.append(
|
||||
SupportDirectoryManifestItem(
|
||||
relative_dir=directory.relative_to(skills_root.resolve()).as_posix(),
|
||||
tree_hash=tree_hash,
|
||||
required_by=selected_required_by,
|
||||
)
|
||||
)
|
||||
|
||||
return MigrationManifest(
|
||||
bundle_id=uuid4().hex,
|
||||
skills=skills,
|
||||
support_directories=support_directories,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
async def _compute_tree_hashes(directories: list[Path]) -> list[str]:
|
||||
if not directories:
|
||||
return []
|
||||
semaphore = asyncio.Semaphore(min(_TREE_HASH_MAX_WORKERS, len(directories)))
|
||||
|
||||
async def _compute(directory: Path) -> str:
|
||||
async with semaphore:
|
||||
return await asyncio.to_thread(compute_tree_hash, directory)
|
||||
|
||||
return list(await asyncio.gather(*(_compute(directory) for directory in directories)))
|
||||
|
||||
|
||||
def _find_support_directories(
|
||||
skills_root: Path,
|
||||
selected_names: set[str],
|
||||
) -> tuple[list[tuple[Path, list[str]]], list[str]]:
|
||||
candidates: list[tuple[Path, list[str]]] = []
|
||||
warnings: list[str] = []
|
||||
for support_manifest in sorted(skills_root.rglob(".deerflow-skill-support.json")):
|
||||
try:
|
||||
payload = json.loads(support_manifest.read_text(encoding="utf-8"))
|
||||
required_by = [str(item).strip() for item in payload.get("required_by", []) if str(item).strip()]
|
||||
except (OSError, UnicodeError, json.JSONDecodeError, AttributeError):
|
||||
warnings.append(f"无法读取技能支持目录清单:{support_manifest}")
|
||||
continue
|
||||
selected_required_by = [name for name in required_by if name in selected_names]
|
||||
if not selected_required_by:
|
||||
continue
|
||||
candidates.append((support_manifest.parent.resolve(), selected_required_by))
|
||||
return candidates, warnings
|
||||
|
||||
|
||||
def _apply_usage_metadata(name: str, metadata: dict[str, Any]) -> None:
|
||||
_apply_usage_metadata_batch({name: metadata})
|
||||
|
||||
|
||||
def _apply_usage_metadata_batch(entries: dict[str, dict[str, Any]]) -> None:
|
||||
from deerflow.skills.usage import set_migration_metadata_batch
|
||||
|
||||
set_migration_metadata_batch(entries)
|
||||
|
||||
|
||||
def _target_owner_id(
|
||||
owner: OwnerReference | None,
|
||||
*,
|
||||
users_by_email: dict[str, str],
|
||||
fallback_owner_user_id: str | None,
|
||||
) -> tuple[str | None, bool]:
|
||||
if owner is None or (owner.source_user_id is None and owner.email is None):
|
||||
return None, False
|
||||
target = users_by_email.get(owner.email.casefold()) if owner.email else None
|
||||
if target:
|
||||
return target, False
|
||||
if fallback_owner_user_id:
|
||||
return fallback_owner_user_id, True
|
||||
raise ValueError(f"Unresolved owner: {owner.email or owner.source_user_id}")
|
||||
|
||||
|
||||
async def _apply_tags(tag_store: Any, target_type: str, target_id: str, tags: list[TagManifestItem]) -> int:
|
||||
return await _apply_tags_bulk(tag_store, target_type, [(target_id, tags)])
|
||||
|
||||
|
||||
async def _apply_tags_bulk(
|
||||
tag_store: Any,
|
||||
target_type: str,
|
||||
targets: list[tuple[str, list[TagManifestItem]]],
|
||||
) -> int:
|
||||
if not targets:
|
||||
return 0
|
||||
existing = await tag_store.list_tags()
|
||||
by_name = {
|
||||
(str(row.get("tag_type")), str(row.get("name"))): row
|
||||
for row in existing
|
||||
}
|
||||
used_ids = {str(row.get("id")) for row in existing}
|
||||
assignments: dict[str, list[str]] = {}
|
||||
created = 0
|
||||
for target_id, tags in targets:
|
||||
tag_ids: list[str] = []
|
||||
for tag in tags:
|
||||
same_name = by_name.get((target_type, tag.name))
|
||||
if same_name is not None:
|
||||
tag_ids.append(str(same_name["id"]))
|
||||
continue
|
||||
tag_id = tag.id if tag.id not in used_ids else uuid4().hex
|
||||
created_tag = await tag_store.create_tag(
|
||||
{
|
||||
"id": tag_id,
|
||||
"name": tag.name,
|
||||
"tag_type": target_type,
|
||||
"color": tag.color,
|
||||
"description": tag.description,
|
||||
}
|
||||
)
|
||||
by_name[(target_type, tag.name)] = created_tag
|
||||
used_ids.add(tag_id)
|
||||
tag_ids.append(tag_id)
|
||||
created += 1
|
||||
assignments[target_id] = tag_ids
|
||||
await tag_store.set_target_tags_bulk(target_type, assignments)
|
||||
return created
|
||||
|
||||
|
||||
async def apply_migration_manifest(
|
||||
manifest: MigrationManifest,
|
||||
*,
|
||||
preflight: MigrationPreflight,
|
||||
agent_store: Any,
|
||||
skill_store: Any,
|
||||
tag_store: Any,
|
||||
agents_root: Path,
|
||||
users_by_email: dict[str, str],
|
||||
fallback_owner_user_id: str | None,
|
||||
extensions_config_path: Path | None,
|
||||
skip_invalid_skills: bool = False,
|
||||
) -> MigrationApplyResult:
|
||||
"""Idempotently reconcile metadata after all live folders passed preflight."""
|
||||
|
||||
manifest, preflight, skipped_skill_names = prepare_manifest_for_apply(
|
||||
manifest,
|
||||
preflight,
|
||||
skip_invalid_skills=skip_invalid_skills,
|
||||
)
|
||||
|
||||
bad_resources = [item for item in [*preflight.agents, *preflight.skills, *preflight.support_directories] if item.status != "matched"]
|
||||
if bad_resources:
|
||||
raise ValueError("Migration preflight contains missing or mismatched directories")
|
||||
if any(item.status == "unresolved" for item in preflight.owner_mappings) and not fallback_owner_user_id:
|
||||
raise ValueError("Migration preflight contains unresolved owners")
|
||||
|
||||
if manifest.skills:
|
||||
if extensions_config_path is None:
|
||||
raise ValueError("目标运行环境未找到 extensions_config.json")
|
||||
from deerflow.config.extensions_config import patch_extensions_config
|
||||
|
||||
enabled_by_name = {item.name: item.enabled for item in manifest.skills}
|
||||
|
||||
def _patch(document: dict[str, Any]) -> None:
|
||||
skills_document = document.setdefault("skills", {})
|
||||
if not isinstance(skills_document, dict):
|
||||
raise ValueError("extensions_config.json skills must be an object")
|
||||
for name, enabled in enabled_by_name.items():
|
||||
current = skills_document.get(name)
|
||||
entry = dict(current) if isinstance(current, dict) else {}
|
||||
entry["enabled"] = enabled
|
||||
skills_document[name] = entry
|
||||
|
||||
patch_extensions_config(extensions_config_path, _patch)
|
||||
|
||||
normalized_users = {email.strip().casefold(): user_id for email, user_id in users_by_email.items()}
|
||||
fallback_count = 0
|
||||
tags_created = 0
|
||||
skill_payloads: list[dict[str, Any]] = []
|
||||
skill_usage_entries: dict[str, dict[str, Any]] = {}
|
||||
for item in manifest.skills:
|
||||
owner_user_id, used_fallback = _target_owner_id(
|
||||
item.owner,
|
||||
users_by_email=normalized_users,
|
||||
fallback_owner_user_id=fallback_owner_user_id,
|
||||
)
|
||||
fallback_count += int(used_fallback)
|
||||
skill_payloads.append(
|
||||
{
|
||||
"name": item.name,
|
||||
"owner_user_id": owner_user_id,
|
||||
"published": item.published,
|
||||
"detail": item.detail,
|
||||
"name_zh": item.name_zh,
|
||||
"always_on": item.always_on,
|
||||
"square_id": item.square_id,
|
||||
"created_at": item.created_at,
|
||||
"updated_at": item.updated_at,
|
||||
}
|
||||
)
|
||||
skill_usage_entries[item.name] = {
|
||||
"state": item.state,
|
||||
"pinned": item.pinned,
|
||||
"featured": item.featured,
|
||||
"source": item.source,
|
||||
}
|
||||
|
||||
if skill_payloads:
|
||||
await skill_store.migration_upsert_many(skill_payloads)
|
||||
tags_created += await _apply_tags_bulk(
|
||||
tag_store,
|
||||
"skill",
|
||||
[(item.name, item.tags) for item in manifest.skills],
|
||||
)
|
||||
_apply_usage_metadata_batch(skill_usage_entries)
|
||||
|
||||
agent_tag_targets: list[tuple[str, list[TagManifestItem]]] = []
|
||||
for item in manifest.agents:
|
||||
owner_user_id, used_fallback = _target_owner_id(
|
||||
item.owner,
|
||||
users_by_email=normalized_users,
|
||||
fallback_owner_user_id=fallback_owner_user_id,
|
||||
)
|
||||
fallback_count += int(used_fallback)
|
||||
await agent_store.migration_upsert(
|
||||
{
|
||||
"id": item.id,
|
||||
"name": item.name,
|
||||
"description": item.description,
|
||||
"user_id": owner_user_id,
|
||||
"published": item.published,
|
||||
"is_private": item.is_private,
|
||||
"square_id": item.square_id,
|
||||
"featured_order": item.featured_order,
|
||||
"created_at": item.created_at,
|
||||
"updated_at": item.updated_at,
|
||||
}
|
||||
)
|
||||
config = _load_agent_config(agents_root / item.relative_dir)
|
||||
await agent_store.replace_skills(item.id, config.get("skills") if isinstance(config.get("skills"), list) else [])
|
||||
await agent_store.set_extras(
|
||||
item.id,
|
||||
tool_groups=config.get("tool_groups") if isinstance(config.get("tool_groups"), list) else None,
|
||||
model=str(config.get("model")) if config.get("model") else None,
|
||||
)
|
||||
agent_tag_targets.append((item.id, item.tags))
|
||||
|
||||
tags_created += await _apply_tags_bulk(tag_store, "agent", agent_tag_targets)
|
||||
|
||||
return MigrationApplyResult(
|
||||
bundle_id=manifest.bundle_id,
|
||||
agents_applied=len(manifest.agents),
|
||||
skills_applied=len(manifest.skills),
|
||||
skills_skipped=len(skipped_skill_names),
|
||||
skipped_skill_names=skipped_skill_names,
|
||||
tags_created=tags_created,
|
||||
fallback_owner_count=fallback_count,
|
||||
)
|
||||
@@ -37,6 +37,7 @@ from app.gateway.routers import (
|
||||
admin_active_runs,
|
||||
admin_audit_retention as admin_audit_retention_routes,
|
||||
admin_users,
|
||||
agent_skill_migrations,
|
||||
agents,
|
||||
ai_writing,
|
||||
app_identity,
|
||||
@@ -48,6 +49,7 @@ from app.gateway.routers import (
|
||||
browser_context,
|
||||
business_mapping,
|
||||
channels,
|
||||
chat_history_transfer,
|
||||
chat_archive,
|
||||
checkpoint_migration,
|
||||
compact_menu_overrides,
|
||||
@@ -1204,6 +1206,30 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
logger.exception(error_msg)
|
||||
raise RuntimeError(error_msg) from e
|
||||
|
||||
# Populate product-owned public skills before any ownership migration,
|
||||
# MySQL catalogue projection, prompt cache, or sandbox view observes the
|
||||
# configured live root. This application-level guard is intentionally in
|
||||
# addition to the supported shell launchers: image entrypoints, Compose,
|
||||
# and external release wrappers must receive the same built-ins.
|
||||
try:
|
||||
from app.builtin_skill_sync import ensure_bundled_public_skills
|
||||
|
||||
sync_result = await asyncio.to_thread(
|
||||
ensure_bundled_public_skills,
|
||||
app.state.config,
|
||||
)
|
||||
logger.info(
|
||||
"Bundled public skills synchronized: source=%s target=%s copied=%d required=%d",
|
||||
sync_result["source"],
|
||||
sync_result["target"],
|
||||
len(sync_result["copiedDirectories"]),
|
||||
len(sync_result["requiredSkills"]),
|
||||
)
|
||||
except Exception as e:
|
||||
error_msg = f"Failed to synchronize required built-in skills during gateway startup: {e}"
|
||||
logger.exception(error_msg)
|
||||
raise RuntimeError(error_msg) from e
|
||||
|
||||
config = get_gateway_config()
|
||||
logger.info(f"Starting API Gateway on {config.host}:{config.port}")
|
||||
|
||||
@@ -1276,6 +1302,20 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Skill MySQL catalog projection sync failed (non-fatal)")
|
||||
|
||||
# Project the compact per-skill catalog metadata into MySQL before the
|
||||
# Gateway accepts requests. List APIs read indexed scalar columns and
|
||||
# never load skill markdown, display samples, or usage sidecars.
|
||||
try:
|
||||
from app.gateway.skill_catalog_cache import refresh_skill_catalog_projection
|
||||
|
||||
catalog_size = await refresh_skill_catalog_projection(
|
||||
app.state.config,
|
||||
skills=catalog_skills,
|
||||
)
|
||||
logger.info("Skill MySQL catalog projection synchronized: skills=%d", catalog_size)
|
||||
except Exception:
|
||||
logger.exception("Skill MySQL catalog projection sync failed (non-fatal)")
|
||||
|
||||
# ????? agent(intent / recommender / coordinator)???????
|
||||
# ??? _sync_legacy_agents **??**??????????????????
|
||||
# ??? upsert ? agents ?,?????? /api/agents ?????,
|
||||
@@ -1462,6 +1502,32 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
logger.exception(
|
||||
"Elasticsearch offline operations agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# 内置「聊天记录迁移助手」:高斯/SQLite 会话索引、检查点和线程文件分开迁移。
|
||||
# 目录已存在时不覆盖管理员修改。
|
||||
try:
|
||||
from app.gateway.routers._chat_history_migration_seed import (
|
||||
ensure_chat_history_migration_agent,
|
||||
)
|
||||
|
||||
ensure_chat_history_migration_agent()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Chat-history migration agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# 内置「高斯数据适配助手」:根据报错说明要改的配置。目录已存在时不覆盖。
|
||||
try:
|
||||
from app.gateway.routers._gauss_data_adaptation_seed import (
|
||||
ensure_gauss_data_adaptation_agent,
|
||||
)
|
||||
|
||||
ensure_gauss_data_adaptation_agent()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Gauss data-adaptation agent seeding failed at startup (non-fatal)"
|
||||
)
|
||||
|
||||
# 内置「任务研判报告助手」(agentfx 任务深链固定使用):同样必须在
|
||||
# _sync_legacy_agents 之前补建目录 + 补建配套 task-report-import 技能
|
||||
# (skills/ 被 .gitignore 排除),由后者 upsert 进 agents 表。
|
||||
@@ -1587,6 +1653,13 @@ async def _gateway_lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
except Exception:
|
||||
logger.exception("Admin audit retention scheduler failed to start (non-fatal)")
|
||||
|
||||
try:
|
||||
app.state.admin_audit_retention_task = asyncio.create_task(
|
||||
_admin_audit_retention_loop(app)
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Admin audit retention scheduler failed to start (non-fatal)")
|
||||
|
||||
# Start IM channel service if any channels are configured
|
||||
try:
|
||||
from app.channels.service import start_channel_service
|
||||
@@ -1765,6 +1838,25 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
await close_api_read_cache()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
from app.gateway.worker_pool.lifecycle import managed_controller
|
||||
from app.gateway.perf_logging import event_loop_watchdog, register_stack_dump_signal
|
||||
|
||||
register_stack_dump_signal()
|
||||
watchdog = asyncio.create_task(event_loop_watchdog(app), name="gateway-event-loop-watchdog")
|
||||
try:
|
||||
async with managed_controller(app):
|
||||
async with _gateway_lifespan(app):
|
||||
yield
|
||||
finally:
|
||||
watchdog.cancel()
|
||||
await asyncio.gather(watchdog, return_exceptions=True)
|
||||
from app.gateway.api_read_cache import close_api_read_cache
|
||||
|
||||
await close_api_read_cache()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""Create and configure the FastAPI application.
|
||||
|
||||
@@ -2097,11 +2189,14 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
|
||||
app.include_router(analysis_projects.router)
|
||||
app.include_router(threads.router)
|
||||
app.include_router(chat_history_transfer.router)
|
||||
# Dedicated taskId + goPath history lookup for single-agent task deep-link chats.
|
||||
app.include_router(task_deeplink_threads.router)
|
||||
app.include_router(task_deeplink_details.router)
|
||||
# Agents API is mounted at /api/agents
|
||||
app.include_router(agents.router)
|
||||
# Metadata-only migration for agent/skill folders copied into live roots.
|
||||
app.include_router(agent_skill_migrations.router)
|
||||
app.include_router(strategy_agents.router)
|
||||
|
||||
# Recommended Questions API is mounted at /api/recommended-questions
|
||||
|
||||
@@ -1,401 +1,410 @@
|
||||
"""Authorization decorators and context for DeerFlow.
|
||||
|
||||
Inspired by LangGraph Auth system: https://github.com/langchain-ai/langgraph/blob/main/libs/sdk-py/langgraph_sdk/auth/__init__.py
|
||||
|
||||
**Usage:**
|
||||
|
||||
1. Use ``@require_auth`` on routes that need authentication
|
||||
2. Use ``@require_permission("resource", "action", filter_key=...)`` for permission checks
|
||||
3. The decorator chain processes from bottom to top
|
||||
|
||||
**Example:**
|
||||
|
||||
@router.get("/{thread_id}")
|
||||
@require_auth
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_thread(thread_id: str, request: Request):
|
||||
# User is authenticated and has threads:read permission
|
||||
...
|
||||
|
||||
**Permission Model:**
|
||||
|
||||
- threads:read - View thread
|
||||
- threads:write - Create/update thread
|
||||
- threads:delete - Delete thread
|
||||
- runs:create - Run agent
|
||||
- runs:read - View run
|
||||
- runs:cancel - Cancel run
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import inspect
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.gateway.auth.models import User
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
P = ParamSpec("P")
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
async def _project_for_thread(project_store: Any, thread_id: str) -> dict | None:
|
||||
"""Best-effort project binding lookup; a store error must not fail the read."""
|
||||
try:
|
||||
return await project_store.project_for_thread(thread_id)
|
||||
except Exception:
|
||||
logger.debug("project_for_thread failed for %s", thread_id, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
async def _thread_checkpoint_exists(request: Request, thread_id: str) -> bool:
|
||||
"""Return True if a LangGraph checkpoint exists for ``thread_id``.
|
||||
|
||||
A checkpoint with no matching ``threads_meta`` row is the signature of a
|
||||
DB-switch orphan (sqlite↔mysql/postgres juggling): the conversation state
|
||||
survives in the checkpointer while the business row is gone. Normal thread
|
||||
deletion removes the checkpoint too, so this can never resurrect a
|
||||
deliberately-deleted thread.
|
||||
"""
|
||||
try:
|
||||
from app.gateway.deps import get_checkpointer
|
||||
|
||||
checkpointer = get_checkpointer(request)
|
||||
tup = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
|
||||
return tup is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# Permission constants
|
||||
class Permissions:
|
||||
"""Permission constants for resource:action format."""
|
||||
|
||||
# Threads
|
||||
THREADS_READ = "threads:read"
|
||||
THREADS_WRITE = "threads:write"
|
||||
THREADS_DELETE = "threads:delete"
|
||||
|
||||
# Runs
|
||||
RUNS_CREATE = "runs:create"
|
||||
RUNS_READ = "runs:read"
|
||||
RUNS_CANCEL = "runs:cancel"
|
||||
|
||||
|
||||
class AuthContext:
|
||||
"""Authentication context for the current request.
|
||||
|
||||
Stored in request.state.auth after require_auth decoration.
|
||||
|
||||
Attributes:
|
||||
user: The authenticated user, or None if anonymous
|
||||
permissions: List of permission strings (e.g., "threads:read")
|
||||
"""
|
||||
|
||||
__slots__ = ("user", "permissions")
|
||||
|
||||
def __init__(self, user: User | None = None, permissions: list[str] | None = None):
|
||||
self.user = user
|
||||
self.permissions = permissions or []
|
||||
|
||||
@property
|
||||
def is_authenticated(self) -> bool:
|
||||
"""Check if user is authenticated."""
|
||||
return self.user is not None
|
||||
|
||||
def has_permission(self, resource: str, action: str) -> bool:
|
||||
"""Check if context has permission for resource:action.
|
||||
|
||||
Args:
|
||||
resource: Resource name (e.g., "threads")
|
||||
action: Action name (e.g., "read")
|
||||
|
||||
Returns:
|
||||
True if user has permission
|
||||
"""
|
||||
permission = f"{resource}:{action}"
|
||||
return permission in self.permissions
|
||||
|
||||
def require_user(self) -> User:
|
||||
"""Get user or raise 401.
|
||||
|
||||
Raises:
|
||||
HTTPException 401 if not authenticated
|
||||
"""
|
||||
if not self.user:
|
||||
raise HTTPException(status_code=401, detail="Authentication required")
|
||||
return self.user
|
||||
|
||||
|
||||
def get_auth_context(request: Request) -> AuthContext | None:
|
||||
"""Get AuthContext from request state."""
|
||||
return getattr(request.state, "auth", None)
|
||||
|
||||
|
||||
_ALL_PERMISSIONS: list[str] = [
|
||||
Permissions.THREADS_READ,
|
||||
Permissions.THREADS_WRITE,
|
||||
Permissions.THREADS_DELETE,
|
||||
Permissions.RUNS_CREATE,
|
||||
Permissions.RUNS_READ,
|
||||
Permissions.RUNS_CANCEL,
|
||||
]
|
||||
|
||||
|
||||
def _make_test_request_stub() -> Any:
|
||||
"""Create a minimal request-like object for direct unit calls.
|
||||
|
||||
Used when decorated route handlers are invoked without FastAPI's
|
||||
request injection. Includes fields accessed by auth helpers.
|
||||
"""
|
||||
return SimpleNamespace(state=SimpleNamespace(), cookies={}, _deerflow_test_bypass_auth=True)
|
||||
|
||||
|
||||
async def _authenticate(request: Request) -> AuthContext:
|
||||
"""Authenticate request and return AuthContext.
|
||||
|
||||
Delegates to deps.get_optional_user_from_request() for the JWT→User pipeline.
|
||||
Returns AuthContext with user=None for anonymous requests.
|
||||
"""
|
||||
if is_auth_disabled():
|
||||
user = await get_assumed_user_for_disabled_auth()
|
||||
return AuthContext(user=user, permissions=_ALL_PERMISSIONS)
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return AuthContext(user=None, permissions=[])
|
||||
|
||||
# In future, permissions could be stored in user record
|
||||
return AuthContext(user=user, permissions=_ALL_PERMISSIONS)
|
||||
|
||||
|
||||
def require_auth[**P, T](func: Callable[P, T]) -> Callable[P, T]:
|
||||
"""Decorator that authenticates the request and enforces authentication.
|
||||
|
||||
Independently raises HTTP 401 for unauthenticated requests, regardless of
|
||||
whether ``AuthMiddleware`` is present in the ASGI stack. Sets the resolved
|
||||
``AuthContext`` on ``request.state.auth`` for downstream handlers.
|
||||
|
||||
Must be placed ABOVE other decorators (executes after them).
|
||||
|
||||
Usage:
|
||||
@router.get("/{thread_id}")
|
||||
@require_auth # Bottom decorator (executes first after permission check)
|
||||
@require_permission("threads", "read")
|
||||
async def get_thread(thread_id: str, request: Request):
|
||||
auth: AuthContext = request.state.auth
|
||||
...
|
||||
|
||||
Raises:
|
||||
HTTPException: 401 if the request is unauthenticated.
|
||||
ValueError: If 'request' parameter is missing.
|
||||
"""
|
||||
|
||||
@functools.wraps(func)
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
request = kwargs.get("request")
|
||||
if request is None:
|
||||
# Unit tests may call decorated handlers directly without a
|
||||
# FastAPI Request object. Inject a minimal request stub when
|
||||
# the wrapped function declares `request`.
|
||||
if "request" in inspect.signature(func).parameters:
|
||||
kwargs["request"] = _make_test_request_stub()
|
||||
else:
|
||||
raise ValueError("require_auth decorator requires 'request' parameter")
|
||||
request = kwargs["request"]
|
||||
|
||||
if getattr(request, "_deerflow_test_bypass_auth", False):
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
# Authenticate and set context
|
||||
auth_context = await _authenticate(request)
|
||||
request.state.auth = auth_context
|
||||
|
||||
if not auth_context.is_authenticated:
|
||||
raise HTTPException(status_code=401, detail="Authentication required")
|
||||
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def require_permission(
|
||||
resource: str,
|
||||
action: str,
|
||||
owner_check: bool = False,
|
||||
require_existing: bool = False,
|
||||
claim_orphan: bool = False,
|
||||
) -> Callable[[Callable[P, T]], Callable[P, T]]:
|
||||
"""Decorator that checks permission for resource:action.
|
||||
|
||||
Must be used AFTER @require_auth.
|
||||
|
||||
Args:
|
||||
resource: Resource name (e.g., "threads", "runs")
|
||||
action: Action name (e.g., "read", "write", "delete")
|
||||
owner_check: If True, validates that the current user owns the resource.
|
||||
Requires 'thread_id' path parameter and performs ownership check.
|
||||
require_existing: Only meaningful with ``owner_check=True``. If True, a
|
||||
missing ``threads_meta`` row counts as a denial (404)
|
||||
instead of "untracked legacy thread, allow". Use on
|
||||
**destructive / mutating** routes (DELETE, PATCH,
|
||||
state-update) so a deleted thread can't be re-targeted
|
||||
by another user via the missing-row code path.
|
||||
claim_orphan: Only meaningful with ``owner_check=True`` and
|
||||
``require_existing=True``. When the ``threads_meta``
|
||||
row is missing **and** a LangGraph checkpoint exists
|
||||
for the thread (a DB-switch orphan), adopt it for the
|
||||
current user instead of 404ing. Use on **run-create**
|
||||
routes so a conversation whose business row was lost
|
||||
(sqlite↔mysql swap) stays answerable. Safe: normal
|
||||
deletion removes the checkpoint, so this never revives
|
||||
a deliberately-deleted thread.
|
||||
|
||||
Usage:
|
||||
# Read-style: legacy untracked threads are allowed
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_thread(thread_id: str, request: Request):
|
||||
...
|
||||
|
||||
# Destructive: thread row MUST exist and be owned by caller
|
||||
@require_permission("threads", "delete", owner_check=True, require_existing=True)
|
||||
async def delete_thread(thread_id: str, request: Request):
|
||||
...
|
||||
|
||||
Raises:
|
||||
HTTPException 401: If authentication required but user is anonymous
|
||||
HTTPException 403: If user lacks permission
|
||||
HTTPException 404: If owner_check=True but user doesn't own the thread
|
||||
ValueError: If owner_check=True but no ``thread_id`` is supplied,
|
||||
either directly or on the Pydantic request body
|
||||
"""
|
||||
|
||||
def decorator(func: Callable[P, T]) -> Callable[P, T]:
|
||||
@functools.wraps(func)
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
request = kwargs.get("request")
|
||||
if request is None:
|
||||
# Unit tests may call decorated route handlers directly without
|
||||
# constructing a FastAPI Request object. Inject a minimal stub
|
||||
# when the wrapped function declares `request`.
|
||||
if "request" in inspect.signature(func).parameters:
|
||||
kwargs["request"] = _make_test_request_stub()
|
||||
else:
|
||||
return await func(*args, **kwargs)
|
||||
request = kwargs["request"]
|
||||
|
||||
if getattr(request, "_deerflow_test_bypass_auth", False):
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
auth: AuthContext = getattr(request.state, "auth", None)
|
||||
if auth is None:
|
||||
auth = await _authenticate(request)
|
||||
request.state.auth = auth
|
||||
|
||||
if not auth.is_authenticated:
|
||||
raise HTTPException(status_code=401, detail="Authentication required")
|
||||
|
||||
# Check permission
|
||||
if not auth.has_permission(resource, action):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=f"Permission denied: {resource}:{action}",
|
||||
)
|
||||
|
||||
# Owner check for thread-specific resources.
|
||||
#
|
||||
# 2.0-rc moved thread metadata into the SQL persistence layer
|
||||
# (``threads_meta`` table). We verify ownership via
|
||||
# ``ThreadMetaStore.check_access``: it returns True for
|
||||
# missing rows (untracked legacy thread) and for rows whose
|
||||
# ``user_id`` is NULL (shared / pre-auth data), so this is
|
||||
# strict-deny rather than strict-allow — only an *existing*
|
||||
# row with a *different* user_id triggers 404.
|
||||
if owner_check:
|
||||
thread_id = kwargs.get("thread_id")
|
||||
# Most thread-scoped routes expose ``thread_id`` in their URL.
|
||||
# A small number of mutation endpoints (for example complete
|
||||
# document rewrite) intentionally keep the target inside a
|
||||
# Pydantic request body so their public URL stays stable. The
|
||||
# FastAPI wrapper passes that body as ``body=...``; accept its
|
||||
# Python field name here while preserving the exact same
|
||||
# ownership verification below.
|
||||
if thread_id is None:
|
||||
body = kwargs.get("body")
|
||||
thread_id = getattr(body, "thread_id", None)
|
||||
if thread_id is None:
|
||||
raise ValueError(
|
||||
"require_permission with owner_check=True requires "
|
||||
"a 'thread_id' parameter or body.thread_id"
|
||||
)
|
||||
|
||||
from app.gateway.deps import get_thread_store
|
||||
|
||||
thread_store = get_thread_store(request)
|
||||
user_id = str(auth.user.id)
|
||||
project_store = getattr(request.app.state, "analysis_project_store", None)
|
||||
allowed_coro = thread_store.check_access(
|
||||
thread_id,
|
||||
user_id,
|
||||
require_existing=require_existing,
|
||||
)
|
||||
if project_store is not None:
|
||||
project, allowed = await asyncio.gather(
|
||||
_project_for_thread(project_store, thread_id),
|
||||
allowed_coro,
|
||||
)
|
||||
if project and project["user_id"] != user_id:
|
||||
raise HTTPException(status_code=404, detail="Project thread not accessible")
|
||||
else:
|
||||
allowed = await allowed_coro
|
||||
if not allowed:
|
||||
# Tell apart three denial causes so the client gets an
|
||||
# accurate message and so DB-switch orphans can be adopted
|
||||
# rather than hard-404'd:
|
||||
# 1. row exists but owned by another user -> not accessible
|
||||
# 2. no row, no checkpoint -> truly missing
|
||||
# 3. no row + checkpoint + claim_orphan -> orphan, claim
|
||||
existing = await thread_store.get(thread_id, user_id=None)
|
||||
if existing is not None:
|
||||
# Distinct detail string lets the frontend show "无权访问"
|
||||
# instead of "不存在" for a cross-user / stale-session hit.
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Thread {thread_id} not accessible",
|
||||
)
|
||||
claimed = False
|
||||
if claim_orphan and await _thread_checkpoint_exists(request, thread_id):
|
||||
try:
|
||||
await thread_store.create(thread_id, user_id=user_id)
|
||||
logger.info(
|
||||
"Claimed orphan thread %s for user %s (checkpoint present, no meta row)",
|
||||
thread_id,
|
||||
user_id,
|
||||
)
|
||||
claimed = True
|
||||
except Exception:
|
||||
logger.warning("Failed to claim orphan thread %s", thread_id, exc_info=True)
|
||||
if not claimed:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Thread {thread_id} not found",
|
||||
)
|
||||
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
"""Authorization decorators and context for DeerFlow.
|
||||
|
||||
Inspired by LangGraph Auth system: https://github.com/langchain-ai/langgraph/blob/main/libs/sdk-py/langgraph_sdk/auth/__init__.py
|
||||
|
||||
**Usage:**
|
||||
|
||||
1. Use ``@require_auth`` on routes that need authentication
|
||||
2. Use ``@require_permission("resource", "action", filter_key=...)`` for permission checks
|
||||
3. The decorator chain processes from bottom to top
|
||||
|
||||
**Example:**
|
||||
|
||||
@router.get("/{thread_id}")
|
||||
@require_auth
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_thread(thread_id: str, request: Request):
|
||||
# User is authenticated and has threads:read permission
|
||||
...
|
||||
|
||||
**Permission Model:**
|
||||
|
||||
- threads:read - View thread
|
||||
- threads:write - Create/update thread
|
||||
- threads:delete - Delete thread
|
||||
- runs:create - Run agent
|
||||
- runs:read - View run
|
||||
- runs:cancel - Cancel run
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
import inspect
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from types import SimpleNamespace
|
||||
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
|
||||
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.gateway.auth.models import User
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
P = ParamSpec("P")
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
async def _project_for_thread(project_store: Any, thread_id: str) -> dict | None:
|
||||
"""Best-effort project binding lookup; a store error must not fail the read."""
|
||||
try:
|
||||
return await project_store.project_for_thread(thread_id)
|
||||
except Exception:
|
||||
logger.debug("project_for_thread failed for %s", thread_id, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
async def _project_for_thread(project_store: Any, thread_id: str) -> dict | None:
|
||||
"""Best-effort project binding lookup; a store error must not fail the read."""
|
||||
try:
|
||||
return await project_store.project_for_thread(thread_id)
|
||||
except Exception:
|
||||
logger.debug("project_for_thread failed for %s", thread_id, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
async def _thread_checkpoint_exists(request: Request, thread_id: str) -> bool:
|
||||
"""Return True if a LangGraph checkpoint exists for ``thread_id``.
|
||||
|
||||
A checkpoint with no matching ``threads_meta`` row is the signature of a
|
||||
DB-switch orphan (sqlite↔mysql/postgres juggling): the conversation state
|
||||
survives in the checkpointer while the business row is gone. Normal thread
|
||||
deletion removes the checkpoint too, so this can never resurrect a
|
||||
deliberately-deleted thread.
|
||||
"""
|
||||
try:
|
||||
from app.gateway.deps import get_checkpointer
|
||||
|
||||
checkpointer = get_checkpointer(request)
|
||||
tup = await checkpointer.aget_tuple({"configurable": {"thread_id": thread_id}})
|
||||
return tup is not None
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# Permission constants
|
||||
class Permissions:
|
||||
"""Permission constants for resource:action format."""
|
||||
|
||||
# Threads
|
||||
THREADS_READ = "threads:read"
|
||||
THREADS_WRITE = "threads:write"
|
||||
THREADS_DELETE = "threads:delete"
|
||||
|
||||
# Runs
|
||||
RUNS_CREATE = "runs:create"
|
||||
RUNS_READ = "runs:read"
|
||||
RUNS_CANCEL = "runs:cancel"
|
||||
|
||||
|
||||
class AuthContext:
|
||||
"""Authentication context for the current request.
|
||||
|
||||
Stored in request.state.auth after require_auth decoration.
|
||||
|
||||
Attributes:
|
||||
user: The authenticated user, or None if anonymous
|
||||
permissions: List of permission strings (e.g., "threads:read")
|
||||
"""
|
||||
|
||||
__slots__ = ("user", "permissions")
|
||||
|
||||
def __init__(self, user: User | None = None, permissions: list[str] | None = None):
|
||||
self.user = user
|
||||
self.permissions = permissions or []
|
||||
|
||||
@property
|
||||
def is_authenticated(self) -> bool:
|
||||
"""Check if user is authenticated."""
|
||||
return self.user is not None
|
||||
|
||||
def has_permission(self, resource: str, action: str) -> bool:
|
||||
"""Check if context has permission for resource:action.
|
||||
|
||||
Args:
|
||||
resource: Resource name (e.g., "threads")
|
||||
action: Action name (e.g., "read")
|
||||
|
||||
Returns:
|
||||
True if user has permission
|
||||
"""
|
||||
permission = f"{resource}:{action}"
|
||||
return permission in self.permissions
|
||||
|
||||
def require_user(self) -> User:
|
||||
"""Get user or raise 401.
|
||||
|
||||
Raises:
|
||||
HTTPException 401 if not authenticated
|
||||
"""
|
||||
if not self.user:
|
||||
raise HTTPException(status_code=401, detail="Authentication required")
|
||||
return self.user
|
||||
|
||||
|
||||
def get_auth_context(request: Request) -> AuthContext | None:
|
||||
"""Get AuthContext from request state."""
|
||||
return getattr(request.state, "auth", None)
|
||||
|
||||
|
||||
_ALL_PERMISSIONS: list[str] = [
|
||||
Permissions.THREADS_READ,
|
||||
Permissions.THREADS_WRITE,
|
||||
Permissions.THREADS_DELETE,
|
||||
Permissions.RUNS_CREATE,
|
||||
Permissions.RUNS_READ,
|
||||
Permissions.RUNS_CANCEL,
|
||||
]
|
||||
|
||||
|
||||
def _make_test_request_stub() -> Any:
|
||||
"""Create a minimal request-like object for direct unit calls.
|
||||
|
||||
Used when decorated route handlers are invoked without FastAPI's
|
||||
request injection. Includes fields accessed by auth helpers.
|
||||
"""
|
||||
return SimpleNamespace(state=SimpleNamespace(), cookies={}, _deerflow_test_bypass_auth=True)
|
||||
|
||||
|
||||
async def _authenticate(request: Request) -> AuthContext:
|
||||
"""Authenticate request and return AuthContext.
|
||||
|
||||
Delegates to deps.get_optional_user_from_request() for the JWT→User pipeline.
|
||||
Returns AuthContext with user=None for anonymous requests.
|
||||
"""
|
||||
if is_auth_disabled():
|
||||
user = await get_assumed_user_for_disabled_auth()
|
||||
return AuthContext(user=user, permissions=_ALL_PERMISSIONS)
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return AuthContext(user=None, permissions=[])
|
||||
|
||||
# In future, permissions could be stored in user record
|
||||
return AuthContext(user=user, permissions=_ALL_PERMISSIONS)
|
||||
|
||||
|
||||
def require_auth[**P, T](func: Callable[P, T]) -> Callable[P, T]:
|
||||
"""Decorator that authenticates the request and enforces authentication.
|
||||
|
||||
Independently raises HTTP 401 for unauthenticated requests, regardless of
|
||||
whether ``AuthMiddleware`` is present in the ASGI stack. Sets the resolved
|
||||
``AuthContext`` on ``request.state.auth`` for downstream handlers.
|
||||
|
||||
Must be placed ABOVE other decorators (executes after them).
|
||||
|
||||
Usage:
|
||||
@router.get("/{thread_id}")
|
||||
@require_auth # Bottom decorator (executes first after permission check)
|
||||
@require_permission("threads", "read")
|
||||
async def get_thread(thread_id: str, request: Request):
|
||||
auth: AuthContext = request.state.auth
|
||||
...
|
||||
|
||||
Raises:
|
||||
HTTPException: 401 if the request is unauthenticated.
|
||||
ValueError: If 'request' parameter is missing.
|
||||
"""
|
||||
|
||||
@functools.wraps(func)
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
request = kwargs.get("request")
|
||||
if request is None:
|
||||
# Unit tests may call decorated handlers directly without a
|
||||
# FastAPI Request object. Inject a minimal request stub when
|
||||
# the wrapped function declares `request`.
|
||||
if "request" in inspect.signature(func).parameters:
|
||||
kwargs["request"] = _make_test_request_stub()
|
||||
else:
|
||||
raise ValueError("require_auth decorator requires 'request' parameter")
|
||||
request = kwargs["request"]
|
||||
|
||||
if getattr(request, "_deerflow_test_bypass_auth", False):
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
# Authenticate and set context
|
||||
auth_context = await _authenticate(request)
|
||||
request.state.auth = auth_context
|
||||
|
||||
if not auth_context.is_authenticated:
|
||||
raise HTTPException(status_code=401, detail="Authentication required")
|
||||
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def require_permission(
|
||||
resource: str,
|
||||
action: str,
|
||||
owner_check: bool = False,
|
||||
require_existing: bool = False,
|
||||
claim_orphan: bool = False,
|
||||
) -> Callable[[Callable[P, T]], Callable[P, T]]:
|
||||
"""Decorator that checks permission for resource:action.
|
||||
|
||||
Must be used AFTER @require_auth.
|
||||
|
||||
Args:
|
||||
resource: Resource name (e.g., "threads", "runs")
|
||||
action: Action name (e.g., "read", "write", "delete")
|
||||
owner_check: If True, validates that the current user owns the resource.
|
||||
Requires 'thread_id' path parameter and performs ownership check.
|
||||
require_existing: Only meaningful with ``owner_check=True``. If True, a
|
||||
missing ``threads_meta`` row counts as a denial (404)
|
||||
instead of "untracked legacy thread, allow". Use on
|
||||
**destructive / mutating** routes (DELETE, PATCH,
|
||||
state-update) so a deleted thread can't be re-targeted
|
||||
by another user via the missing-row code path.
|
||||
claim_orphan: Only meaningful with ``owner_check=True`` and
|
||||
``require_existing=True``. When the ``threads_meta``
|
||||
row is missing **and** a LangGraph checkpoint exists
|
||||
for the thread (a DB-switch orphan), adopt it for the
|
||||
current user instead of 404ing. Use on **run-create**
|
||||
routes so a conversation whose business row was lost
|
||||
(sqlite↔mysql swap) stays answerable. Safe: normal
|
||||
deletion removes the checkpoint, so this never revives
|
||||
a deliberately-deleted thread.
|
||||
|
||||
Usage:
|
||||
# Read-style: legacy untracked threads are allowed
|
||||
@require_permission("threads", "read", owner_check=True)
|
||||
async def get_thread(thread_id: str, request: Request):
|
||||
...
|
||||
|
||||
# Destructive: thread row MUST exist and be owned by caller
|
||||
@require_permission("threads", "delete", owner_check=True, require_existing=True)
|
||||
async def delete_thread(thread_id: str, request: Request):
|
||||
...
|
||||
|
||||
Raises:
|
||||
HTTPException 401: If authentication required but user is anonymous
|
||||
HTTPException 403: If user lacks permission
|
||||
HTTPException 404: If owner_check=True but user doesn't own the thread
|
||||
ValueError: If owner_check=True but no ``thread_id`` is supplied,
|
||||
either directly or on the Pydantic request body
|
||||
"""
|
||||
|
||||
def decorator(func: Callable[P, T]) -> Callable[P, T]:
|
||||
@functools.wraps(func)
|
||||
async def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
request = kwargs.get("request")
|
||||
if request is None:
|
||||
# Unit tests may call decorated route handlers directly without
|
||||
# constructing a FastAPI Request object. Inject a minimal stub
|
||||
# when the wrapped function declares `request`.
|
||||
if "request" in inspect.signature(func).parameters:
|
||||
kwargs["request"] = _make_test_request_stub()
|
||||
else:
|
||||
return await func(*args, **kwargs)
|
||||
request = kwargs["request"]
|
||||
|
||||
if getattr(request, "_deerflow_test_bypass_auth", False):
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
auth: AuthContext = getattr(request.state, "auth", None)
|
||||
if auth is None:
|
||||
auth = await _authenticate(request)
|
||||
request.state.auth = auth
|
||||
|
||||
if not auth.is_authenticated:
|
||||
raise HTTPException(status_code=401, detail="Authentication required")
|
||||
|
||||
# Check permission
|
||||
if not auth.has_permission(resource, action):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail=f"Permission denied: {resource}:{action}",
|
||||
)
|
||||
|
||||
# Owner check for thread-specific resources.
|
||||
#
|
||||
# 2.0-rc moved thread metadata into the SQL persistence layer
|
||||
# (``threads_meta`` table). We verify ownership via
|
||||
# ``ThreadMetaStore.check_access``: it returns True for
|
||||
# missing rows (untracked legacy thread) and for rows whose
|
||||
# ``user_id`` is NULL (shared / pre-auth data), so this is
|
||||
# strict-deny rather than strict-allow — only an *existing*
|
||||
# row with a *different* user_id triggers 404.
|
||||
if owner_check:
|
||||
thread_id = kwargs.get("thread_id")
|
||||
# Most thread-scoped routes expose ``thread_id`` in their URL.
|
||||
# A small number of mutation endpoints (for example complete
|
||||
# document rewrite) intentionally keep the target inside a
|
||||
# Pydantic request body so their public URL stays stable. The
|
||||
# FastAPI wrapper passes that body as ``body=...``; accept its
|
||||
# Python field name here while preserving the exact same
|
||||
# ownership verification below.
|
||||
if thread_id is None:
|
||||
body = kwargs.get("body")
|
||||
thread_id = getattr(body, "thread_id", None)
|
||||
if thread_id is None:
|
||||
raise ValueError(
|
||||
"require_permission with owner_check=True requires "
|
||||
"a 'thread_id' parameter or body.thread_id"
|
||||
)
|
||||
|
||||
from app.gateway.deps import get_thread_store
|
||||
|
||||
thread_store = get_thread_store(request)
|
||||
user_id = str(auth.user.id)
|
||||
project_store = getattr(request.app.state, "analysis_project_store", None)
|
||||
allowed_coro = thread_store.check_access(
|
||||
thread_id,
|
||||
user_id,
|
||||
require_existing=require_existing,
|
||||
)
|
||||
if project_store is not None:
|
||||
project, allowed = await asyncio.gather(
|
||||
_project_for_thread(project_store, thread_id),
|
||||
allowed_coro,
|
||||
)
|
||||
if project and project["user_id"] != user_id:
|
||||
raise HTTPException(status_code=404, detail="Project thread not accessible")
|
||||
else:
|
||||
allowed = await allowed_coro
|
||||
if not allowed:
|
||||
# Tell apart three denial causes so the client gets an
|
||||
# accurate message and so DB-switch orphans can be adopted
|
||||
# rather than hard-404'd:
|
||||
# 1. row exists but owned by another user -> not accessible
|
||||
# 2. no row, no checkpoint -> truly missing
|
||||
# 3. no row + checkpoint + claim_orphan -> orphan, claim
|
||||
existing = await thread_store.get(thread_id, user_id=None)
|
||||
if existing is not None:
|
||||
# Distinct detail string lets the frontend show "无权访问"
|
||||
# instead of "不存在" for a cross-user / stale-session hit.
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Thread {thread_id} not accessible",
|
||||
)
|
||||
claimed = False
|
||||
if claim_orphan and await _thread_checkpoint_exists(request, thread_id):
|
||||
try:
|
||||
await thread_store.create(thread_id, user_id=user_id)
|
||||
logger.info(
|
||||
"Claimed orphan thread %s for user %s (checkpoint present, no meta row)",
|
||||
thread_id,
|
||||
user_id,
|
||||
)
|
||||
claimed = True
|
||||
except Exception:
|
||||
logger.warning("Failed to claim orphan thread %s", thread_id, exc_info=True)
|
||||
if not claimed:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Thread {thread_id} not found",
|
||||
)
|
||||
|
||||
return await func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
@@ -204,6 +204,35 @@ class RoundtableJobDispatcher:
|
||||
)
|
||||
return expired
|
||||
|
||||
async def _expire_overdue_step2_rooms(self, now: datetime) -> int:
|
||||
"""Persist hard-timeout cancellation even when no browser is connected."""
|
||||
max_minutes = getattr(
|
||||
get_roundtable_room_limit_settings(),
|
||||
"max_hold_minutes",
|
||||
30,
|
||||
)
|
||||
request_cancel_expired = getattr(
|
||||
self._store,
|
||||
"request_cancel_expired_step2",
|
||||
None,
|
||||
)
|
||||
if request_cancel_expired is None:
|
||||
return 0
|
||||
try:
|
||||
expired = await request_cancel_expired(
|
||||
created_before=now - timedelta(minutes=max_minutes),
|
||||
non_occupying_phases=tuple(NON_OCCUPYING_JOB_PHASES),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("[roundtable-jobs] hard-timeout scan failed", exc_info=True)
|
||||
return 0
|
||||
if expired:
|
||||
logger.info(
|
||||
"[roundtable-jobs] requested cancellation for %d overdue Step-2 room(s)",
|
||||
expired,
|
||||
)
|
||||
return expired
|
||||
|
||||
async def _reap_cancel_requested(self, now: datetime) -> int:
|
||||
"""收口持租 worker 崩溃后遗留的 ``cancel_requested`` 作业。
|
||||
|
||||
|
||||
@@ -69,6 +69,26 @@ _DRAFT_WRITE_RETRIES = 3
|
||||
ROUNDTABLE_BACKGROUND_REPORT_ENABLED = True
|
||||
|
||||
|
||||
def _background_step2_room_expired(row: dict[str, Any] | None) -> bool:
|
||||
"""Whether a background job exceeded the configurable Step-2 hard limit."""
|
||||
if not row or str(row.get("phase") or "") in NON_OCCUPYING_JOB_PHASES:
|
||||
return False
|
||||
raw_created = row.get("created_at")
|
||||
if isinstance(raw_created, datetime):
|
||||
created_at = raw_created
|
||||
elif isinstance(raw_created, str) and raw_created:
|
||||
try:
|
||||
created_at = datetime.fromisoformat(raw_created)
|
||||
except ValueError:
|
||||
return False
|
||||
else:
|
||||
return False
|
||||
if created_at.tzinfo is None:
|
||||
created_at = created_at.replace(tzinfo=UTC)
|
||||
max_minutes = getattr(get_roundtable_room_limit_settings(), "max_hold_minutes", 30)
|
||||
return datetime.now(UTC) >= created_at + timedelta(minutes=max_minutes)
|
||||
|
||||
|
||||
def _background_step2_room_expired(row: dict[str, Any] | None) -> bool:
|
||||
"""Whether a background job exceeded the configurable Step-2 hard limit."""
|
||||
if not row or str(row.get("phase") or "") in NON_OCCUPYING_JOB_PHASES:
|
||||
|
||||
@@ -54,6 +54,8 @@ _AGENT_PURPOSE: dict[str, str] = {
|
||||
# Workflow Studio(候选流程总控,和圆桌总控隔离)
|
||||
"workflow-planner": "工作流编排",
|
||||
"task-analysis-coordinator": "任务分析定时",
|
||||
"chat-history-migrator": "其他内置功能",
|
||||
"gauss-data-adapter": "其他内置功能",
|
||||
}
|
||||
|
||||
_TARGET_TYPE = "agent"
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Seed the built-in chat-history migration assistant.
|
||||
|
||||
The agent and its public skill are copied only when their target directories are
|
||||
missing. Existing deployments are never overwritten, so administrator edits made
|
||||
through agent/skill management survive restarts. This seeder must run before
|
||||
``_sync_legacy_agents``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CHAT_HISTORY_MIGRATION_AGENT_ID = "chat-history-migrator"
|
||||
CHAT_HISTORY_MIGRATION_SKILL_NAME = "chat-history-migration"
|
||||
|
||||
_ASSETS_DIR = Path(__file__).parent / "_chat_history_migration_seed_assets"
|
||||
_AGENT_FILES: tuple[str, ...] = ("config.yaml", "SOUL.md")
|
||||
|
||||
|
||||
def _skills_public_dir() -> Path:
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
|
||||
return Path(get_or_new_skill_storage().get_skills_root_path()) / "public"
|
||||
|
||||
|
||||
def _ensure_skill() -> bool:
|
||||
try:
|
||||
source = _ASSETS_DIR / "skills" / CHAT_HISTORY_MIGRATION_SKILL_NAME
|
||||
target = _skills_public_dir() / CHAT_HISTORY_MIGRATION_SKILL_NAME
|
||||
if target.exists():
|
||||
return False
|
||||
if not (source / "SKILL.md").is_file():
|
||||
logger.error("Missing chat-history migration skill seed at %s", source)
|
||||
return False
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(source, target)
|
||||
logger.info("Seeded public skill '%s'", CHAT_HISTORY_MIGRATION_SKILL_NAME)
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("Failed to seed chat-history migration skill")
|
||||
return False
|
||||
|
||||
|
||||
def _ensure_agent() -> bool:
|
||||
try:
|
||||
target = get_paths().agent_dir(CHAT_HISTORY_MIGRATION_AGENT_ID)
|
||||
if target.exists():
|
||||
return False
|
||||
source = _ASSETS_DIR / CHAT_HISTORY_MIGRATION_AGENT_ID
|
||||
if not source.is_dir():
|
||||
logger.error("Missing chat-history migration agent seed at %s", source)
|
||||
return False
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for filename in _AGENT_FILES:
|
||||
asset = source / filename
|
||||
if asset.is_file():
|
||||
(target / filename).write_bytes(asset.read_bytes())
|
||||
logger.info("Seeded built-in agent '%s'", CHAT_HISTORY_MIGRATION_AGENT_ID)
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("Failed to seed chat-history migration agent")
|
||||
return False
|
||||
|
||||
|
||||
def ensure_chat_history_migration_agent() -> bool:
|
||||
"""Install the chat-history migration skill and agent when missing."""
|
||||
_ensure_skill()
|
||||
return _ensure_agent()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CHAT_HISTORY_MIGRATION_AGENT_ID",
|
||||
"CHAT_HISTORY_MIGRATION_SKILL_NAME",
|
||||
"ensure_chat_history_migration_agent",
|
||||
]
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
# 角色
|
||||
|
||||
你是 **聊天记录迁移助手**。你帮助管理员把一台 DeerFlow 服务器上的普通聊天记录迁到另一台服务器。源站的业务库可以是 openGauss/高斯,目标站可以是高斯、SQLite 或 MySQL。
|
||||
|
||||
你面对的操作者可能不熟悉数据库。先用一句通俗的话解释术语,再给可复制的命令。命令里的主机、端口、库名、路径一律用对方提供的值;没有提供时用占位符,禁止猜。
|
||||
|
||||
# 聊天记录实际放在哪
|
||||
|
||||
一次可见的聊天不是高斯库里的一张表。它有三层,缺任何一层都会表现为「列表空」或「有标题但点进去没有内容」:
|
||||
|
||||
1. **会话索引**:`threads_meta` 决定聊天列表里的标题、所属用户和时间。`runs`、`run_events`、`feedback` 是运行记录和评价。项目里的会话还依赖 `analysis_project_threads`。这一层可以在高斯库里。
|
||||
2. **消息正文**:LangGraph 检查点。推荐放在 SQLite 文件,也可以是真正的 PostgreSQL。这里才是问答内容。
|
||||
3. **线程文件**:`.deer-flow/users/<user_id>/threads/<thread_id>/user-data/` 下面的上传文件和生成结果。
|
||||
|
||||
不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上。高斯只承接业务索引。检查点文件原样进入目标的 SQLite 检查点库。
|
||||
|
||||
# 必须遵守的边界
|
||||
|
||||
1. **智能体沙箱不是源服务器,也不是目标服务器。** 除非用户明确说明当前环境就是那台机器,否则不要根据沙箱里的主机名、localhost 或某个 `.db` 文件推断现场。需要现场执行的命令要写明在哪一台机器、哪个目录运行。
|
||||
2. **不要接收密码。** 不要让用户把高斯密码、token、私钥贴进对话。`gsql` / `mysql` 使用 `~/.pgpass` 或客户端自己的密码提示。脚本不会连接高斯,也没有密码参数。
|
||||
3. **先盘点,再导出,再演练,最后才写入。** 导入命令默认是演练。用户明确说过目标库已有可恢复备份之前,不要在命令里加 `--apply`。
|
||||
4. **不导出账号密码。** `users` 表和密码列不进入迁移包。目标站必须已经有对应账号。`user_id` 不同时只用 `--user-map 旧ID=新ID` 改归属,不改 `thread_id`。
|
||||
5. **默认跳过已有会话。** 只有用户明确要求覆盖时才使用 `--on-conflict replace`。替换只影响迁移包里的那些 `thread_id`,不会清空目标库其他会话。已有项目的名称和配置保持目标站原样。
|
||||
6. **系统线程默认不迁。** 调度器和内部工作线程会让列表变脏。用户点名要包含时才加 `--include-system`。
|
||||
7. **源库只读。** 不在源库执行 INSERT、UPDATE、DELETE、DROP。导出 SQL 只有查询和 `\copy TO`。
|
||||
|
||||
# 工作方式
|
||||
|
||||
收到请求后先归入一种模式:
|
||||
|
||||
- **还不清楚数据在哪**:读取技能里的 `references/storage-map.md`,根据用户贴出的 `database.backend` 和 `checkpointer` 判断三层位置。一次只追问会改变命令的几项:业务库类型、检查点文件路径、`.deer-flow` 目录、要迁的用户或时间范围、目标库类型。
|
||||
- **准备搬迁**:读取 `references/runbook.md`。按「停写或接受截止点 → 只读导出 → 打包 → 传到目标 → 演练 → 备份确认 → `--apply` → 验收」推进。每一步都写成功信号和失败时停在哪里。
|
||||
- **迁完对不上**:先区分「列表没有这条会话」「有标题但正文是空的」「附件不见了」「进了别的账号」。分别对应索引、检查点、线程文件和 `user_id`。不要一次改三层。
|
||||
|
||||
脚本在技能目录 `scripts/migrate_chat_history.py`,只用 Python 标准库。用户要在源服务器上跑时,把脚本复制到输出目录并交付:
|
||||
|
||||
```bash
|
||||
cp /mnt/skills/public/chat-history-migration/scripts/migrate_chat_history.py /mnt/user-data/outputs/migrate_chat_history.py
|
||||
```
|
||||
|
||||
沙箱里没有 `bash` 时,用 `read_file` 读取脚本,再用 `write_file` 原样写到该路径。用户已经把 SQLite 或 JSONL 传到当前会话时,可以对 `/mnt/user-data/uploads/` 里的文件运行脚本,把迁移包写到 `/mnt/user-data/outputs/`。
|
||||
|
||||
# 交付格式
|
||||
|
||||
每次给操作步骤时包含这几项;没有的信息写「还不知道」:
|
||||
|
||||
1. **在哪台机器做**:源服务器、目标服务器,或当前会话里的上传文件。
|
||||
2. **这一步会不会改数据**:只读、演练,或写入。写入前重复备份要求。
|
||||
3. **命令**:可复制,占位符用 `<GAUSS_HOST>`、`<GAUSS_DB>`、`<CHECKPOINT_SQLITE>`、`<DEER_FLOW_ROOT>` 这种形式。
|
||||
4. **成功信号**:应该看到的线程数、某个已知会话标题、检查点行数。
|
||||
5. **停机点**:命令失败或数量对不上时不要继续下一步。
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
id: chat-history-migrator
|
||||
name: 聊天记录迁移助手
|
||||
description: 把一台 DeerFlow 服务器上的聊天记录迁到另一台。源站业务库可以是高斯/openGauss。会话索引、LangGraph 检查点和线程文件分开处理,不把 PostgreSQL 检查点迁移直接套到高斯上。
|
||||
tool_groups:
|
||||
- file:read
|
||||
- file:write
|
||||
- bash
|
||||
skills:
|
||||
- chat-history-migration
|
||||
common_questions:
|
||||
- title: 高斯库里的聊天记录怎么迁
|
||||
prompt: >-
|
||||
源服务器的业务库是高斯/openGauss,我要把聊天记录迁到另一台 DeerFlow。
|
||||
请先用通俗的话说明聊天记录分哪几层存放,再给我一份只读盘点步骤。
|
||||
不要让我把数据库密码贴进对话,也不要给出把 LangGraph PostgreSQL 检查点迁移直接执行到高斯库上的命令。
|
||||
- title: 先做只读盘点
|
||||
prompt: >-
|
||||
请根据我提供的 database.backend、checkpointer 类型和目录,判断会话索引、消息正文和线程文件各在哪里。
|
||||
缺哪一项就告诉我该在哪台机器上看哪个文件,不要猜路径,也不要开始拷贝。
|
||||
- title: 导出迁移包
|
||||
prompt: >-
|
||||
请带我从源服务器导出聊天迁移包。业务库如果是高斯或 MySQL 协议的高斯,先生成只读导出 SQL;
|
||||
检查点仍从 SQLite 文件提取。导出前说明会排除系统线程、不会导出用户密码。
|
||||
- title: 导入前先演练
|
||||
prompt: >-
|
||||
迁移包已经放到目标服务器。请先做导入演练,告诉我会新增哪些会话、会跳过哪些已有会话、哪些文件会拷贝。
|
||||
我明确确认已经有可恢复备份之前,不要加上 --apply。
|
||||
- title: 导入后怎么验收
|
||||
prompt: >-
|
||||
导入已经执行。请给我验收步骤:会话列表能看到原标题,点进会话能看到原来的问答,附件还在原线程目录。
|
||||
如果只有标题、打开是空的,请判断是检查点没迁过去,还是 user_id 对不上。
|
||||
- title: 两边的用户 ID 不一样
|
||||
prompt: >-
|
||||
源服务器和目标服务器都有这些账号,但 user_id 不同,聊天记录迁过去后列表是空的。
|
||||
请说明怎样用 --user-map 只改归属、不改 thread_id,以及哪些字段不会自动改写。
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
---
|
||||
name: chat-history-migration
|
||||
description: 把 DeerFlow 聊天记录从一台服务器迁到另一台。源业务库可以是 openGauss/高斯或 MySQL 协议高斯。分别导出会话索引、SQLite 检查点和线程文件,导入前默认只演练,不把 PostgreSQL 检查点迁移执行到高斯上。
|
||||
---
|
||||
|
||||
# 聊天记录迁移
|
||||
|
||||
本技能的脚本和说明都在本地。先确认用户处于「还不知道数据在哪」「准备导出」「准备导入」还是「迁完对不上」,再只读需要的材料:
|
||||
|
||||
- 判断高斯库、检查点和线程目录各存什么:读取 `references/storage-map.md`。
|
||||
- 已经知道路径,需要逐步导出、传输、演练和验收:读取 `references/runbook.md`。
|
||||
- 两件事都还没确定时先读存储说明,不要把两份参考一次读完。
|
||||
|
||||
脚本路径:
|
||||
|
||||
```text
|
||||
/mnt/skills/public/chat-history-migration/scripts/migrate_chat_history.py
|
||||
```
|
||||
|
||||
它只用 Python 标准库,不连接高斯,也不接收密码。`import` 不加 `--apply` 时只演练,不创建、不修改目标文件。
|
||||
|
||||
## 现场和沙箱
|
||||
|
||||
`bash` 跑在 DeerFlow 沙箱里,通常不是要迁移的那台服务器。沙箱里的数据库文件和目录不能当成源站或目标站,除非用户明确说当前环境就是那台机器。
|
||||
|
||||
两种做法:
|
||||
|
||||
1. 把脚本复制到输出目录,让用户在源服务器和目标服务器上执行,再把 JSON 结果贴回来。
|
||||
2. 用户已经把 SQLite、JSONL 或迁移包上传到当前会话时,直接对 `/mnt/user-data/uploads/` 运行脚本,产物写到 `/mnt/user-data/outputs/`。
|
||||
|
||||
交付脚本:
|
||||
|
||||
```bash
|
||||
cp /mnt/skills/public/chat-history-migration/scripts/migrate_chat_history.py /mnt/user-data/outputs/migrate_chat_history.py
|
||||
```
|
||||
|
||||
没有 `bash` 时,读取脚本原文,再用 `write_file` 原样写到上面的路径。
|
||||
|
||||
## 不允许的做法
|
||||
|
||||
- 不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上。
|
||||
- 不要导出或覆盖 `users` 表,不要读写 `password_hash`、`login_password`。
|
||||
- 不要在命令行参数或对话里出现数据库密码。
|
||||
- 不要在用户确认目标库已有可恢复备份之前执行带 `--apply` 的导入。
|
||||
- 不要因为列表能看到标题,就认为消息正文已经迁完。正文在检查点里。
|
||||
+132
@@ -0,0 +1,132 @@
|
||||
# 迁移步骤
|
||||
|
||||
在源服务器导出,在目标服务器导入。迁移包里有对话正文,只通过可信通道拷贝。下面的命令在对应服务器的 shell 里执行,不在智能体沙箱里执行,除非用户确认当前环境就是那台机器。
|
||||
|
||||
开始前向用户要这些值,缺的用占位符,不要猜:
|
||||
|
||||
- 源、目标的 `database.backend`
|
||||
- 高斯是 `opengauss` 还是 Dolphin(`gauss_mysql`)
|
||||
- 检查点 SQLite 的实际路径
|
||||
- `.deer-flow` 根目录
|
||||
- 要迁的 `user_id` 或时间范围
|
||||
- 目标账号 id 是否和源相同
|
||||
|
||||
建议先停止源 Gateway,或接受导出时刻之后的新消息不在包里。脚本不会锁高斯库,热导出不是一致快照。
|
||||
|
||||
## 1. 源站只读盘点
|
||||
|
||||
SQLite 索引或统一的 `deerflow.db`:
|
||||
|
||||
```bash
|
||||
python3 migrate_chat_history.py inventory \
|
||||
--app-sqlite <APP_SQLITE> \
|
||||
--checkpoint-sqlite <CHECKPOINT_SQLITE> \
|
||||
--threads-root <DEER_FLOW_ROOT>
|
||||
```
|
||||
|
||||
只传存在的路径。业务库是高斯、检查点是 SQLite 时,不要传 `--app-sqlite`;索引改用下一步的 SQL 盘点。成功信号是 JSON 里的 `threads_meta`、检查点线程数和文件数。`system_threads` 是默认会排除的内部线程。
|
||||
|
||||
## 2. 从高斯导出索引
|
||||
|
||||
在源服务器上执行。密码放在 `~/.pgpass`(权限 600)或让客户端提示,不要写进命令,也不要贴进对话。
|
||||
|
||||
```bash
|
||||
python3 migrate_chat_history.py emit-export-sql --dialect opengauss --out export_chat_index.sql
|
||||
gsql -h <GAUSS_HOST> -p 26000 -d <GAUSS_DB> -U <GAUSS_USER> -f export_chat_index.sql
|
||||
```
|
||||
|
||||
Dolphin / MySQL 协议把 `--dialect` 换成 `mysql`,然后用 `mysql -h <HOST> -P 3306 -u <USER> -p -D <DB> -N` 逐条执行,每条查询的输出存成对应的 `.jsonl`。`-p` 会提示输入密码。
|
||||
|
||||
这份 SQL 只有查询。它不包含检查点表。不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上。
|
||||
|
||||
## 3. 打迁移包
|
||||
|
||||
索引也在 SQLite 时:
|
||||
|
||||
```bash
|
||||
python3 migrate_chat_history.py export \
|
||||
--out <BUNDLE_DIR> \
|
||||
--app-sqlite <APP_SQLITE> \
|
||||
--checkpoint-sqlite <CHECKPOINT_SQLITE> \
|
||||
--threads-root <DEER_FLOW_ROOT> \
|
||||
--user-id <USER_ID>
|
||||
```
|
||||
|
||||
索引来自刚才的 JSONL 时:
|
||||
|
||||
```bash
|
||||
python3 migrate_chat_history.py pack-jsonl \
|
||||
--out <BUNDLE_DIR> \
|
||||
--threads-meta threads_meta.jsonl \
|
||||
--runs runs.jsonl \
|
||||
--run-events run_events.jsonl \
|
||||
--feedback feedback.jsonl \
|
||||
--project-threads analysis_project_threads.jsonl \
|
||||
--projects analysis_projects.jsonl \
|
||||
--checkpoint-sqlite <CHECKPOINT_SQLITE> \
|
||||
--threads-root <DEER_FLOW_ROOT>
|
||||
```
|
||||
|
||||
`<BUNDLE_DIR>` 必须是空目录,并且不能放在源数据库或 `.deer-flow` 目录里面。成功信号是目录里有 `manifest.json`。`excluded_system_thread_ids` 里的会话是故意不迁的。需要它们时两边都加 `--include-system`。
|
||||
|
||||
只迁某个时间之后的会话时加 `--since 2026-10-01T00:00:00`。
|
||||
|
||||
## 4. 目标站先演练
|
||||
|
||||
先复制备份:目标检查点 SQLite、目标业务库(高斯做库级备份)。没有备份就不加 `--apply`。
|
||||
|
||||
SQLite 业务库:
|
||||
|
||||
```bash
|
||||
python3 migrate_chat_history.py import \
|
||||
--bundle <BUNDLE_DIR> \
|
||||
--app-sqlite <TARGET_APP_SQLITE> \
|
||||
--checkpoint-sqlite <TARGET_CHECKPOINT_SQLITE> \
|
||||
--threads-root <TARGET_DEER_FLOW_ROOT>
|
||||
```
|
||||
|
||||
目标业务库是高斯时,索引和检查点分开:
|
||||
|
||||
```bash
|
||||
python3 migrate_chat_history.py emit-gauss-sql --bundle <BUNDLE_DIR> --out load_chat_index.sql
|
||||
python3 migrate_chat_history.py import \
|
||||
--bundle <BUNDLE_DIR> \
|
||||
--checkpoint-sqlite <TARGET_CHECKPOINT_SQLITE> \
|
||||
--threads-root <TARGET_DEER_FLOW_ROOT>
|
||||
```
|
||||
|
||||
演练的成功信号是 `"dry_run": true`,并且目标文件的修改时间没有变。看 `inserted`、`skipped_threads` 和 `files_copied`。已有的 `thread_id` 默认跳过;如果索引已在而检查点缺失,演练会显示还要补检查点。
|
||||
|
||||
账号 id 不同:
|
||||
|
||||
```bash
|
||||
python3 migrate_chat_history.py import --bundle <BUNDLE_DIR> --user-map <OLD_USER_ID>=<NEW_USER_ID>
|
||||
```
|
||||
|
||||
可以重复写多个 `--user-map`。
|
||||
|
||||
## 5. 确认备份后再写入
|
||||
|
||||
SQLite 目标把同一条 `import` 加上 `--apply`。高斯索引另执行:
|
||||
|
||||
```bash
|
||||
gsql -h <GAUSS_HOST> -p 26000 -d <GAUSS_DB> -U <GAUSS_USER> -f load_chat_index.sql
|
||||
```
|
||||
|
||||
`load_chat_index.sql` 默认 `INSERT ... WHERE NOT EXISTS`,不覆盖已有主键。用户明确要求覆盖这些会话时,生成 SQL 才加 `--on-conflict replace`。替换会删除目标上这些 `thread_id` 的索引行再写入,不会删除同 id 的已有项目定义,也不会动迁移包以外的会话。
|
||||
|
||||
目标 SQLite 文件必须已经存在(先启动一次目标 Gateway,让它建好表)。检查点文件不存在时,`--apply` 可以按迁移包里的表结构创建。
|
||||
|
||||
## 6. 验收
|
||||
|
||||
```bash
|
||||
python3 migrate_chat_history.py verify \
|
||||
--bundle <BUNDLE_DIR> \
|
||||
--app-sqlite <TARGET_APP_SQLITE> \
|
||||
--checkpoint-sqlite <TARGET_CHECKPOINT_SQLITE> \
|
||||
--threads-root <TARGET_DEER_FLOW_ROOT>
|
||||
```
|
||||
|
||||
退出码 0 表示会话、检查点行数和文件都在。高斯索引不经过这个 SQLite 校验;在目标高斯里对一个已知 `thread_id` 查询 `threads_meta`,再在网页上打开该会话,确认能看到迁前的问答和附件。
|
||||
|
||||
数量不对就停。跳过模式下重新执行 `--apply` 只会补缺失的检查点或文件,不会覆盖已有会话。要用备份恢复时,恢复第 4 步复制的目标库和检查点文件,并删掉本次新建的 `users/<user_id>/threads/<thread_id>` 目录。源服务器始终没有被修改。
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
# 聊天记录存在哪
|
||||
|
||||
DeerFlow 的一条聊天要三层都在,打开后才能看到原来的问答。
|
||||
|
||||
## 1. 会话索引(可以在高斯里)
|
||||
|
||||
`config.yaml` 的 `database.backend` 决定这一层:
|
||||
|
||||
| backend | 索引在哪 | 导出方式 |
|
||||
| --- | --- | --- |
|
||||
| `gauss` | `database.gauss_url` 指向的 openGauss/GaussDB | `emit-export-sql --dialect opengauss`,再用 `gsql` 执行 |
|
||||
| `mysql`,且部署设置了 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql` | 高斯的 MySQL 协议(Dolphin) | `emit-export-sql --dialect mysql`,再用 `mysql` 客户端执行 |
|
||||
| `sqlite` | `{sqlite_dir}/deerflow.db`,默认 `.deer-flow/data/deerflow.db` | 脚本直接读这个文件 |
|
||||
| `postgres` | `database.postgres_url` | 先把索引导出成 JSONL,再 `pack-jsonl`。不要改检查点迁移方式 |
|
||||
|
||||
和聊天直接相关的表:
|
||||
|
||||
- `threads_meta`:聊天列表。`thread_id`、`user_id`、`display_name`、`metadata_json`、时间。
|
||||
- `runs` / `run_events`:一轮运行的状态、首句预览和事件。
|
||||
- `feedback`:回答评价。
|
||||
- `analysis_projects` / `analysis_project_threads`:会话属于哪个分析项目。目标上已有的同 id 项目不会被覆盖;项目下的知识库文件不在这个迁移里。
|
||||
|
||||
`users` 表不迁。密码列不迁。目标服务器要先有账号,并且 `threads_meta.user_id` 等于目标账号的 id。两边 id 不同时,导入用 `--user-map 旧ID=新ID`。这个映射只改 `user_id` 以及 `users/<旧ID>/threads/...` 目录,不改 `thread_id`,也不改消息正文里的文字。
|
||||
|
||||
`metadata_json.system = true` 的是调度器或内部工作线程,默认排除。
|
||||
|
||||
## 2. 消息正文(不要放进高斯迁移)
|
||||
|
||||
问答正文在 LangGraph 检查点,不在 `threads_meta`。
|
||||
|
||||
`config.yaml` 的 `checkpointer`:
|
||||
|
||||
- `type: sqlite`:`connection_string` 指向的文件。业务库是 SQLite 且没有单独的 checkpointer 文件时,检查点表和 `threads_meta` 可能在同一个 `deerflow.db` 里。脚本遇到这种文件会把两层拆开。
|
||||
- `type: postgres`:只适用于真正的 PostgreSQL。它的建表迁移不能对高斯执行。
|
||||
- 业务库是 `gauss` 时,检查点应继续用 SQLite。如果现场误把检查点建在了高斯里,先停下来说明这个部署和当前脚本的支持范围不一致,不要自动改跑 PostgreSQL 迁移。
|
||||
|
||||
检查点表通常叫 `checkpoints`、`writes`、`checkpoint_blobs`、`checkpoint_writes`。迁移包里的 `checkpoints.sqlite` 只含选中会话的这些表。
|
||||
|
||||
只迁了 `threads_meta` 时,列表有标题,点进去没有历史消息。
|
||||
|
||||
## 3. 线程文件
|
||||
|
||||
默认在后端运行目录:
|
||||
|
||||
```text
|
||||
<DEER_FLOW_ROOT>/users/<user_id>/threads/<thread_id>/user-data/
|
||||
```
|
||||
|
||||
旧数据可能在 `<DEER_FLOW_ROOT>/threads/<thread_id>/`。`--threads-root` 传 `<DEER_FLOW_ROOT>`,也就是同时能看到 `users/` 或旧 `threads/` 的那一层,不要只传某一个会话目录。
|
||||
|
||||
这里有上传文件和生成的 HTML、Markdown。知识库原文、记忆库和分享链接不在这份迁移里。
|
||||
|
||||
## 迁完后的对照
|
||||
|
||||
| 现象 | 先看 |
|
||||
| --- | --- |
|
||||
| 目标列表完全没有这些会话 | 索引没进目标库,或 `user_id` 对不上 |
|
||||
| 有标题,打开是空的 | 检查点没进目标 SQLite |
|
||||
| 文字在,附件不见 | `--threads-root` 没指到同一个 `.deer-flow` 根 |
|
||||
| 普通聊天在,项目里看不到 | 目标没有同 id 的 `analysis_projects`,或绑定表没导入 |
|
||||
+1591
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,81 @@
|
||||
"""Seed the built-in GaussDB data-adaptation assistant.
|
||||
|
||||
The agent and its public skill are copied only when their target directories are
|
||||
missing. Existing deployments are never overwritten, so administrator edits made
|
||||
through agent/skill management survive restarts. This seeder must run before
|
||||
``_sync_legacy_agents``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
GAUSS_DATA_ADAPTATION_AGENT_ID = "gauss-data-adapter"
|
||||
GAUSS_DATA_ADAPTATION_SKILL_NAME = "gauss-data-adaptation"
|
||||
|
||||
_ASSETS_DIR = Path(__file__).parent / "_gauss_data_adaptation_seed_assets"
|
||||
_AGENT_FILES: tuple[str, ...] = ("config.yaml", "SOUL.md")
|
||||
|
||||
|
||||
def _skills_public_dir() -> Path:
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
|
||||
return Path(get_or_new_skill_storage().get_skills_root_path()) / "public"
|
||||
|
||||
|
||||
def _ensure_skill() -> bool:
|
||||
try:
|
||||
source = _ASSETS_DIR / "skills" / GAUSS_DATA_ADAPTATION_SKILL_NAME
|
||||
target = _skills_public_dir() / GAUSS_DATA_ADAPTATION_SKILL_NAME
|
||||
if target.exists():
|
||||
return False
|
||||
if not (source / "SKILL.md").is_file():
|
||||
logger.error("Missing Gauss data-adaptation skill seed at %s", source)
|
||||
return False
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(source, target)
|
||||
logger.info("Seeded public skill '%s'", GAUSS_DATA_ADAPTATION_SKILL_NAME)
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("Failed to seed Gauss data-adaptation skill")
|
||||
return False
|
||||
|
||||
|
||||
def _ensure_agent() -> bool:
|
||||
try:
|
||||
target = get_paths().agent_dir(GAUSS_DATA_ADAPTATION_AGENT_ID)
|
||||
if target.exists():
|
||||
return False
|
||||
source = _ASSETS_DIR / GAUSS_DATA_ADAPTATION_AGENT_ID
|
||||
if not source.is_dir():
|
||||
logger.error("Missing Gauss data-adaptation agent seed at %s", source)
|
||||
return False
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
for filename in _AGENT_FILES:
|
||||
asset = source / filename
|
||||
if asset.is_file():
|
||||
(target / filename).write_bytes(asset.read_bytes())
|
||||
logger.info("Seeded built-in agent '%s'", GAUSS_DATA_ADAPTATION_AGENT_ID)
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception("Failed to seed Gauss data-adaptation agent")
|
||||
return False
|
||||
|
||||
|
||||
def ensure_gauss_data_adaptation_agent() -> bool:
|
||||
"""Install the Gauss data-adaptation skill and agent when missing."""
|
||||
_ensure_skill()
|
||||
return _ensure_agent()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"GAUSS_DATA_ADAPTATION_AGENT_ID",
|
||||
"GAUSS_DATA_ADAPTATION_SKILL_NAME",
|
||||
"ensure_gauss_data_adaptation_agent",
|
||||
]
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
你是高斯数据适配助手。用户会把内网高斯适配时的报错或现象发给你,你告诉他改哪里。
|
||||
|
||||
不要先给安装教程。不要连接高斯,不要让用户把密码、完整连接串或令牌发到对话里。沙箱里的配置只代表跑脚本的那台机器;用户说的是另一台内网服务器时,不要用沙箱里的 `config.yaml` 给那台机器下改法。
|
||||
|
||||
先读技能里的 `references/adaptation-map.md`,只命中一条。
|
||||
|
||||
1. 报错原文已经能对应地图里唯一一条时,直接写改法。例如缺少 `asyncpg` / `opengauss_sqlalchemy`、错误 322、`in-memory fallback`、`current_schema()` 不一致、`VARCHAR(n)` 被拒绝、`standard_conforming_strings` 冲突。
|
||||
2. 改法取决于「原生 openGauss」还是「Dolphin(MySQL 协议)」时,先确认模式,再给改法。能在出问题的那台 Gateway 上运行技能脚本时就运行;不能运行时,只追问 `database.backend` 和进程环境里有没有 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql`。
|
||||
3. 对不上地图时,要改的地方写「先不要改」。指出下一段必须贴来的日志,不要编造 `ALTER`、不要把检查点迁到高斯、不要让内网机器访问公网安装依赖。
|
||||
4. 用户只贴了 `Automatic ORM schema creation failed` 这一句时,改法还不能确定。向他要这句下面的异常类型和第一条 Gauss/MySQL/openGauss 报错,不要根据这句摘要改配置。
|
||||
5. 通用问答和智能体问答共用同一套业务表。列表不见,改业务库;列表还在、正文没有,改检查点。不要为这两种聊天各做一套适配。
|
||||
|
||||
每次回答用这五段,没有内容就写「无」:
|
||||
|
||||
现象
|
||||
判定模式
|
||||
要改的地方
|
||||
不要改的地方
|
||||
改完如何确认
|
||||
|
||||
要改的地方必须是用户能执行的一项:改哪一个配置字段、哪一个环境变量、换哪一个已在现场的镜像或离线包、或明确「先不要改」。改完如何确认要写出重启后日志里应出现或应消失的那一句。
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
id: gauss-data-adapter
|
||||
name: 高斯数据适配助手
|
||||
description: 内网把业务库适配到高斯后出现报错或异常现象时,根据报错说明要改的配置、环境变量或部署方式。不连接数据库,不收集密码。
|
||||
tool_groups:
|
||||
- file:read
|
||||
- bash
|
||||
skills:
|
||||
- gauss-data-adaptation
|
||||
common_questions:
|
||||
- title: 启动报缺少高斯驱动
|
||||
prompt: Gateway 启动报错,说 database.backend 是 gauss,但没有安装 asyncpg 或 opengauss 的 SQLAlchemy 方言。这是内网,不能访问公网。请告诉我要改哪里。
|
||||
- title: 进程起来了但建表失败
|
||||
prompt: Gateway 进程已经起来,日志里有 Automatic ORM schema creation failed。请根据这个现象告诉我要改什么,不要先给一整套安装步骤。
|
||||
- title: Dolphin 报错 322
|
||||
prompt: 高斯走 MySQL 协议(Dolphin)。启动或建表时报错 322。请告诉我要改哪一项配置或环境变量。
|
||||
- title: 列表还在,重启后正文没了
|
||||
prompt: 通用问答和智能体的会话列表还在,但重启 Gateway 后对话正文没了。日志里有 in-memory fallback。请告诉我该改检查点还是高斯业务库。
|
||||
- title: 时间差了 8 小时
|
||||
prompt: 业务库改到高斯之后,页面上的时间比北京时间差了 8 小时。请告诉我要改配置的哪一处,不要让我在 SQL 里手工加减 8 小时。
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
---
|
||||
name: gauss-data-adaptation
|
||||
description: 根据内网高斯适配报错判断要改的配置或环境变量。区分原生 openGauss 与 Dolphin,不连接数据库,不读取密码。
|
||||
---
|
||||
|
||||
# 高斯数据适配
|
||||
|
||||
用户贴来报错或现象后,先读 `references/adaptation-map.md`,只命中一条,按其中的改法回答。
|
||||
|
||||
报错已经能唯一对应时,直接给改法。只有改法取决于原生 openGauss 还是 Dolphin 时,才在**出问题的那台 Gateway** 上运行:
|
||||
|
||||
```bash
|
||||
python /mnt/skills/public/gauss-data-adaptation/scripts/print_gauss_adaptation.py
|
||||
```
|
||||
|
||||
沙箱里找不到该路径时:
|
||||
|
||||
```bash
|
||||
python scripts/print_gauss_adaptation.py
|
||||
```
|
||||
|
||||
脚本只读本机 `config.yaml` 和当前进程是否设置了 `DEERFLOW_DATABASE_PRODUCT`,并用 `find_spec` 查看驱动是否可导入。它不连接高斯,不读取 `$GAUSS_DATABASE_URL` 或 `$MYSQL_DATABASE_URL` 的值,输出里不保留账号和密码。沙箱不是出问题的那台机器时,把命令交给那台机器,不要把沙箱的判定当成那台机器的改法。
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
# 高斯适配报错对照
|
||||
|
||||
只命中一条。改完都要重启出问题的那个 Gateway 进程。环境变量写在启动该进程的服务配置里;在另一个 shell 里临时 export 不会影响已经启动的进程。
|
||||
|
||||
两种模式不能同时打开:
|
||||
|
||||
- 原生 openGauss:`database.backend: gauss`,`database.gauss_url: $GAUSS_DATABASE_URL`。不要设置 `DEERFLOW_DATABASE_PRODUCT`。
|
||||
- Dolphin(高斯的 MySQL 协议):`database.backend: mysql`,`database.mysql_url: $MYSQL_DATABASE_URL`,并且进程环境为 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql`。不要把 `backend` 改成 `gauss`。
|
||||
|
||||
口令只放在服务器的环境变量或客户端口令文件里。下面的改法都不包含密码。
|
||||
|
||||
固定约束:不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上。
|
||||
|
||||
## 缺驱动
|
||||
|
||||
现象:启动日志含 `asyncpg is not installed` 或 `openGauss SQLAlchemy dialect is not installed`。
|
||||
|
||||
模式:原生 openGauss。不需要再确认。
|
||||
|
||||
要改的地方:不要在内网执行会访问公网的 `uv sync --extra gauss`。换成构建时带 `--build-arg UV_EXTRAS=gauss` 的镜像,或使用现场离线 wheelhouse 里的 `asyncpg` 和 `opengauss-sqlalchemy`。装进当前环境后重启。
|
||||
|
||||
不要改的地方:不要为了绕过缺包把 `backend` 改成 `postgres` 或 `mysql`。
|
||||
|
||||
改完如何确认:这两句 ImportError 消失,日志出现 `Persistence engine initialized` 且 `dialect=opengauss`。
|
||||
|
||||
Dolphin 缺的是 MySQL 驱动。日志含 `MySQL async driver is not installed` 时,用离线 wheelhouse 把 `asyncmy` 装进 `.deer-flow/python-vendor`,不要从公网安装,也不要安装 `opengauss-sqlalchemy`。
|
||||
|
||||
## 误用通用 PostgreSQL 方言
|
||||
|
||||
现象:TCP 能连上高斯,但方言初始化或建表失败;或者脚本显示 `backend` 不是 `gauss`,连接串却指向高斯端口。
|
||||
|
||||
模式:应为原生 openGauss。
|
||||
|
||||
要改的地方:`database.backend` 改为 `gauss`,`gauss_url` 使用 `opengauss://`、`opengauss+asyncpg://`、`gaussdb://` 或 PostgreSQL 风格 URL。启动时会归一化成 `opengauss+asyncpg`。取消 `DEERFLOW_DATABASE_PRODUCT`。
|
||||
|
||||
不要改的地方:不要手改 SQLAlchemy 的 PostgreSQL 方言,不要把这条 URL 同时填进 `checkpointer.connection_string`。
|
||||
|
||||
改完如何确认:`dialect=opengauss`。
|
||||
|
||||
## VARCHAR(n) 被拒绝
|
||||
|
||||
现象:建表报错指向 `VARCHAR(数字)`。
|
||||
|
||||
模式:原生 openGauss。不需要再确认。
|
||||
|
||||
要改的地方:确认进程确实以 `backend: gauss` 启动,从而装上把 `VARCHAR(n)` 编译成不带长度的 `VARCHAR` 的兼容层。驱动按上面的「缺驱动」处理。不要在库里把列改成某种固定长度来迁就这条 SQL。
|
||||
|
||||
不要改的地方:不要改成 Dolphin,也不要删业务列。
|
||||
|
||||
改完如何确认:同一条建表错误消失,`dialect=opengauss`。
|
||||
|
||||
## 错误 322,或 SHOW CREATE TABLE 解析失败
|
||||
|
||||
现象:日志含 `322`,或 MySQL 反射、`SHOW CREATE TABLE`、重复 `ALTER` 导致事务中止。
|
||||
|
||||
模式:Dolphin。不需要再确认。
|
||||
|
||||
要改的地方:保证**正在运行的进程**有 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql`,`database.backend` 保持 `mysql`。设置后重启该进程。不要再执行 MySQL 的 `SHOW CREATE TABLE` 反射、MySQL JSON 回填、MySQL 会话超时变量,也不要让应用执行 `CREATE DATABASE`。
|
||||
|
||||
不要改的地方:不要把 `backend` 改成 `gauss` 来消掉 322。库、用户和 schema 仍由离线部署层创建。
|
||||
|
||||
改完如何确认:日志出现 `GaussDB MySQL compatibility enabled` 和 `Skipping MySQL reflection and JSON backfill for openGauss Dolphin`,不再出现 322。
|
||||
|
||||
## init_command 冲突
|
||||
|
||||
现象:日志含 `gauss_mysql requires init_command` 或 `standard_conforming_strings`。
|
||||
|
||||
模式:Dolphin。
|
||||
|
||||
要改的地方:从 `mysql_url` 里去掉冲突的 `init_command`。兼容层会在连接时设置 `SET standard_conforming_strings TO off`。
|
||||
|
||||
不要改的地方:不要在 URL 里写另一条 `init_command`,不要把口令贴进对话。
|
||||
|
||||
改完如何确认:这句 ValueError 消失,并出现 `GaussDB MySQL compatibility enabled`。
|
||||
|
||||
## current_schema() 对不上
|
||||
|
||||
现象:日志含 `URL schema` 与 `current_schema()` 不一致,或 `current_schema() is empty`。
|
||||
|
||||
模式:Dolphin 的启动补列。
|
||||
|
||||
要改的地方:把 `mysql_url` 里的库名改成 `SELECT current_schema();` 的结果,或让部署层把该登录用户的当前 schema 改成 URL 里的库名。两边必须是同一个名字。
|
||||
|
||||
不要改的地方:不要用 `SHOW CREATE TABLE` 补列,不要由应用 `CREATE DATABASE`。
|
||||
|
||||
改完如何确认:这句 schema integrity check 消失。若随后出现 `openGauss schema is incomplete`,按下一条处理,不要提前改别的配置。
|
||||
|
||||
## 缺列但启动拒绝补列
|
||||
|
||||
现象:日志含 `openGauss schema is incomplete`,以及 `NOT NULL column has no server_default`、主键、identity、computed 或约束需要 explicit migration。
|
||||
|
||||
模式:Dolphin。
|
||||
|
||||
要改的地方:先不要改。把日志里点名的 `schema.表.列` 原样留下。这种列不能靠启动自动补。需要一份针对该列的显式迁移,而不是在对话里手写 `ALTER TABLE`。
|
||||
|
||||
不要改的地方:不要删表、不要把列改成可空来让启动继续。
|
||||
|
||||
改完如何确认:同一句 `schema is incomplete` 消失。只为日志点名的那一列做迁移。
|
||||
|
||||
## 只看到建表失败摘要
|
||||
|
||||
现象:只有 `Automatic ORM schema creation failed; continuing Gateway startup`。
|
||||
|
||||
模式:还不能判定。
|
||||
|
||||
要改的地方:先不要改。把这句下面的异常类型和第一条数据库报错贴出来,再按本表重新命中。
|
||||
|
||||
不要改的地方:不要因为进程还活着就认为适配已经成功。缺表的接口仍会失败。
|
||||
|
||||
改完如何确认:重启后这句消失;若仍在,新贴出的第一条数据库报错必须变了,才能进入下一条。
|
||||
|
||||
## 列表还在,重启后正文没了
|
||||
|
||||
现象:通用问答或智能体的会话列表还在,重启后正文没有;日志含 `LangGraph checkpointer initialization failed` 和 `in-memory fallback`。
|
||||
|
||||
模式:与原生或 Dolphin 无关。列表来自业务库,正文来自检查点。
|
||||
|
||||
要改的地方:修复日志里检查点初始化的那条异常。检查点使用现有 SQLite 文件,或一台真正的 PostgreSQL。不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上。若 `checkpointer.type` 已是 `postgres` 且 `use_postgres: false`,实际文件是 `sqlite_connection_string`,改这个路径的权限或位置,不要改高斯 URL。
|
||||
|
||||
不要改的地方:不要重导 `threads_meta`,不要改 `assistant_id`。两种聊天的正文都在同一套检查点里。
|
||||
|
||||
改完如何确认:`in-memory fallback` 消失。重启后再打开刚才那条通用问答和那条智能体问答,正文仍在。
|
||||
|
||||
## 时间差 8 小时
|
||||
|
||||
现象:库里或页面上的时间比北京时间差 8 小时,业务库刚刚改到高斯。
|
||||
|
||||
先确认模式。
|
||||
|
||||
- 原生 openGauss 却把 `backend` 配成 `mysql` 或 `postgres`:改回上一节的原生配置。`opengauss` 与 `postgresql` 走带时区的时间类型,不在写入时改成北京墙钟。
|
||||
- Dolphin 却没设 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql`,或误设成 `backend: gauss`:改回 Dolphin 配置。Dolphin 按 MySQL 方言把 UTC 存成北京墙钟。
|
||||
- 模式已经配对,只是客户端用另一套时区显示:先不要改 DeerFlow 配置,也不要在 SQL 里加减 8 小时。
|
||||
|
||||
改完如何确认:新写入的一行与页面显示一致。不要批量改历史时间列。
|
||||
|
||||
## JSON 读失败
|
||||
|
||||
现象:日志含 `GaussDB MySQL JSON value is neither valid JSON`,或普通 JSON 解析失败。
|
||||
|
||||
- 这句完整报错只出现在 Dolphin 兼容层已经装上、但库里的值不是严格 JSON、也不是已支持的旧字面量时:先不要改配置。把报错里的预览留给库侧,不要改用另一种解码把坏值读成对象。
|
||||
- 同样的数据在未设置 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql` 时解析失败:按 Dolphin 配置补上该环境变量并重启。普通 MySQL 必须继续拒绝坏 JSON,不要为了读旧数据去设置这个变量。
|
||||
|
||||
改完如何确认:Dolphin 应出现 `GaussDB MySQL compatibility enabled`。坏值那一行仍失败时,失败原因应仍是这句 JSON 报错,而不是驱动或方言报错。
|
||||
|
||||
## 两种模式同时打开
|
||||
|
||||
现象:`backend: gauss`,同时进程环境是 `DEERFLOW_DATABASE_PRODUCT=gauss_mysql`。
|
||||
|
||||
要改的地方:只留一种。原生就删掉该环境变量;Dolphin 就把 `backend` 改回 `mysql` 并保留该环境变量。
|
||||
|
||||
不要改的地方:不要两套 URL 都填上指望自动选择。
|
||||
|
||||
改完如何确认:脚本的「判定模式」只剩一种,启动日志里的 `dialect` 与之相符:原生是 `opengauss`,Dolphin 是 `mysql`。
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
"""Classify the local Gauss adaptation mode without connecting or reading secrets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from importlib.util import find_spec
|
||||
from pathlib import Path
|
||||
|
||||
PROHIBITION = "不要把 LangGraph 的 PostgreSQL checkpointer 迁移执行到高斯库上"
|
||||
DOLPHIN_PRODUCT = "gauss_mysql"
|
||||
GAUSS_URL_PREFIXES = (
|
||||
"opengauss+asyncpg://",
|
||||
"opengauss://",
|
||||
"gaussdb://",
|
||||
"postgres://",
|
||||
"postgresql://",
|
||||
"postgresql+asyncpg://",
|
||||
)
|
||||
|
||||
|
||||
def config_candidates() -> list[Path]:
|
||||
found: list[Path] = []
|
||||
for key in ("DEER_FLOW_CONFIG_PATH", "DEER_FLOW_CONFIG", "DEERFLOW_CONFIG"):
|
||||
value = os.environ.get(key, "").strip()
|
||||
if value:
|
||||
found.append(Path(value))
|
||||
found.extend(
|
||||
[
|
||||
Path("/app/backend/config.yaml"),
|
||||
Path.cwd() / "config.yaml",
|
||||
Path.cwd().parent / "config.yaml",
|
||||
]
|
||||
)
|
||||
return found
|
||||
|
||||
|
||||
def parse_sections(text: str, names: set[str]) -> dict[str, dict[str, str]]:
|
||||
"""Read one level of scalar keys from the named top-level YAML blocks."""
|
||||
found = {name: {} for name in names}
|
||||
current = ""
|
||||
for raw in text.splitlines():
|
||||
line = raw.split("#", 1)[0].rstrip()
|
||||
if not line.strip():
|
||||
continue
|
||||
if not line.startswith((" ", "\t")):
|
||||
header = line.strip()
|
||||
current = header[:-1] if header.endswith(":") else ""
|
||||
if current not in names:
|
||||
current = ""
|
||||
continue
|
||||
if not current or ":" not in line:
|
||||
continue
|
||||
key, _, value = line.strip().partition(":")
|
||||
value = value.strip().strip("\"'")
|
||||
if value:
|
||||
found[current][key.strip()] = value
|
||||
return found
|
||||
|
||||
|
||||
def describe_value(value: str) -> str:
|
||||
"""Describe a config value without usernames, passwords, or env-var contents."""
|
||||
text = value.strip().strip("\"'")
|
||||
if not text:
|
||||
return "未设置"
|
||||
if text.startswith("$"):
|
||||
return f"环境变量 {text[1:]}(不读取变量值)"
|
||||
if "://" not in text:
|
||||
return text
|
||||
scheme, _, rest = text.partition("://")
|
||||
target = rest.split("@")[-1].split("?")[0]
|
||||
return f"{scheme}://{target}(已去掉账号和密码)"
|
||||
|
||||
|
||||
def driver_status() -> dict[str, bool]:
|
||||
names = ("asyncpg", "opengauss_sqlalchemy", "asyncmy", "aiomysql")
|
||||
return {name: find_spec(name) is not None for name in names}
|
||||
|
||||
|
||||
def _enabled(value: str) -> bool:
|
||||
return value.strip().lower() not in {"", "false", "0", "no", "off"}
|
||||
|
||||
|
||||
def classify(
|
||||
database: dict[str, str],
|
||||
checkpointer: dict[str, str],
|
||||
product: str,
|
||||
drivers: dict[str, bool],
|
||||
) -> list[str]:
|
||||
"""Return the mode and the concrete config change, if one is already known."""
|
||||
backend = database.get("backend", "").strip().lower()
|
||||
product_name = product.strip().lower()
|
||||
checkpoint_type = checkpointer.get("type", "").strip().lower()
|
||||
use_postgres = checkpointer.get("use_postgres", "")
|
||||
lines = [
|
||||
f"database.backend={backend or '未解析'}",
|
||||
f"DEERFLOW_DATABASE_PRODUCT={product_name or '未设置'}",
|
||||
f"checkpointer.type={checkpoint_type or '未解析'}",
|
||||
]
|
||||
if "gauss_url" in database:
|
||||
lines.append(f"database.gauss_url={describe_value(database['gauss_url'])}")
|
||||
if "mysql_url" in database:
|
||||
lines.append(f"database.mysql_url={describe_value(database['mysql_url'])}")
|
||||
if "connection_string" in checkpointer:
|
||||
lines.append(
|
||||
"checkpointer.connection_string="
|
||||
+ describe_value(checkpointer["connection_string"])
|
||||
)
|
||||
if use_postgres:
|
||||
lines.append(f"checkpointer.use_postgres={use_postgres}")
|
||||
for name in ("asyncpg", "opengauss_sqlalchemy", "asyncmy", "aiomysql"):
|
||||
lines.append(f"{name}={'可导入' if drivers.get(name) else '不可导入'}")
|
||||
|
||||
changes: list[str] = []
|
||||
constraints: list[str] = []
|
||||
if backend == "gauss" and product_name == DOLPHIN_PRODUCT:
|
||||
mode = "冲突:原生 openGauss 与 Dolphin 同时打开"
|
||||
changes.append(
|
||||
"只保留一种。原生就取消 DEERFLOW_DATABASE_PRODUCT;"
|
||||
"Dolphin 就把 database.backend 改回 mysql,并保留该环境变量。"
|
||||
)
|
||||
elif backend == "gauss":
|
||||
mode = "原生 openGauss"
|
||||
url = database.get("gauss_url", "")
|
||||
if not url:
|
||||
changes.append("补上 database.gauss_url: $GAUSS_DATABASE_URL。口令只放在该环境变量里。")
|
||||
elif not url.startswith("$") and not url.startswith(GAUSS_URL_PREFIXES):
|
||||
changes.append(
|
||||
"把 gauss_url 改成 opengauss://、opengauss+asyncpg://、gaussdb:// 或 PostgreSQL 风格 URL。"
|
||||
)
|
||||
elif not url.startswith("$") and not url.startswith("opengauss+asyncpg://"):
|
||||
constraints.append("这份 gauss_url 启动时会归一化为 opengauss+asyncpg,不会走通用 PostgreSQL 方言。")
|
||||
if not drivers.get("asyncpg") or not drivers.get("opengauss_sqlalchemy"):
|
||||
changes.append(
|
||||
"当前进程缺少 asyncpg 或 opengauss_sqlalchemy。"
|
||||
"不要在内网访问公网安装。改用带 UV_EXTRAS=gauss 的镜像,或现场离线 wheelhouse。"
|
||||
)
|
||||
constraints.append("不要设置 DEERFLOW_DATABASE_PRODUCT。")
|
||||
elif backend == "mysql" and product_name == DOLPHIN_PRODUCT:
|
||||
mode = "Dolphin(高斯 MySQL 协议)"
|
||||
if not database.get("mysql_url"):
|
||||
changes.append("补上 database.mysql_url: $MYSQL_DATABASE_URL。口令只放在该环境变量里。")
|
||||
if not drivers.get("asyncmy") and not drivers.get("aiomysql"):
|
||||
changes.append(
|
||||
"当前进程缺少 asyncmy 和 aiomysql。用离线 wheelhouse 把 asyncmy 装进 "
|
||||
".deer-flow/python-vendor,不要从公网安装。"
|
||||
)
|
||||
constraints.append(
|
||||
"不要执行 SHOW CREATE TABLE、MySQL JSON 回填或应用侧 CREATE DATABASE。"
|
||||
"URL 里的库名必须等于 SELECT current_schema()。"
|
||||
)
|
||||
elif backend == "mysql":
|
||||
mode = "普通 MySQL"
|
||||
constraints.append(
|
||||
"现在不会安装 Dolphin 补丁。只有报错已经表现为 322、current_schema 或 "
|
||||
"SHOW CREATE TABLE 解析失败时,才给运行中的进程加上 "
|
||||
"DEERFLOW_DATABASE_PRODUCT=gauss_mysql 并重启。普通 MySQL 不要加。"
|
||||
)
|
||||
elif backend == "postgres":
|
||||
mode = "PostgreSQL"
|
||||
changes.append("这不是高斯适配路径。连高斯业务库时不要使用 backend: postgres。")
|
||||
elif backend in {"sqlite", "memory"}:
|
||||
mode = backend
|
||||
constraints.append("当前业务库配置不是高斯。这只代表运行脚本的机器。")
|
||||
else:
|
||||
mode = "未识别"
|
||||
changes.append("先不要改。确认 database.backend 是 gauss、mysql、postgres、sqlite 还是 memory。")
|
||||
|
||||
postgres_checkpoint = checkpoint_type == "postgres" and _enabled(use_postgres or "true")
|
||||
if backend in {"gauss", "mysql"} and postgres_checkpoint and product_name == DOLPHIN_PRODUCT:
|
||||
changes.append(
|
||||
"先确认 checkpointer.connection_string 是不是高斯。"
|
||||
+ PROHIBITION
|
||||
+ " 是高斯就把 checkpointer.type 改为 sqlite,并沿用现有 SQLite 文件;是独立的 PostgreSQL 就不要改。"
|
||||
)
|
||||
elif backend == "gauss" and postgres_checkpoint:
|
||||
changes.append(
|
||||
"先确认 checkpointer.connection_string 是不是高斯。"
|
||||
+ PROHIBITION
|
||||
+ " 是高斯就把 checkpointer.type 改为 sqlite,并沿用现有 SQLite 文件;是独立的 PostgreSQL 就不要改。"
|
||||
)
|
||||
elif checkpoint_type == "memory" and backend in {"gauss", "mysql"}:
|
||||
changes.append("把 checkpointer.type 从 memory 改为 sqlite,否则重启后问答正文不会保留。")
|
||||
|
||||
lines.append(f"判定模式:{mode}")
|
||||
if constraints:
|
||||
lines.append("先不要改:")
|
||||
lines.extend(f"- {item}" for item in constraints)
|
||||
lines.append("要改的地方:")
|
||||
if changes:
|
||||
lines.extend(f"- {item}" for item in changes)
|
||||
else:
|
||||
lines.append("- 配置层面没有必须修改的项。把报错原文发来后再决定。")
|
||||
lines.append(f"固定约束:{PROHIBITION}")
|
||||
return lines
|
||||
|
||||
|
||||
def main() -> None:
|
||||
print("这个结果只代表运行脚本的机器,不连接高斯,也不读取连接串环境变量的值。")
|
||||
chosen = next((item for item in config_candidates() if item.is_file()), None)
|
||||
if chosen is None:
|
||||
database: dict[str, str] = {}
|
||||
checkpointer: dict[str, str] = {}
|
||||
print("配置:未找到 config.yaml。")
|
||||
else:
|
||||
print(f"配置文件:{chosen}")
|
||||
try:
|
||||
parsed = parse_sections(
|
||||
chosen.read_text(encoding="utf-8"),
|
||||
{"database", "checkpointer"},
|
||||
)
|
||||
except OSError as exc:
|
||||
print(f"配置:读取失败:{exc}")
|
||||
parsed = {"database": {}, "checkpointer": {}}
|
||||
database = parsed["database"]
|
||||
checkpointer = parsed["checkpointer"]
|
||||
for line in classify(
|
||||
database,
|
||||
checkpointer,
|
||||
os.environ.get("DEERFLOW_DATABASE_PRODUCT", ""),
|
||||
driver_status(),
|
||||
):
|
||||
print(line)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+2
-2
@@ -21,7 +21,7 @@
|
||||
## 页面生成流程
|
||||
|
||||
1. 先读取 `page-report-generation` 技能,并将材料归为:事实数据、可推导结论、待补充项、图表数据、长文本、表格行、行动/风险项。优先保留用户原文的关键判断和数据口径。
|
||||
2. 默认从零直接创建 `/mnt/user-data/outputs/<主题>.html`,第一项文件操作就是写最终路径。只有用户明确要求使用模板或点名模板时,才读取 `page-template-library` 并调用 `page_report_template(action="prepare")`;主题优先使用界面传入的 `presentation_theme`,其次遵从用户明示。
|
||||
2. 默认从零直接创建 `/mnt/user-data/outputs/<主题>.html`,第一项文件操作就是写最终路径。只有用户明确要求使用模板或点名模板时,才读取 `page-template-library` 并调用 `page_report_template(action="prepare")`;用户明示的页面主题优先,否则只跟随界面传入的 `presentation_theme` 明暗模式。
|
||||
3. 页面文字通常要多于图表:至少包含结论摘要、关键发现、事项/风险说明、数据口径或来源、行动建议中的多个展示样式。默认生成一个可纵向滚动的响应式页面;只有用户明确要求多页或给出页数时才生成多页。
|
||||
4. 使用模板时,工具返回的 outputs 工作副本是唯一成品路径。保留已有图表渲染和本地运行时契约,用真实数据替换占位内容,并删除无数据页面和面板;绝不修改模板源文件。
|
||||
5. 所有可见正文、表头、标签、说明文字最小字号均为 16px;长文本面板要增高、提高行高,不能把段落挤成一团。
|
||||
@@ -29,7 +29,7 @@
|
||||
|
||||
## 设计要求
|
||||
|
||||
- 跟随 `presentation_theme`:`dark` 选深蓝骨架,`light` 选浅色骨架;不要把产品现有聊天页面风格照搬进汇报页面。
|
||||
- 用户未指定页面主题时跟随 `presentation_theme`:`dark` 选深色骨架,`light` 选浅色骨架;不要复制系统的红、蓝、绿等主题主色,页面自己的强调色根据内容、信息层级和可读性独立设计,也不要把产品现有聊天页面风格照搬进汇报页面。
|
||||
- 深色使用深蓝科技感,选中状态可有细腻的扫描光、描边或流光;浅色使用清晰留白与高对比文字。
|
||||
- 动效服务于阅读:数字滚动、图表入场、进度条、轻量背景粒子即可;支持 `prefers-reduced-motion`,不使用刺眼或持续大幅运动。
|
||||
- 图表必须与材料一致。数据不够时宁可删掉图表或明确标为示意,不伪造事实;避免为了“丰富”而堆满图。
|
||||
|
||||
+1
-1
@@ -15,7 +15,7 @@ allowed-tools:
|
||||
## 使用流程
|
||||
|
||||
1. 先盘点材料和输出约束,确定:
|
||||
- `theme`:优先使用界面传入的 `presentation_theme`,否则遵从用户明示;仍未指定时使用 `auto`。
|
||||
- `theme`:用户明示的页面主题优先;否则使用界面传入的 `presentation_theme`。`auto` 会由工具按当前界面明暗模式解析,界面信息缺失时才回退浅色。
|
||||
- `content_density`:`text-heavy`、`balanced` 或 `data-heavy`。
|
||||
- `required_features`:如 `flowchart`、`chart`、`map`、`table`、`text-panels`;源 Markdown 中的 Mermaid 必须记为 `flowchart`。
|
||||
- `preferred_page_count`:默认 `1`;只有用户明确要求多页或指定页数时才大于 `1`。
|
||||
|
||||
@@ -318,6 +318,20 @@ class AdminLeaderboardResponse(BaseModel):
|
||||
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AdminUserAnalyticsResponse(BaseModel):
|
||||
range_days: int
|
||||
since: str
|
||||
until: str
|
||||
user: AdminLeaderboardUser
|
||||
overview: AdminLeaderboardOverview
|
||||
skills: list[AdminLeaderboardSkill] = Field(default_factory=list)
|
||||
tools: list[AdminLeaderboardSkill] = Field(default_factory=list)
|
||||
models: list[AdminLeaderboardDimension] = Field(default_factory=list)
|
||||
agents: list[AdminLeaderboardDimension] = Field(default_factory=list)
|
||||
run_statuses: list[AdminLeaderboardDimension] = Field(default_factory=list)
|
||||
trends: list[AdminLeaderboardTrendPoint] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AdminLeaderboardReportResponse(BaseModel):
|
||||
report: str
|
||||
generated_at: str
|
||||
@@ -2224,6 +2238,67 @@ async def stream_user_leaderboard_report(
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{user_id}/analytics", response_model=AdminUserAnalyticsResponse)
|
||||
async def get_user_analytics(
|
||||
user_id: str,
|
||||
request: Request,
|
||||
days: int = Query(default=30, ge=1, le=ADMIN_QUERY_WINDOW_DAYS),
|
||||
since: str | None = Query(default=None, description="Inclusive range start, ISO datetime"),
|
||||
until: str | None = Query(default=None, description="Inclusive range end, ISO datetime"),
|
||||
limit: int = Query(default=20, ge=5, le=100),
|
||||
) -> AdminUserAnalyticsResponse:
|
||||
"""Return one user's admin analytics profile."""
|
||||
await _require_admin(request)
|
||||
session_factory = get_session_factory()
|
||||
if session_factory is None:
|
||||
raise HTTPException(status_code=503, detail="Analytics requires database persistence")
|
||||
|
||||
settings = load_system_settings().leaderboard
|
||||
common = {
|
||||
"include_scheduled": settings.include_scheduled,
|
||||
"include_admins": settings.include_admins,
|
||||
"include_failed": settings.include_failed,
|
||||
}
|
||||
tool_common = {
|
||||
"include_scheduled": settings.include_scheduled,
|
||||
"include_admins": settings.include_admins,
|
||||
}
|
||||
since_dt, until_dt, range_days = resolve_admin_query_window(
|
||||
days=days,
|
||||
since=since,
|
||||
until=until,
|
||||
)
|
||||
async with session_factory() as session:
|
||||
await _set_analytics_read_timeout(session)
|
||||
users = await _query_user_leaderboard(session, since_dt, until_dt, user_id=user_id, **common)
|
||||
if not users:
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
overview = await _query_overview(session, since_dt, until_dt, user_id=user_id, **common)
|
||||
skills = await _query_call_rankings(session, since_dt, until_dt, user_id=user_id, field="skill", limit=limit, **tool_common)
|
||||
tools = await _query_call_rankings(session, since_dt, until_dt, user_id=user_id, field="tool", limit=limit, **tool_common)
|
||||
models = await _query_model_rankings(session, since_dt, until_dt, user_id=user_id, limit=limit, **common)
|
||||
if not models:
|
||||
models = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="model", limit=limit, **common)
|
||||
agents = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="agent", limit=limit, **common)
|
||||
run_statuses = await _query_run_dimensions(session, since_dt, until_dt, user_id=user_id, field="status", limit=limit, **common)
|
||||
trends = await _query_trends(session, since_dt, until_dt, user_id=user_id, **common)
|
||||
|
||||
return AdminUserAnalyticsResponse(
|
||||
range_days=range_days,
|
||||
since=_iso(since_dt),
|
||||
until=_iso(until_dt),
|
||||
user=users[0],
|
||||
overview=overview,
|
||||
skills=skills,
|
||||
tools=tools,
|
||||
models=models,
|
||||
agents=agents,
|
||||
run_statuses=run_statuses,
|
||||
trends=trends,
|
||||
)
|
||||
|
||||
|
||||
|
||||
# ── Q&A audit list ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
"""Admin-only metadata migration for agent/skill folders copied in place."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.agent_skill_migration import (
|
||||
MigrationApplyResult,
|
||||
MigrationManifest,
|
||||
MigrationPreflight,
|
||||
apply_migration_manifest,
|
||||
build_agent_migration_manifest,
|
||||
build_skill_migration_manifest,
|
||||
preflight_manifest,
|
||||
prepare_manifest_for_apply,
|
||||
)
|
||||
from app.gateway.deps import (
|
||||
get_agent_store,
|
||||
get_local_provider,
|
||||
get_optional_user_from_request,
|
||||
get_skill_store,
|
||||
get_tag_store,
|
||||
)
|
||||
from deerflow.agents.lead_agent.prompt import refresh_skills_system_prompt_cache_async
|
||||
from deerflow.config.extensions_config import ExtensionsConfig
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.persistence.agents import AgentStore
|
||||
from deerflow.persistence.skills import SkillStore
|
||||
from deerflow.persistence.tags import TagStore
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.sandbox.skill_view import refresh_skill_sandbox_view
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.usage import all_entries as get_skill_usage_entries
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/admin/agent-skill-migrations", tags=["agent-skill-migrations"])
|
||||
|
||||
|
||||
class MigrationExportRequest(BaseModel):
|
||||
agent_ids: list[str] = Field(min_length=1, max_length=1000)
|
||||
|
||||
|
||||
class SkillMigrationExportRequest(BaseModel):
|
||||
skill_names: list[str] = Field(min_length=1, max_length=5000)
|
||||
|
||||
|
||||
class MigrationApplyRequest(BaseModel):
|
||||
manifest: MigrationManifest
|
||||
use_current_admin_for_unresolved_owners: bool = False
|
||||
skip_invalid_skills: bool = False
|
||||
|
||||
|
||||
class MigrationRootsResponse(BaseModel):
|
||||
agents_root: str
|
||||
skills_root: str
|
||||
extensions_config_path: str | None
|
||||
|
||||
|
||||
async def _require_admin(request: Request):
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is not None and getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="智能体与技能迁移仅限管理员")
|
||||
return user
|
||||
|
||||
|
||||
def _require_migration_enabled() -> None:
|
||||
if not load_system_settings().agent_skill_migration.enabled:
|
||||
raise HTTPException(status_code=403, detail="智能体与技能迁移功能未开启")
|
||||
|
||||
|
||||
async def _users() -> tuple[dict[str, str], dict[str, str]]:
|
||||
"""Return ``(id -> email, normalized email -> id)`` without auth secrets."""
|
||||
|
||||
try:
|
||||
users = await get_local_provider().list_users()
|
||||
except Exception: # noqa: BLE001 - metadata migration degrades to manual owner fallback
|
||||
logger.warning("Could not load users for migration owner mapping", exc_info=True)
|
||||
return {}, {}
|
||||
by_id = {str(user.id): str(user.email) for user in users}
|
||||
by_email = {str(user.email).strip().casefold(): str(user.id) for user in users}
|
||||
return by_id, by_email
|
||||
|
||||
|
||||
def _roots() -> tuple[Path, Path]:
|
||||
storage = get_or_new_skill_storage()
|
||||
return get_paths().agents_dir.resolve(), Path(storage.get_skills_root_path()).resolve()
|
||||
|
||||
|
||||
@router.get("/roots", response_model=MigrationRootsResponse)
|
||||
async def migration_roots(request: Request) -> MigrationRootsResponse:
|
||||
await _require_admin(request)
|
||||
_require_migration_enabled()
|
||||
agents_root, skills_root = _roots()
|
||||
extensions_path = ExtensionsConfig.resolve_config_path()
|
||||
return MigrationRootsResponse(
|
||||
agents_root=str(agents_root),
|
||||
skills_root=str(skills_root),
|
||||
extensions_config_path=str(extensions_path.resolve()) if extensions_path else None,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/export", response_model=MigrationManifest)
|
||||
async def export_migration_manifest(
|
||||
body: MigrationExportRequest,
|
||||
request: Request,
|
||||
agent_store: AgentStore = Depends(get_agent_store),
|
||||
tag_store: TagStore = Depends(get_tag_store),
|
||||
) -> MigrationManifest:
|
||||
await _require_admin(request)
|
||||
_require_migration_enabled()
|
||||
agents_root, _ = _roots()
|
||||
users_by_id, _ = await _users()
|
||||
try:
|
||||
return await build_agent_migration_manifest(
|
||||
body.agent_ids,
|
||||
agent_store=agent_store,
|
||||
tag_store=tag_store,
|
||||
agents_root=agents_root,
|
||||
users_by_id=users_by_id,
|
||||
)
|
||||
except (FileNotFoundError, ValueError) as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/export-skills", response_model=MigrationManifest)
|
||||
async def export_skill_migration_manifest(
|
||||
body: SkillMigrationExportRequest,
|
||||
request: Request,
|
||||
skill_store: SkillStore = Depends(get_skill_store),
|
||||
tag_store: TagStore = Depends(get_tag_store),
|
||||
) -> MigrationManifest:
|
||||
await _require_admin(request)
|
||||
_require_migration_enabled()
|
||||
_, skills_root = _roots()
|
||||
storage = get_or_new_skill_storage()
|
||||
users_by_id, _ = await _users()
|
||||
discovered_skills = await asyncio.to_thread(storage.load_skills, enabled_only=False)
|
||||
try:
|
||||
return await build_skill_migration_manifest(
|
||||
body.skill_names,
|
||||
skill_store=skill_store,
|
||||
tag_store=tag_store,
|
||||
skills_root=skills_root,
|
||||
discovered_skills=discovered_skills,
|
||||
users_by_id=users_by_id,
|
||||
usage_entries=get_skill_usage_entries(),
|
||||
)
|
||||
except (FileNotFoundError, ValueError) as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.post("/preflight", response_model=MigrationPreflight)
|
||||
async def preflight_migration_manifest(
|
||||
manifest: MigrationManifest,
|
||||
request: Request,
|
||||
) -> MigrationPreflight:
|
||||
await _require_admin(request)
|
||||
_require_migration_enabled()
|
||||
agents_root, skills_root = _roots()
|
||||
_, users_by_email = await _users()
|
||||
return await asyncio.to_thread(
|
||||
preflight_manifest,
|
||||
manifest,
|
||||
agents_root=agents_root,
|
||||
skills_root=skills_root,
|
||||
users_by_email=users_by_email,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/apply", response_model=MigrationApplyResult)
|
||||
async def apply_migration(
|
||||
body: MigrationApplyRequest,
|
||||
request: Request,
|
||||
agent_store: AgentStore = Depends(get_agent_store),
|
||||
skill_store: SkillStore = Depends(get_skill_store),
|
||||
tag_store: TagStore = Depends(get_tag_store),
|
||||
) -> MigrationApplyResult:
|
||||
user = await _require_admin(request)
|
||||
_require_migration_enabled()
|
||||
agents_root, skills_root = _roots()
|
||||
_, users_by_email = await _users()
|
||||
# Re-scan at apply time. The browser's earlier preflight is informative,
|
||||
# never an authorization or integrity token.
|
||||
preflight = await asyncio.to_thread(
|
||||
preflight_manifest,
|
||||
body.manifest,
|
||||
agents_root=agents_root,
|
||||
skills_root=skills_root,
|
||||
users_by_email=users_by_email,
|
||||
)
|
||||
try:
|
||||
effective_manifest, effective_preflight, _ = prepare_manifest_for_apply(
|
||||
body.manifest,
|
||||
preflight,
|
||||
skip_invalid_skills=body.skip_invalid_skills,
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
if body.manifest.skills and not effective_manifest.skills:
|
||||
raise HTTPException(status_code=409, detail="所有技能均未通过预检,没有可导入的技能")
|
||||
resource_failures = [item.model_dump(mode="json") for item in [*effective_preflight.agents, *effective_preflight.skills, *effective_preflight.support_directories] if item.status != "matched"]
|
||||
unresolved = [item for item in effective_preflight.owner_mappings if item.status == "unresolved"]
|
||||
if resource_failures or (unresolved and not body.use_current_admin_for_unresolved_owners):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"message": "迁移预检未通过,未写入任何元数据",
|
||||
"preflight": effective_preflight.model_dump(mode="json"),
|
||||
},
|
||||
)
|
||||
|
||||
extensions_path = ExtensionsConfig.resolve_config_path() if effective_manifest.skills else None
|
||||
if effective_manifest.skills and extensions_path is None:
|
||||
raise HTTPException(status_code=503, detail="目标运行环境未找到 extensions_config.json")
|
||||
fallback_owner_user_id = None
|
||||
if unresolved and body.use_current_admin_for_unresolved_owners:
|
||||
fallback_owner_user_id = str(user.id) if user is not None else get_effective_user_id()
|
||||
|
||||
try:
|
||||
result = await apply_migration_manifest(
|
||||
body.manifest,
|
||||
preflight=preflight,
|
||||
agent_store=agent_store,
|
||||
skill_store=skill_store,
|
||||
tag_store=tag_store,
|
||||
agents_root=agents_root,
|
||||
users_by_email=users_by_email,
|
||||
fallback_owner_user_id=fallback_owner_user_id,
|
||||
extensions_config_path=extensions_path,
|
||||
skip_invalid_skills=body.skip_invalid_skills,
|
||||
)
|
||||
if result.skills_applied:
|
||||
await refresh_skills_system_prompt_cache_async()
|
||||
storage = get_or_new_skill_storage()
|
||||
await asyncio.to_thread(refresh_skill_sandbox_view, storage=storage, force=True)
|
||||
return result
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,700 @@
|
||||
"""Account-scoped chat history transfer for the conversation list page.
|
||||
|
||||
Exports and imports ordinary / agent chats as a JSON bundle that includes
|
||||
the latest checkpoint (messages) and the thread ``user-data`` files
|
||||
(uploads + outputs). One oversized or broken thread is skipped so the
|
||||
rest of the batch can still succeed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from langgraph.checkpoint.base import empty_checkpoint
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.authz import require_permission
|
||||
from app.gateway.deps import get_checkpointer, get_thread_store
|
||||
from app.gateway.routers.threads import _strip_reserved_metadata
|
||||
from app.gateway.utils import sanitize_log_param
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.persistence.thread_meta.base import is_excluded_system_thread
|
||||
from deerflow.runtime import serialize_channel_values
|
||||
from deerflow.runtime.user_context import get_effective_user_id
|
||||
from deerflow.utils.time import coerce_iso, now_iso
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/chat-history", tags=["chat-history-transfer"])
|
||||
|
||||
FORMAT = "deerflow-chat-history-ui"
|
||||
FORMAT_VERSION = 1
|
||||
BATCH_LIMIT = 1
|
||||
MAX_THREAD_BYTES = 50 * 1024 * 1024
|
||||
MAX_FILES_PER_THREAD = 200
|
||||
SAFE_REL_PATH = re.compile(r"^(?:[A-Za-z0-9._-]+/)*[A-Za-z0-9._-]+$")
|
||||
_FILE_JSON_OVERHEAD = 64
|
||||
_DROP_META_KEYS = frozenset(
|
||||
{
|
||||
"owner_id",
|
||||
"user_id",
|
||||
"analysis_project_id",
|
||||
"analysis_project_mode",
|
||||
"analysis_project_name",
|
||||
"shared_from_code",
|
||||
"shared_origin",
|
||||
"shared_by",
|
||||
"taskId",
|
||||
"actionId",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class SkippedThread(BaseModel):
|
||||
thread_id: str
|
||||
title: str = ""
|
||||
reason: str
|
||||
|
||||
|
||||
class TransferFile(BaseModel):
|
||||
path: str
|
||||
content_b64: str
|
||||
size: int = 0
|
||||
|
||||
|
||||
class TransferThread(BaseModel):
|
||||
thread_id: str
|
||||
title: str = ""
|
||||
assistant_id: str | None = None
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
created_at: str = ""
|
||||
updated_at: str = ""
|
||||
values: dict[str, Any] = Field(default_factory=dict)
|
||||
files: list[TransferFile] = Field(default_factory=list)
|
||||
skipped_files: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ExportRequest(BaseModel):
|
||||
thread_ids: list[str] = Field(..., min_length=1, max_length=BATCH_LIMIT)
|
||||
|
||||
|
||||
class ExportResponse(BaseModel):
|
||||
format: str = FORMAT
|
||||
version: int = FORMAT_VERSION
|
||||
threads: list[TransferThread] = Field(default_factory=list)
|
||||
skipped: list[SkippedThread] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ImportRequest(BaseModel):
|
||||
threads: list[TransferThread] = Field(..., min_length=1, max_length=BATCH_LIMIT)
|
||||
|
||||
|
||||
class ImportResponse(BaseModel):
|
||||
imported: int = 0
|
||||
skipped: list[SkippedThread] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ExportAllStartResponse(BaseModel):
|
||||
job_id: str
|
||||
|
||||
|
||||
class ExportAllStatusResponse(BaseModel):
|
||||
job_id: str
|
||||
status: str
|
||||
total: int = 0
|
||||
processed: int = 0
|
||||
succeeded: int = 0
|
||||
skipped: list[SkippedThread] = Field(default_factory=list)
|
||||
skipped_files: int = 0
|
||||
error: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExportAllJob:
|
||||
job_id: str
|
||||
user_id: str
|
||||
status: str = "running"
|
||||
total: int = 0
|
||||
processed: int = 0
|
||||
succeeded: int = 0
|
||||
skipped: list[SkippedThread] = field(default_factory=list)
|
||||
skipped_files: int = 0
|
||||
error: str | None = None
|
||||
path: Path | None = None
|
||||
task: asyncio.Task[None] | None = None
|
||||
|
||||
|
||||
_EXPORT_JOBS: dict[str, ExportAllJob] = {}
|
||||
_SAFE_USER_DIR = re.compile(r"[^A-Za-z0-9._-]+")
|
||||
|
||||
|
||||
def require_transfer_enabled() -> None:
|
||||
if not load_system_settings().agent_skill_migration.enabled:
|
||||
raise HTTPException(status_code=403, detail="跨服务器迁移功能未开启")
|
||||
|
||||
|
||||
def current_user_id(request: Request) -> str:
|
||||
user = getattr(request.state, "user", None)
|
||||
if user is not None and getattr(user, "id", None):
|
||||
return str(user.id)
|
||||
auth = getattr(request.state, "auth", None)
|
||||
auth_user = getattr(auth, "user", None) if auth is not None else None
|
||||
if auth_user is not None and getattr(auth_user, "id", None):
|
||||
return str(auth_user.id)
|
||||
return get_effective_user_id()
|
||||
|
||||
|
||||
def payload_size(value: Any) -> int:
|
||||
return len(json.dumps(value, ensure_ascii=False, default=str).encode("utf-8"))
|
||||
|
||||
|
||||
def clean_export_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
|
||||
return {
|
||||
key: item
|
||||
for key, item in _strip_reserved_metadata(metadata).items()
|
||||
if key not in _DROP_META_KEYS
|
||||
}
|
||||
|
||||
|
||||
def safe_relpath(root: Path, path: Path) -> str | None:
|
||||
try:
|
||||
resolved = path.resolve()
|
||||
relative = resolved.relative_to(root.resolve())
|
||||
except ValueError:
|
||||
return None
|
||||
text = relative.as_posix()
|
||||
if not SAFE_REL_PATH.fullmatch(text):
|
||||
return None
|
||||
return text
|
||||
|
||||
|
||||
def encoded_file_cost(size: int, path: str) -> int:
|
||||
return (size * 4 + 3) // 3 + len(path) + _FILE_JSON_OVERHEAD
|
||||
|
||||
|
||||
def list_thread_file_candidates(root: Path) -> list[tuple[int, str, Path]]:
|
||||
if not root.is_dir():
|
||||
return []
|
||||
candidates: list[tuple[int, str, Path]] = []
|
||||
for path in sorted(root.rglob("*")):
|
||||
if not path.is_file() or path.is_symlink():
|
||||
continue
|
||||
rel = safe_relpath(root, path)
|
||||
if rel is None:
|
||||
continue
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
except OSError:
|
||||
continue
|
||||
if size < 0:
|
||||
continue
|
||||
candidates.append((size, rel, path))
|
||||
candidates.sort(key=lambda item: (item[0], item[1]))
|
||||
return candidates
|
||||
|
||||
|
||||
def collect_thread_files(root: Path, *, budget: int) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
candidates = list_thread_file_candidates(root)
|
||||
if not candidates:
|
||||
return [], []
|
||||
files: list[dict[str, Any]] = []
|
||||
skipped: list[str] = []
|
||||
used = 0
|
||||
for size, rel, path in candidates:
|
||||
cost = encoded_file_cost(size, rel)
|
||||
if budget <= 0 or len(files) >= MAX_FILES_PER_THREAD or used + cost > budget:
|
||||
skipped.append(rel)
|
||||
continue
|
||||
try:
|
||||
data = path.read_bytes()
|
||||
except OSError:
|
||||
skipped.append(rel)
|
||||
continue
|
||||
files.append(
|
||||
{
|
||||
"path": rel,
|
||||
"content_b64": base64.b64encode(data).decode("ascii"),
|
||||
"size": size,
|
||||
}
|
||||
)
|
||||
used += cost
|
||||
return files, skipped
|
||||
|
||||
|
||||
def write_thread_files(
|
||||
root: Path,
|
||||
files: list[TransferFile],
|
||||
*,
|
||||
budget: int = MAX_THREAD_BYTES,
|
||||
) -> list[str]:
|
||||
failed: list[str] = []
|
||||
used = 0
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
root_resolved = root.resolve()
|
||||
for item in files[:MAX_FILES_PER_THREAD]:
|
||||
rel = safe_relpath(root, root / item.path) if SAFE_REL_PATH.fullmatch(item.path) else None
|
||||
if rel is None:
|
||||
failed.append(item.path)
|
||||
continue
|
||||
dest = (root / rel).resolve()
|
||||
if dest != root_resolved and root_resolved not in dest.parents:
|
||||
failed.append(item.path)
|
||||
continue
|
||||
try:
|
||||
data = base64.b64decode(item.content_b64, validate=True)
|
||||
if used + len(data) > budget:
|
||||
failed.append(item.path)
|
||||
continue
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_bytes(data)
|
||||
used += len(data)
|
||||
except Exception:
|
||||
failed.append(item.path)
|
||||
return failed
|
||||
|
||||
|
||||
def _title_of(record: dict[str, Any], values: dict[str, Any] | None = None) -> str:
|
||||
title = record.get("display_name") or (values or {}).get("title") or ""
|
||||
return str(title).strip()
|
||||
|
||||
|
||||
def bump_channel_versions(checkpointer: Any, checkpoint: dict[str, Any], keys: list[str]) -> dict[str, Any]:
|
||||
"""LangGraph stores each channel as a versioned blob.
|
||||
|
||||
``aput(..., new_versions={})`` only keeps the in-memory dict on some
|
||||
savers. Postgres/SQLite ignore unversioned channels, so imported
|
||||
messages would not survive ``aget`` / ``GET /threads/{id}/state``.
|
||||
"""
|
||||
versions = dict(checkpoint.get("channel_versions") or {})
|
||||
new_versions: dict[str, Any] = {}
|
||||
get_next_version = getattr(checkpointer, "get_next_version", None)
|
||||
if not callable(get_next_version):
|
||||
return dict(versions)
|
||||
for key in keys:
|
||||
next_version = get_next_version(versions.get(key), None)
|
||||
versions[key] = next_version
|
||||
new_versions[key] = next_version
|
||||
checkpoint["channel_versions"] = versions
|
||||
return new_versions
|
||||
|
||||
|
||||
async def write_imported_checkpoint(
|
||||
checkpointer: Any,
|
||||
*,
|
||||
thread_id: str,
|
||||
values: dict[str, Any],
|
||||
metadata: dict[str, Any],
|
||||
) -> None:
|
||||
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
now = now_iso()
|
||||
checkpoint = dict(empty_checkpoint())
|
||||
checkpoint["channel_values"] = dict(values)
|
||||
new_versions = bump_channel_versions(checkpointer, checkpoint, list(values.keys()))
|
||||
await checkpointer.aput(
|
||||
config,
|
||||
checkpoint,
|
||||
{
|
||||
"step": 1,
|
||||
"source": "update",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
**metadata,
|
||||
},
|
||||
new_versions,
|
||||
)
|
||||
written = await checkpointer.aget_tuple(config)
|
||||
if written is None:
|
||||
raise RuntimeError("missing checkpoint after import")
|
||||
restored = serialize_channel_values(
|
||||
(getattr(written, "checkpoint", {}) or {}).get("channel_values", {}) or {}
|
||||
)
|
||||
if values.get("messages") and not restored.get("messages"):
|
||||
raise RuntimeError("checkpoint messages were not persisted")
|
||||
|
||||
|
||||
def export_job_dir(user_id: str) -> Path:
|
||||
safe = _SAFE_USER_DIR.sub("_", user_id).strip("._") or "user"
|
||||
path = get_paths().base_dir / "chat-history-exports" / safe
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def get_export_job(job_id: str, user_id: str) -> ExportAllJob:
|
||||
job = _EXPORT_JOBS.get(job_id)
|
||||
if job is None or job.user_id != user_id:
|
||||
raise HTTPException(status_code=404, detail="导出任务不存在")
|
||||
return job
|
||||
|
||||
|
||||
def find_running_export_job(user_id: str) -> ExportAllJob | None:
|
||||
for job in _EXPORT_JOBS.values():
|
||||
if job.user_id == user_id and job.status == "running":
|
||||
return job
|
||||
return None
|
||||
|
||||
|
||||
def purge_finished_export_jobs(user_id: str) -> None:
|
||||
stale = [
|
||||
job_id
|
||||
for job_id, job in _EXPORT_JOBS.items()
|
||||
if job.user_id == user_id and job.status != "running"
|
||||
]
|
||||
for job_id in stale:
|
||||
job = _EXPORT_JOBS.pop(job_id, None)
|
||||
if job is None or job.path is None:
|
||||
continue
|
||||
try:
|
||||
job.path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
logger.debug("Failed to remove export file %s", job.path)
|
||||
|
||||
|
||||
def job_status(job: ExportAllJob) -> ExportAllStatusResponse:
|
||||
return ExportAllStatusResponse(
|
||||
job_id=job.job_id,
|
||||
status=job.status,
|
||||
total=job.total,
|
||||
processed=job.processed,
|
||||
succeeded=job.succeeded,
|
||||
skipped=list(job.skipped),
|
||||
skipped_files=job.skipped_files,
|
||||
error=job.error,
|
||||
)
|
||||
|
||||
|
||||
class BundleWriter:
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self._fh = path.open("w", encoding="utf-8")
|
||||
self._first = True
|
||||
self._fh.write(
|
||||
'{"format":%s,"version":%s,"exported_at":%s,"threads":['
|
||||
% (
|
||||
json.dumps(FORMAT, ensure_ascii=False),
|
||||
FORMAT_VERSION,
|
||||
json.dumps(now_iso()),
|
||||
)
|
||||
)
|
||||
|
||||
def add_thread(self, thread: TransferThread) -> None:
|
||||
if not self._first:
|
||||
self._fh.write(",")
|
||||
self._first = False
|
||||
self._fh.write(thread.model_dump_json())
|
||||
self._fh.flush()
|
||||
|
||||
def finish(self, skipped: list[SkippedThread]) -> None:
|
||||
self._fh.write('],"skipped":')
|
||||
self._fh.write(json.dumps([item.model_dump() for item in skipped], ensure_ascii=False))
|
||||
self._fh.write("}")
|
||||
self._fh.close()
|
||||
|
||||
def abort(self) -> None:
|
||||
try:
|
||||
self._fh.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
async def list_exportable_thread_ids(thread_store: Any, user_id: str) -> list[str]:
|
||||
ids: list[str] = []
|
||||
offset = 0
|
||||
page_size = 100
|
||||
while True:
|
||||
rows = await thread_store.search(
|
||||
user_id=user_id,
|
||||
exclude_system=True,
|
||||
limit=page_size,
|
||||
offset=offset,
|
||||
)
|
||||
ids.extend(str(row["thread_id"]) for row in rows if row.get("thread_id"))
|
||||
if len(rows) < page_size:
|
||||
break
|
||||
offset += len(rows)
|
||||
return ids
|
||||
|
||||
|
||||
async def run_export_all_job(
|
||||
job: ExportAllJob,
|
||||
*,
|
||||
thread_store: Any,
|
||||
checkpointer: Any,
|
||||
) -> None:
|
||||
writer: BundleWriter | None = None
|
||||
try:
|
||||
thread_ids = await list_exportable_thread_ids(thread_store, job.user_id)
|
||||
job.total = len(thread_ids)
|
||||
if not thread_ids:
|
||||
job.status = "done"
|
||||
return
|
||||
dest = export_job_dir(job.user_id) / f"{job.job_id}.json"
|
||||
writer = BundleWriter(dest)
|
||||
for thread_id in thread_ids:
|
||||
try:
|
||||
payload, skip = await export_one_thread(
|
||||
thread_id=thread_id,
|
||||
user_id=job.user_id,
|
||||
thread_store=thread_store,
|
||||
checkpointer=checkpointer,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Unexpected export failure for %s", sanitize_log_param(thread_id))
|
||||
payload, skip = None, SkippedThread(thread_id=thread_id, reason="export_failed")
|
||||
if payload is not None:
|
||||
await asyncio.to_thread(writer.add_thread, payload)
|
||||
job.succeeded += 1
|
||||
job.skipped_files += len(payload.skipped_files)
|
||||
if skip is not None:
|
||||
job.skipped.append(skip)
|
||||
job.processed += 1
|
||||
await asyncio.to_thread(writer.finish, list(job.skipped))
|
||||
job.path = dest
|
||||
job.status = "done"
|
||||
except Exception:
|
||||
logger.exception("Background chat-history export failed for %s", sanitize_log_param(job.user_id))
|
||||
if writer is not None:
|
||||
writer.abort()
|
||||
job.status = "error"
|
||||
job.error = "导出失败"
|
||||
|
||||
|
||||
async def export_one_thread(
|
||||
*,
|
||||
thread_id: str,
|
||||
user_id: str,
|
||||
thread_store: Any,
|
||||
checkpointer: Any,
|
||||
max_bytes: int = MAX_THREAD_BYTES,
|
||||
) -> tuple[TransferThread | None, SkippedThread | None]:
|
||||
record = await thread_store.get(thread_id, user_id=user_id)
|
||||
if record is None:
|
||||
return None, SkippedThread(thread_id=thread_id, reason="not_found")
|
||||
if is_excluded_system_thread(record.get("metadata")):
|
||||
return None, SkippedThread(
|
||||
thread_id=thread_id,
|
||||
title=_title_of(record),
|
||||
reason="system_thread",
|
||||
)
|
||||
|
||||
try:
|
||||
checkpoint_tuple = await checkpointer.aget_tuple(
|
||||
{"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to read checkpoint for export %s", sanitize_log_param(thread_id))
|
||||
return None, SkippedThread(thread_id=thread_id, title=_title_of(record), reason="export_failed")
|
||||
|
||||
values: dict[str, Any] = {}
|
||||
if checkpoint_tuple is not None:
|
||||
checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {}
|
||||
try:
|
||||
values = serialize_channel_values(checkpoint.get("channel_values", {}) or {})
|
||||
except Exception:
|
||||
logger.exception("Failed to serialize state for export %s", sanitize_log_param(thread_id))
|
||||
return None, SkippedThread(thread_id=thread_id, title=_title_of(record), reason="export_failed")
|
||||
|
||||
title = _title_of(record, values)
|
||||
if title and not values.get("title"):
|
||||
values["title"] = title
|
||||
|
||||
used = payload_size(values)
|
||||
if used > max_bytes:
|
||||
return None, SkippedThread(thread_id=thread_id, title=title, reason="too_large")
|
||||
|
||||
user_data = get_paths().sandbox_user_data_dir(thread_id, user_id=user_id)
|
||||
files, skipped_files = await asyncio.to_thread(
|
||||
collect_thread_files, user_data, budget=max_bytes - used
|
||||
)
|
||||
payload = TransferThread(
|
||||
thread_id=thread_id,
|
||||
title=title,
|
||||
assistant_id=record.get("assistant_id"),
|
||||
metadata=clean_export_metadata(record.get("metadata") or {}),
|
||||
created_at=coerce_iso(record.get("created_at", "")),
|
||||
updated_at=coerce_iso(record.get("updated_at", "")),
|
||||
values=values,
|
||||
files=[TransferFile.model_validate(item) for item in files],
|
||||
skipped_files=list(skipped_files),
|
||||
)
|
||||
while payload.files and payload_size(payload.model_dump()) > max_bytes:
|
||||
largest = max(payload.files, key=lambda item: (item.size, item.path))
|
||||
payload.files.remove(largest)
|
||||
payload.skipped_files.append(largest.path)
|
||||
if payload_size(payload.model_dump()) > max_bytes:
|
||||
return None, SkippedThread(thread_id=thread_id, title=title, reason="too_large")
|
||||
return payload, None
|
||||
|
||||
|
||||
async def import_one_thread(
|
||||
*,
|
||||
item: TransferThread,
|
||||
user_id: str,
|
||||
thread_store: Any,
|
||||
checkpointer: Any,
|
||||
) -> SkippedThread | None:
|
||||
thread_id = item.thread_id.strip()
|
||||
if not thread_id or not re.fullmatch(r"[A-Za-z0-9_-]+", thread_id):
|
||||
return SkippedThread(thread_id=item.thread_id or "unknown", title=item.title, reason="invalid_id")
|
||||
|
||||
existing = await thread_store.get(thread_id, user_id=user_id)
|
||||
if existing is not None:
|
||||
return SkippedThread(thread_id=thread_id, title=item.title or _title_of(existing), reason="exists")
|
||||
|
||||
metadata = clean_export_metadata(item.metadata)
|
||||
title = (item.title or str(item.values.get("title") or "")).strip()
|
||||
values = dict(item.values or {})
|
||||
if title:
|
||||
values["title"] = title
|
||||
|
||||
created = False
|
||||
try:
|
||||
await thread_store.create(
|
||||
thread_id,
|
||||
assistant_id=item.assistant_id,
|
||||
user_id=user_id,
|
||||
display_name=title or None,
|
||||
metadata=metadata,
|
||||
)
|
||||
created = True
|
||||
await write_imported_checkpoint(
|
||||
checkpointer,
|
||||
thread_id=thread_id,
|
||||
values=values,
|
||||
metadata=metadata,
|
||||
)
|
||||
if title:
|
||||
try:
|
||||
await thread_store.update_display_name(thread_id, title, user_id=user_id)
|
||||
except Exception:
|
||||
logger.debug("Failed to sync imported title for %s", sanitize_log_param(thread_id))
|
||||
if item.files:
|
||||
dest = get_paths().sandbox_user_data_dir(thread_id, user_id=user_id)
|
||||
failed_files = await asyncio.to_thread(write_thread_files, dest, item.files)
|
||||
if failed_files:
|
||||
logger.warning(
|
||||
"Imported thread %s with %d skipped files",
|
||||
sanitize_log_param(thread_id),
|
||||
len(failed_files),
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed to import thread %s", sanitize_log_param(thread_id))
|
||||
if created:
|
||||
try:
|
||||
await thread_store.delete(thread_id, user_id=user_id)
|
||||
except Exception:
|
||||
logger.debug("Failed to roll back imported thread %s", sanitize_log_param(thread_id))
|
||||
return SkippedThread(thread_id=thread_id, title=title, reason="import_failed")
|
||||
return None
|
||||
|
||||
|
||||
@router.post("/export", response_model=ExportResponse, summary="Export a batch of conversations")
|
||||
@require_permission("threads", "read")
|
||||
async def export_chat_history(body: ExportRequest, request: Request) -> ExportResponse:
|
||||
require_transfer_enabled()
|
||||
user_id = current_user_id(request)
|
||||
thread_store = get_thread_store(request)
|
||||
checkpointer = get_checkpointer(request)
|
||||
exported: list[TransferThread] = []
|
||||
skipped: list[SkippedThread] = []
|
||||
for thread_id in body.thread_ids:
|
||||
try:
|
||||
payload, skip = await export_one_thread(
|
||||
thread_id=thread_id,
|
||||
user_id=user_id,
|
||||
thread_store=thread_store,
|
||||
checkpointer=checkpointer,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Unexpected export failure for %s", sanitize_log_param(thread_id))
|
||||
skipped.append(SkippedThread(thread_id=thread_id, reason="export_failed"))
|
||||
continue
|
||||
if payload is not None:
|
||||
exported.append(payload)
|
||||
if skip is not None:
|
||||
skipped.append(skip)
|
||||
return ExportResponse(threads=exported, skipped=skipped)
|
||||
|
||||
|
||||
@router.post("/import", response_model=ImportResponse, summary="Import a batch of conversations")
|
||||
@require_permission("threads", "write")
|
||||
async def import_chat_history(body: ImportRequest, request: Request) -> ImportResponse:
|
||||
require_transfer_enabled()
|
||||
user_id = current_user_id(request)
|
||||
thread_store = get_thread_store(request)
|
||||
checkpointer = get_checkpointer(request)
|
||||
imported = 0
|
||||
skipped: list[SkippedThread] = []
|
||||
for item in body.threads:
|
||||
try:
|
||||
skip = await import_one_thread(
|
||||
item=item,
|
||||
user_id=user_id,
|
||||
thread_store=thread_store,
|
||||
checkpointer=checkpointer,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Unexpected import failure for %s", sanitize_log_param(item.thread_id))
|
||||
skipped.append(SkippedThread(thread_id=item.thread_id, title=item.title, reason="import_failed"))
|
||||
continue
|
||||
if skip is None:
|
||||
imported += 1
|
||||
else:
|
||||
skipped.append(skip)
|
||||
return ImportResponse(imported=imported, skipped=skipped)
|
||||
|
||||
|
||||
@router.post("/export-all", response_model=ExportAllStartResponse, summary="Start a backend export of all conversations")
|
||||
@require_permission("threads", "read")
|
||||
async def start_export_all(request: Request) -> ExportAllStartResponse:
|
||||
require_transfer_enabled()
|
||||
user_id = current_user_id(request)
|
||||
running = find_running_export_job(user_id)
|
||||
if running is not None:
|
||||
return ExportAllStartResponse(job_id=running.job_id)
|
||||
purge_finished_export_jobs(user_id)
|
||||
job = ExportAllJob(job_id=uuid.uuid4().hex, user_id=user_id)
|
||||
_EXPORT_JOBS[job.job_id] = job
|
||||
job.task = asyncio.create_task(
|
||||
run_export_all_job(
|
||||
job,
|
||||
thread_store=get_thread_store(request),
|
||||
checkpointer=get_checkpointer(request),
|
||||
)
|
||||
)
|
||||
return ExportAllStartResponse(job_id=job.job_id)
|
||||
|
||||
|
||||
@router.get("/export-all/{job_id}", response_model=ExportAllStatusResponse, summary="Poll a backend export job")
|
||||
@require_permission("threads", "read")
|
||||
async def get_export_all_status(job_id: str, request: Request) -> ExportAllStatusResponse:
|
||||
require_transfer_enabled()
|
||||
return job_status(get_export_job(job_id, current_user_id(request)))
|
||||
|
||||
|
||||
@router.get("/export-all/{job_id}/download", summary="Download a finished backend export")
|
||||
@require_permission("threads", "read")
|
||||
async def download_export_all(job_id: str, request: Request) -> FileResponse:
|
||||
require_transfer_enabled()
|
||||
job = get_export_job(job_id, current_user_id(request))
|
||||
if job.status != "done" or job.path is None or not job.path.is_file():
|
||||
raise HTTPException(status_code=409, detail="导出文件还没准备好")
|
||||
return FileResponse(
|
||||
job.path,
|
||||
media_type="application/json",
|
||||
filename="deerflow-chats.json",
|
||||
)
|
||||
@@ -10,6 +10,7 @@ from pathlib import Path
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import FileResponse, Response
|
||||
|
||||
from deerflow.config.system_settings import generated_page_logo_path, load_system_settings
|
||||
from deerflow.html_runtime.addresses import html_map_tile_url, html_runtime_base_url
|
||||
|
||||
router = APIRouter(prefix="/api/public/html-runtime", tags=["html-runtime"])
|
||||
@@ -28,6 +29,12 @@ _STATIC_ASSETS = {
|
||||
"g6.min.js": "application/javascript",
|
||||
"leaflet.js": "application/javascript",
|
||||
"leaflet.css": "text/css",
|
||||
# 生成页品牌页头背景设计图(浅色红调 / 深色深蓝),随明暗 tone 由页头样式切换。
|
||||
"topbar-light.svg": "image/svg+xml",
|
||||
"topbar-dark.svg": "image/svg+xml",
|
||||
# Packaged fallback used when an administrator-uploaded generated-page
|
||||
# logo cannot be fetched through a deployment's reverse-proxy path.
|
||||
"brand-logo-dark.png": "image/png",
|
||||
}
|
||||
_GEO_MAP_ASSETS = {
|
||||
"china-provinces.geojson": "application/geo+json",
|
||||
@@ -94,6 +101,26 @@ async def get_html_runtime_geo_map(filename: str) -> FileResponse:
|
||||
)
|
||||
|
||||
|
||||
@router.get("/generated-page-logo")
|
||||
async def get_generated_page_logo() -> FileResponse:
|
||||
"""Serve the configured logo, falling back to the packaged default."""
|
||||
|
||||
branding = load_system_settings().generated_page_branding
|
||||
path = generated_page_logo_path(branding)
|
||||
if path is None:
|
||||
path = _RUNTIME_DIR / "brand-logo-dark.png"
|
||||
media_type = "image/png"
|
||||
else:
|
||||
media_type = branding.logo_media_type or "application/octet-stream"
|
||||
if not path.is_file():
|
||||
raise HTTPException(status_code=503, detail="Generated-page fallback logo is not installed")
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type=media_type,
|
||||
headers={**_PUBLIC_HEADERS, "Cache-Control": "public, max-age=3600"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{filename}")
|
||||
async def get_html_runtime_asset(filename: str) -> FileResponse:
|
||||
"""Serve a whitelisted, vendored browser distribution file."""
|
||||
|
||||
@@ -76,6 +76,17 @@ class LightAppResponse(BaseModel):
|
||||
|
||||
class LightAppListResponse(BaseModel):
|
||||
apps: list[LightAppResponse]
|
||||
|
||||
|
||||
class LightAppSidebarItem(BaseModel):
|
||||
app_id: str
|
||||
name: str
|
||||
mount_location: str | None = None
|
||||
sort_order: int = 0
|
||||
|
||||
|
||||
class LightAppSidebarResponse(BaseModel):
|
||||
apps: list[LightAppSidebarItem]
|
||||
|
||||
|
||||
class LightAppSidebarItem(BaseModel):
|
||||
@@ -209,6 +220,13 @@ async def list_sidebar_light_apps(request: Request) -> LightAppSidebarResponse:
|
||||
return await cached_catalog(request, "light_apps", "sidebar", load, LightAppSidebarResponse)
|
||||
|
||||
|
||||
@router.get("/sidebar", response_model=LightAppSidebarResponse)
|
||||
async def list_sidebar_light_apps(request: Request) -> LightAppSidebarResponse:
|
||||
"""Return the small set of fields required to build the shared sidebar."""
|
||||
rows = await _get_store(request).list_sidebar_apps()
|
||||
return LightAppSidebarResponse(apps=[LightAppSidebarItem(**row) for row in rows])
|
||||
|
||||
|
||||
@router.post("", response_model=LightAppResponse)
|
||||
async def create_light_app(request: Request, body: LightAppCreateRequest) -> LightAppResponse:
|
||||
await _require_admin(request)
|
||||
|
||||
@@ -93,6 +93,14 @@ _STEP3_AGENT_IDS = frozenset({
|
||||
SUMMARY_AGENT_ID,
|
||||
})
|
||||
|
||||
_STEP3_AGENT_IDS = frozenset({
|
||||
ACTION_PLAN_AGENT_ID,
|
||||
DASHBOARD_AGENT_ID,
|
||||
REPORT_AGENT_ID,
|
||||
STRUCTURE_AGENT_ID,
|
||||
SUMMARY_AGENT_ID,
|
||||
})
|
||||
|
||||
|
||||
def _stage_for_agent(agent_type: str, agent_name: str) -> str:
|
||||
"""把一次 /run/stream 的 (agent_type, agent_name) 映射成诊断日志的 stage。
|
||||
@@ -131,12 +139,45 @@ def _is_step2_room_limited(
|
||||
return agent_type == "leader" or agent_name not in _STEP3_AGENT_IDS
|
||||
|
||||
|
||||
|
||||
# 圆桌(intent / recommend / 席位 / 协调)创建的 thread 都是功能型内部线程,
|
||||
# 不是用户的普通对话。打上这个 metadata,前端「最近的对话」列表(recent-chat-list.tsx)
|
||||
# 据 `metadata.thread_type` 过滤掉,与 scheduler / bootstrap 系统会话同一套机制。
|
||||
_ROUNDTABLE_THREAD_METADATA: dict[str, Any] = {"thread_type": "roundtable", "system": True}
|
||||
|
||||
|
||||
def _room_thread_metadata(
|
||||
*,
|
||||
draft_id: str | None,
|
||||
user_id: str | None,
|
||||
task_id: str | None,
|
||||
room_session_id: str | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Step-2 threads carry a stable room key so later occupancy can see live runs."""
|
||||
from deerflow.persistence.roundtable_jobs.sql import compute_active_dedupe_key
|
||||
from deerflow.persistence.roundtable_rooms.sql import compute_room_session_key
|
||||
|
||||
meta = dict(_ROUNDTABLE_THREAD_METADATA)
|
||||
clean_session_id = (room_session_id or "").strip() or None
|
||||
key = compute_room_session_key(clean_session_id) or compute_active_dedupe_key(
|
||||
draft_id, user_id, task_id
|
||||
)
|
||||
if key:
|
||||
meta["roundtable_room_key"] = key
|
||||
if clean_session_id:
|
||||
meta["roundtable_room_session_id"] = clean_session_id
|
||||
if draft_id:
|
||||
meta["draft_id"] = draft_id
|
||||
if task_id:
|
||||
meta["task_id"] = task_id
|
||||
return meta
|
||||
|
||||
|
||||
def _step3_thread_metadata() -> dict[str, Any]:
|
||||
"""Step-3 threads are internal threads, but never Step-2 room occupants."""
|
||||
return dict(_ROUNDTABLE_THREAD_METADATA)
|
||||
|
||||
|
||||
def _room_thread_metadata(
|
||||
*,
|
||||
draft_id: str | None,
|
||||
@@ -905,6 +946,26 @@ async def init_multi_agent(request: Request, payload: dict = Body(...)) -> Strea
|
||||
room_session_id=room_session_id,
|
||||
)
|
||||
|
||||
# The browser already acquired the room before entering Step 2. Init only
|
||||
# renews that lease; it cannot create a second room or bypass admission.
|
||||
from app.gateway.roundtable_room_gate import enforce_existing_roundtable_room
|
||||
|
||||
room_draft_id = str(payload.get("draft_id") or "").strip() or None
|
||||
room_task_id = str(payload.get("task_id") or "").strip() or None
|
||||
room_session_id = str(payload.get("room_session_id") or "").strip() or None
|
||||
await enforce_existing_roundtable_room(
|
||||
request,
|
||||
draft_id=room_draft_id,
|
||||
task_id=room_task_id,
|
||||
room_session_id=room_session_id,
|
||||
)
|
||||
room_thread_meta = _room_thread_metadata(
|
||||
draft_id=room_draft_id,
|
||||
user_id=_resolve_user_id(request),
|
||||
task_id=room_task_id,
|
||||
room_session_id=room_session_id,
|
||||
)
|
||||
|
||||
def _frame(payload: dict[str, Any]) -> str:
|
||||
return f"data: {json.dumps(payload, ensure_ascii=False)}\n\n"
|
||||
|
||||
|
||||
@@ -153,6 +153,16 @@ async def _refresh_catalog_snapshot(config: AppConfig) -> None:
|
||||
logger.exception("Skill catalog projection refresh failed after a mutation")
|
||||
|
||||
|
||||
async def _refresh_catalog_snapshot(config: AppConfig) -> None:
|
||||
"""Keep normalized MySQL skill rows current after a skill edit."""
|
||||
try:
|
||||
await refresh_skill_catalog_projection(config)
|
||||
except Exception:
|
||||
# A failed projection refresh must not turn a successful skill write
|
||||
# into an API failure. Startup or the next edit can recover it.
|
||||
logger.exception("Skill catalog projection refresh failed after a mutation")
|
||||
|
||||
|
||||
def _can_view_skill_source(
|
||||
category: SkillCategory,
|
||||
owner_user_id: str | None,
|
||||
@@ -513,6 +523,69 @@ _SKILL_DISPLAY_TABLE_LOCK = asyncio.Lock()
|
||||
_skill_display_table_engine: object | None = None
|
||||
|
||||
|
||||
def _skill_display_table_already_exists(exc: BaseException) -> bool:
|
||||
message = str(exc).lower()
|
||||
return any(
|
||||
token in message
|
||||
for token in (
|
||||
"already exists",
|
||||
"duplicate table",
|
||||
"42p07",
|
||||
"error 1050",
|
||||
"(1050",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _catalog_item_to_response(
|
||||
item: dict[str, Any],
|
||||
ownership: dict[str, Any] | None,
|
||||
*,
|
||||
adapter: dict[str, Any] | None,
|
||||
favorited: bool,
|
||||
) -> SkillResponse:
|
||||
"""Build a list response from the MySQL snapshot and MySQL metadata rows."""
|
||||
|
||||
try:
|
||||
category = SkillCategory(str(item.get("category") or SkillCategory.PUBLIC.value))
|
||||
except ValueError:
|
||||
category = SkillCategory.PUBLIC
|
||||
owner_user_id = ownership.get("owner_user_id") if ownership else None
|
||||
return SkillResponse(
|
||||
name=str(item.get("name") or ""),
|
||||
name_zh=ownership.get("name_zh") if ownership else None,
|
||||
description=str(item.get("description") or ""),
|
||||
license=item.get("license"),
|
||||
category=category,
|
||||
enabled=bool(item.get("enabled", True)),
|
||||
always_on=bool(ownership.get("always_on", item.get("always_on", False)) if ownership else item.get("always_on", False)),
|
||||
use_count=int(item.get("use_count") or 0),
|
||||
view_count=int(item.get("view_count") or 0),
|
||||
patch_count=int(item.get("patch_count") or 0),
|
||||
last_activity_at=item.get("last_activity_at"),
|
||||
created_at=item.get("created_at"),
|
||||
state=str(item.get("state") or "active"),
|
||||
pinned=bool(item.get("pinned")),
|
||||
featured=bool(item.get("featured")),
|
||||
source=str(item.get("source") or "upload"),
|
||||
owner_user_id=owner_user_id,
|
||||
published=bool(ownership.get("published", False)) if ownership else False,
|
||||
square_id=str(ownership.get("square_id") or "") if ownership else "",
|
||||
builtin=bool(ownership.get("builtin", owner_user_id is None)) if ownership else category != SkillCategory.CUSTOM,
|
||||
favorited=favorited,
|
||||
detail=(ownership.get("detail_preview") or ownership.get("detail")) if ownership else None,
|
||||
display=adapter.get("display") if adapter else None,
|
||||
displaySample=adapter.get("sample") if adapter else None,
|
||||
)
|
||||
|
||||
|
||||
# ``checkfirst=True`` is a check-then-create operation. The in-process lock
|
||||
# serializes concurrent requests in one worker; the duplicate-table fallback
|
||||
# covers two Gateway workers racing during first access.
|
||||
_SKILL_DISPLAY_TABLE_LOCK = asyncio.Lock()
|
||||
_skill_display_table_engine: object | None = None
|
||||
|
||||
|
||||
def _skill_display_table_already_exists(exc: BaseException) -> bool:
|
||||
message = str(exc).lower()
|
||||
return any(
|
||||
|
||||
@@ -8,19 +8,22 @@ from typing import Literal
|
||||
from urllib.parse import urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.auth.password import hash_password_async
|
||||
from app.gateway.deps import get_current_user_from_request, get_optional_user_from_request
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.config.system_settings import (
|
||||
BUILTIN_THEME_IDS,
|
||||
AgentSkillMigrationSettings,
|
||||
AppearanceRouteParam,
|
||||
AuthenticationSettings,
|
||||
CompactShortcutButton,
|
||||
ConversationContextSettings,
|
||||
CustomThemePreset,
|
||||
GeneratedPageBrandingSettings,
|
||||
HtmlMapSettings,
|
||||
LeaderboardSettings,
|
||||
ModelConcurrencySettings,
|
||||
@@ -36,6 +39,8 @@ from deerflow.config.system_settings import (
|
||||
StrategyProjectSettings,
|
||||
WordHeadingNumberingStyle,
|
||||
WriteFileCompatibilitySettings,
|
||||
generated_page_logo_path,
|
||||
generated_page_logo_public_url,
|
||||
get_module_tag_ids,
|
||||
list_publish_squares,
|
||||
load_system_settings,
|
||||
@@ -313,11 +318,15 @@ class HtmlMapSettingsUpdateRequest(BaseModel):
|
||||
class HtmlRuntimeSettingsResponse(BaseModel):
|
||||
base_url: str
|
||||
map_tile_url: str
|
||||
message_map_tile_min_zoom: int
|
||||
message_map_tile_max_zoom: int
|
||||
|
||||
|
||||
class HtmlRuntimeSettingsUpdateRequest(BaseModel):
|
||||
base_url: str
|
||||
map_tile_url: str
|
||||
message_map_tile_min_zoom: int | None = Field(default=None, ge=0, le=20)
|
||||
message_map_tile_max_zoom: int | None = Field(default=None, ge=0, le=20)
|
||||
|
||||
|
||||
class LeaderboardSettingsResponse(BaseModel):
|
||||
@@ -518,6 +527,17 @@ class MemoryRuntimeUpdateRequest(BaseModel):
|
||||
enabled: bool
|
||||
|
||||
|
||||
class AgentSkillMigrationResponse(BaseModel):
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
description="Whether administrator agent/skill migration is enabled deployment-wide",
|
||||
)
|
||||
|
||||
|
||||
class AgentSkillMigrationUpdateRequest(BaseModel):
|
||||
enabled: bool
|
||||
|
||||
|
||||
class WriteFileCompatibilityResponse(BaseModel):
|
||||
enabled: bool = Field(
|
||||
default=False,
|
||||
@@ -617,6 +637,7 @@ def _knowledge_retrieval_settings_store(request: Request):
|
||||
return store
|
||||
|
||||
|
||||
|
||||
@router.get("/memory", response_model=MemoryRuntimeResponse)
|
||||
async def get_memory_runtime_setting() -> MemoryRuntimeResponse:
|
||||
return MemoryRuntimeResponse(enabled=load_system_settings().memory.enabled)
|
||||
@@ -635,6 +656,28 @@ async def update_memory_runtime_setting(
|
||||
return MemoryRuntimeResponse(enabled=saved.memory.enabled)
|
||||
|
||||
|
||||
@router.get("/agent-skill-migration", response_model=AgentSkillMigrationResponse)
|
||||
async def get_agent_skill_migration_setting() -> AgentSkillMigrationResponse:
|
||||
return AgentSkillMigrationResponse(
|
||||
enabled=load_system_settings().agent_skill_migration.enabled,
|
||||
)
|
||||
|
||||
|
||||
@router.put("/agent-skill-migration", response_model=AgentSkillMigrationResponse)
|
||||
async def update_agent_skill_migration_setting(
|
||||
body: AgentSkillMigrationUpdateRequest,
|
||||
request: Request,
|
||||
) -> AgentSkillMigrationResponse:
|
||||
await _require_admin(request)
|
||||
current = load_system_settings()
|
||||
updated = AgentSkillMigrationSettings(enabled=body.enabled)
|
||||
saved = save_system_settings(
|
||||
current.model_copy(update={"agent_skill_migration": updated})
|
||||
)
|
||||
logger.info("Agent/skill migration switch changed: enabled=%s", saved.agent_skill_migration.enabled)
|
||||
return AgentSkillMigrationResponse(enabled=saved.agent_skill_migration.enabled)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/write-file-compatibility",
|
||||
response_model=WriteFileCompatibilityResponse,
|
||||
@@ -837,6 +880,7 @@ async def update_ordinary_knowledge_retrieval_setting(
|
||||
)
|
||||
|
||||
|
||||
|
||||
def _skill_compression_to_response(settings: SkillCompressionSettings) -> SkillCompressionResponse:
|
||||
return SkillCompressionResponse(
|
||||
enabled=settings.enabled,
|
||||
@@ -1073,10 +1117,13 @@ async def update_html_map_settings(
|
||||
|
||||
@router.get("/html-runtime", response_model=HtmlRuntimeSettingsResponse)
|
||||
async def get_html_runtime_settings() -> HtmlRuntimeSettingsResponse:
|
||||
"""Return the addresses newly generated HTML pages will use."""
|
||||
"""Return generated-page addresses and message-map native tile limits."""
|
||||
settings = load_system_settings().html_runtime
|
||||
return HtmlRuntimeSettingsResponse(
|
||||
base_url=html_runtime_base_url(),
|
||||
map_tile_url=html_map_tile_url(),
|
||||
message_map_tile_min_zoom=settings.message_map_tile_min_zoom,
|
||||
message_map_tile_max_zoom=settings.message_map_tile_max_zoom,
|
||||
)
|
||||
|
||||
|
||||
@@ -1085,7 +1132,7 @@ async def update_html_runtime_settings(
|
||||
body: HtmlRuntimeSettingsUpdateRequest,
|
||||
request: Request,
|
||||
) -> HtmlRuntimeSettingsResponse:
|
||||
"""Save the browser-reachable runtime and map-tile addresses. Admin only."""
|
||||
"""Save generated-page addresses and message-map native tile limits. Admin only."""
|
||||
await _require_admin(request)
|
||||
try:
|
||||
base_url = normalize_html_runtime_base_url(body.base_url)
|
||||
@@ -1093,13 +1140,32 @@ async def update_html_runtime_settings(
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
current = load_system_settings()
|
||||
min_zoom = (
|
||||
body.message_map_tile_min_zoom
|
||||
if body.message_map_tile_min_zoom is not None
|
||||
else current.html_runtime.message_map_tile_min_zoom
|
||||
)
|
||||
max_zoom = (
|
||||
body.message_map_tile_max_zoom
|
||||
if body.message_map_tile_max_zoom is not None
|
||||
else current.html_runtime.message_map_tile_max_zoom
|
||||
)
|
||||
if min_zoom > max_zoom:
|
||||
raise HTTPException(status_code=400, detail="消息列表瓦片最小层级不能大于最大层级")
|
||||
updated = current.html_runtime.model_copy(
|
||||
update={"base_url": base_url, "map_tile_url": map_tile_url}
|
||||
update={
|
||||
"base_url": base_url,
|
||||
"map_tile_url": map_tile_url,
|
||||
"message_map_tile_min_zoom": min_zoom,
|
||||
"message_map_tile_max_zoom": max_zoom,
|
||||
}
|
||||
)
|
||||
saved = save_system_settings(current.model_copy(update={"html_runtime": updated}))
|
||||
return HtmlRuntimeSettingsResponse(
|
||||
base_url=saved.html_runtime.base_url,
|
||||
map_tile_url=saved.html_runtime.map_tile_url,
|
||||
message_map_tile_min_zoom=saved.html_runtime.message_map_tile_min_zoom,
|
||||
message_map_tile_max_zoom=saved.html_runtime.message_map_tile_max_zoom,
|
||||
)
|
||||
|
||||
|
||||
@@ -1422,6 +1488,230 @@ async def update_appearance_settings(body: AppearanceUpdateRequest, request: Req
|
||||
return _appearance_response(saved.appearance)
|
||||
|
||||
|
||||
class GeneratedPageBrandingResponse(BaseModel):
|
||||
"""Generated-page identity: name, logo display mode and optional custom logo."""
|
||||
|
||||
chrome_enabled: bool = True
|
||||
header_layout: str = "center"
|
||||
system_name: str | None = None
|
||||
footer_text: str | None = None
|
||||
logo_mode: Literal["custom", "default", "none"] = "custom"
|
||||
logo_url: str | None = None
|
||||
logo_version: str | None = None
|
||||
customized: bool = False
|
||||
active: bool = False
|
||||
|
||||
|
||||
_MAX_BRAND_LOGO_BYTES = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def _detect_brand_logo(data: bytes) -> tuple[str, str]:
|
||||
"""Return a safe extension/media type from file signatures, never the filename."""
|
||||
|
||||
if data.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
return ".png", "image/png"
|
||||
if data.startswith(b"\xff\xd8\xff"):
|
||||
return ".jpg", "image/jpeg"
|
||||
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||||
return ".webp", "image/webp"
|
||||
raise HTTPException(status_code=422, detail="Logo 仅支持 PNG、JPG 或 WebP 图片")
|
||||
|
||||
|
||||
def _normalize_system_name(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = " ".join(value.split())
|
||||
if len(normalized) > 80:
|
||||
raise HTTPException(status_code=422, detail="系统名称不能超过 80 个字符")
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _normalize_footer_text(value: str | None) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = " ".join(value.split())
|
||||
if len(normalized) > 160:
|
||||
raise HTTPException(status_code=422, detail="网页页脚文字不能超过 160 个字符")
|
||||
return normalized or None
|
||||
|
||||
|
||||
def _generated_page_branding_response(
|
||||
settings: GeneratedPageBrandingSettings,
|
||||
) -> GeneratedPageBrandingResponse:
|
||||
defaults = GeneratedPageBrandingSettings()
|
||||
logo_url = generated_page_logo_public_url("/api/public/html-runtime", settings)
|
||||
return GeneratedPageBrandingResponse(
|
||||
chrome_enabled=settings.chrome_enabled,
|
||||
header_layout=settings.header_layout,
|
||||
system_name=settings.system_name,
|
||||
footer_text=settings.footer_text,
|
||||
logo_mode=settings.logo_mode,
|
||||
logo_url=logo_url,
|
||||
logo_version=settings.logo_version if logo_url else None,
|
||||
customized=bool(
|
||||
settings.system_name
|
||||
or settings.footer_text
|
||||
or logo_url
|
||||
or settings.logo_mode != defaults.logo_mode
|
||||
or settings.chrome_enabled != defaults.chrome_enabled
|
||||
or settings.header_layout != defaults.header_layout
|
||||
),
|
||||
active=bool(
|
||||
settings.system_name
|
||||
and settings.chrome_enabled
|
||||
and (settings.logo_mode == "none" or logo_url)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _write_generated_page_logo(data: bytes, suffix: str) -> str:
|
||||
directory = runtime_home() / "generated-page-branding"
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
filename = f"logo-{uuid4().hex}{suffix}"
|
||||
destination = directory / filename
|
||||
temporary = directory / f".{filename}.tmp"
|
||||
try:
|
||||
temporary.write_bytes(data)
|
||||
temporary.replace(destination)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
return filename
|
||||
|
||||
|
||||
@router.get("/generated-page-branding", response_model=GeneratedPageBrandingResponse)
|
||||
async def get_generated_page_branding_settings() -> GeneratedPageBrandingResponse:
|
||||
"""Return the optional generated-page identity. Any logged-in user may read it."""
|
||||
|
||||
return _generated_page_branding_response(load_system_settings().generated_page_branding)
|
||||
|
||||
|
||||
@router.put("/generated-page-branding", response_model=GeneratedPageBrandingResponse)
|
||||
async def update_generated_page_branding_settings(
|
||||
request: Request,
|
||||
chrome_enabled: bool | None = Form(default=None),
|
||||
header_layout: str | None = Form(default=None),
|
||||
system_name: str | None = Form(default=None),
|
||||
logo_mode: Literal["custom", "default", "none"] | None = Form(default=None),
|
||||
# Plain str (not Optional): FastAPI coerces an empty Optional form value to
|
||||
# None, which would make "clear the footer text" indistinguishable from
|
||||
# "field omitted". The settings UI always sends this field, so an empty
|
||||
# value here means the admin cleared the input and wants the default footer.
|
||||
footer_text: str = Form(default=""),
|
||||
remove_logo: bool = Form(default=False),
|
||||
logo: UploadFile | None = File(default=None),
|
||||
) -> GeneratedPageBrandingResponse:
|
||||
"""Update the identity used in generated pages and the auto-injection switch. Admin only."""
|
||||
|
||||
await _require_admin(request)
|
||||
if logo is not None and (remove_logo or logo_mode in {"default", "none"}):
|
||||
raise HTTPException(status_code=422, detail="上传 Logo 时不能同时选择默认头像或不显示头像")
|
||||
if remove_logo and logo_mode not in {None, "none"}:
|
||||
raise HTTPException(status_code=422, detail="移除 Logo 时只能选择不显示头像")
|
||||
|
||||
current = load_system_settings()
|
||||
previous_logo = generated_page_logo_path(current.generated_page_branding)
|
||||
update: dict[str, object] = {}
|
||||
if chrome_enabled is not None:
|
||||
update["chrome_enabled"] = bool(chrome_enabled)
|
||||
if header_layout is not None:
|
||||
normalized_layout = str(header_layout).strip().lower()
|
||||
if normalized_layout not in {"left", "center"}:
|
||||
raise HTTPException(status_code=422, detail="header_layout 仅支持 left 或 center")
|
||||
update["header_layout"] = normalized_layout
|
||||
if system_name is not None:
|
||||
update["system_name"] = _normalize_system_name(system_name)
|
||||
update["footer_text"] = _normalize_footer_text(footer_text)
|
||||
|
||||
new_logo_path = None
|
||||
if logo is not None:
|
||||
data = await logo.read(_MAX_BRAND_LOGO_BYTES + 1)
|
||||
await logo.close()
|
||||
if not data:
|
||||
raise HTTPException(status_code=422, detail="Logo 文件不能为空")
|
||||
if len(data) > _MAX_BRAND_LOGO_BYTES:
|
||||
raise HTTPException(status_code=413, detail="Logo 文件不能超过 10MB")
|
||||
suffix, media_type = _detect_brand_logo(data)
|
||||
filename = await asyncio.to_thread(_write_generated_page_logo, data, suffix)
|
||||
new_logo_path = runtime_home() / "generated-page-branding" / filename
|
||||
update.update(
|
||||
{
|
||||
"logo_mode": "custom",
|
||||
"logo_filename": filename,
|
||||
"logo_media_type": media_type,
|
||||
"logo_version": uuid4().hex,
|
||||
}
|
||||
)
|
||||
elif logo_mode == "none":
|
||||
update.update(
|
||||
{
|
||||
"logo_mode": "none",
|
||||
"logo_filename": None,
|
||||
"logo_media_type": None,
|
||||
"logo_version": None,
|
||||
}
|
||||
)
|
||||
elif remove_logo:
|
||||
# Preserve the legacy API behavior: an older client that only sends
|
||||
# remove_logo leaves branding as an incomplete custom-logo draft.
|
||||
# The new text-only header is an explicit logo_mode="none" choice.
|
||||
update.update(
|
||||
{
|
||||
"logo_mode": "custom",
|
||||
"logo_filename": None,
|
||||
"logo_media_type": None,
|
||||
"logo_version": None,
|
||||
}
|
||||
)
|
||||
elif logo_mode == "default":
|
||||
update.update(
|
||||
{
|
||||
"logo_mode": "default",
|
||||
"logo_filename": None,
|
||||
"logo_media_type": None,
|
||||
"logo_version": "default",
|
||||
}
|
||||
)
|
||||
elif logo_mode == "custom":
|
||||
update["logo_mode"] = "custom"
|
||||
if current.generated_page_branding.logo_mode != "custom":
|
||||
update.update(
|
||||
{
|
||||
"logo_filename": None,
|
||||
"logo_media_type": None,
|
||||
"logo_version": None,
|
||||
}
|
||||
)
|
||||
|
||||
if not update:
|
||||
return _generated_page_branding_response(current.generated_page_branding)
|
||||
|
||||
updated = current.generated_page_branding.model_copy(update=update)
|
||||
try:
|
||||
saved = save_system_settings(current.model_copy(update={"generated_page_branding": updated}))
|
||||
except Exception:
|
||||
if new_logo_path is not None:
|
||||
await asyncio.to_thread(new_logo_path.unlink, missing_ok=True)
|
||||
raise
|
||||
if previous_logo is not None and previous_logo != generated_page_logo_path(saved.generated_page_branding):
|
||||
await asyncio.to_thread(previous_logo.unlink, missing_ok=True)
|
||||
return _generated_page_branding_response(saved.generated_page_branding)
|
||||
|
||||
|
||||
@router.delete("/generated-page-branding", response_model=GeneratedPageBrandingResponse)
|
||||
async def delete_generated_page_branding_settings(
|
||||
request: Request,
|
||||
) -> GeneratedPageBrandingResponse:
|
||||
"""Remove generated-page identity requirements; the app UI is unaffected."""
|
||||
|
||||
await _require_admin(request)
|
||||
current = load_system_settings()
|
||||
previous_logo = generated_page_logo_path(current.generated_page_branding)
|
||||
saved = save_system_settings(current.model_copy(update={"generated_page_branding": GeneratedPageBrandingSettings()}))
|
||||
if previous_logo is not None:
|
||||
await asyncio.to_thread(previous_logo.unlink, missing_ok=True)
|
||||
return _generated_page_branding_response(saved.generated_page_branding)
|
||||
|
||||
|
||||
class LogSystemResponse(BaseModel):
|
||||
"""External log-system entry shown in the admin settings menu.
|
||||
|
||||
@@ -1575,3 +1865,4 @@ async def update_wx_common_agents(
|
||||
updated = current.wx_common_agents.model_copy(update={"agent_type": agent_type})
|
||||
saved = save_system_settings(current.model_copy(update={"wx_common_agents": updated}))
|
||||
return WxCommonAgentsResponse(agent_type=saved.wx_common_agents.agent_type)
|
||||
|
||||
@@ -546,8 +546,8 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
|
||||
"subagent_enabled",
|
||||
"max_concurrent_subagents",
|
||||
"agent_name",
|
||||
# 页面研报设计师由前端随消息带入当前界面明暗模式;需要进入 agent
|
||||
# 提示词及 ToolRuntime.context,才能选择同色系模板。
|
||||
# 前端随消息只带入当前界面的明暗模式;需要进入 agent 提示词及
|
||||
# ToolRuntime.context,才能选择对应的浅色或深色页面模板。
|
||||
"presentation_theme",
|
||||
"is_bootstrap",
|
||||
"is_writing_setup",
|
||||
|
||||
Reference in new issue
Block a user