sync snapshot from LAN repo (source master-zg @ 40658c74)
This commit is contained in:
1 parent
ea9dcff9c0
commit
754b61572c
179 files changed
+26085
-8788
No files matched your search
@@ -0,0 +1,924 @@
|
||||
"""Agent/skill metadata migration manifests for already-copied live folders.
|
||||
|
||||
The manifest deliberately contains no agent or skill payload files. Operators
|
||||
copy those directories with their normal offline transfer process; these
|
||||
helpers only prove that the target runtime folders match the exported inventory
|
||||
before database/configuration metadata is reconciled.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
import yaml
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
ResourceStatus = Literal["matched", "missing", "hash_mismatch", "identity_mismatch", "invalid_path"]
|
||||
OwnerStatus = Literal["matched", "unresolved"]
|
||||
|
||||
_TREE_HASH_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||
_TREE_HASH_MAX_WORKERS = 4
|
||||
|
||||
|
||||
def _safe_relative_path(value: str) -> str:
|
||||
normalized = value.strip()
|
||||
if not normalized or "\\" in normalized or ":" in normalized:
|
||||
raise ValueError("relative_dir must be a non-empty portable relative path")
|
||||
path = PurePosixPath(normalized)
|
||||
if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
|
||||
raise ValueError("relative_dir must stay inside the configured runtime root")
|
||||
return path.as_posix()
|
||||
|
||||
|
||||
def compute_tree_hash(root: Path) -> str:
|
||||
"""Return a deterministic SHA-256 for a directory tree.
|
||||
|
||||
Relative paths and bytes are both signed. Symlinks are rejected so a
|
||||
manifest cannot attest content outside the selected runtime directory.
|
||||
"""
|
||||
|
||||
root = root.resolve()
|
||||
if not root.is_dir():
|
||||
raise FileNotFoundError(root)
|
||||
digest = hashlib.sha256()
|
||||
files = sorted((path for path in root.rglob("*") if path.is_file() or path.is_symlink()), key=lambda p: p.relative_to(root).as_posix())
|
||||
for path in files:
|
||||
if path.is_symlink():
|
||||
raise ValueError(f"Symlinks are not supported in migration trees: {path}")
|
||||
relative = path.relative_to(root).as_posix().encode("utf-8")
|
||||
digest.update(len(relative).to_bytes(4, "big"))
|
||||
digest.update(relative)
|
||||
digest.update(path.stat().st_size.to_bytes(8, "big"))
|
||||
with path.open("rb") as stream:
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
class OwnerReference(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
source_user_id: str | None = None
|
||||
email: str | None = None
|
||||
|
||||
@field_validator("source_user_id", "email")
|
||||
@classmethod
|
||||
def _strip_optional(cls, value: str | None) -> str | None:
|
||||
return value.strip() if value and value.strip() else None
|
||||
|
||||
|
||||
class TagManifestItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str = Field(min_length=1, max_length=64)
|
||||
name: str = Field(min_length=1, max_length=128)
|
||||
color: str = Field(default="gray", max_length=32)
|
||||
description: str = Field(default="", max_length=1024)
|
||||
|
||||
|
||||
class AgentManifestItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
id: str
|
||||
relative_dir: str
|
||||
tree_hash: str
|
||||
name: str
|
||||
description: str = ""
|
||||
owner: OwnerReference | None = None
|
||||
published: bool = False
|
||||
is_private: bool = False
|
||||
square_id: str = ""
|
||||
featured_order: int | None = None
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
skills: list[str] = Field(default_factory=list, max_length=5000)
|
||||
tags: list[TagManifestItem] = Field(default_factory=list, max_length=200)
|
||||
|
||||
@field_validator("relative_dir")
|
||||
@classmethod
|
||||
def _relative_dir(cls, value: str) -> str:
|
||||
return _safe_relative_path(value)
|
||||
|
||||
@field_validator("tree_hash")
|
||||
@classmethod
|
||||
def _tree_hash(cls, value: str) -> str:
|
||||
normalized = value.lower()
|
||||
if not _TREE_HASH_RE.fullmatch(normalized):
|
||||
raise ValueError("tree_hash must be a SHA-256 hex digest")
|
||||
return normalized
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _directory_matches_id(self):
|
||||
if self.relative_dir != self.id.lower():
|
||||
raise ValueError("agent relative_dir must equal the stable agent id")
|
||||
return self
|
||||
|
||||
|
||||
class SkillManifestItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str
|
||||
category: Literal["public", "custom"]
|
||||
relative_dir: str
|
||||
tree_hash: str
|
||||
owner: OwnerReference | None = None
|
||||
published: bool = False
|
||||
detail: str | None = None
|
||||
name_zh: str | None = None
|
||||
always_on: bool = False
|
||||
square_id: str = ""
|
||||
enabled: bool = True
|
||||
state: Literal["active", "stale", "archived"] = "active"
|
||||
pinned: bool = False
|
||||
featured: bool = False
|
||||
source: Literal["agent", "upload"] = "upload"
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
tags: list[TagManifestItem] = Field(default_factory=list, max_length=200)
|
||||
|
||||
@field_validator("relative_dir")
|
||||
@classmethod
|
||||
def _relative_dir(cls, value: str) -> str:
|
||||
normalized = _safe_relative_path(value)
|
||||
if PurePosixPath(normalized).parts[0] not in {"public", "custom"}:
|
||||
raise ValueError("skill relative_dir must begin with public/ or custom/")
|
||||
return normalized
|
||||
|
||||
@field_validator("tree_hash")
|
||||
@classmethod
|
||||
def _tree_hash(cls, value: str) -> str:
|
||||
normalized = value.lower()
|
||||
if not _TREE_HASH_RE.fullmatch(normalized):
|
||||
raise ValueError("tree_hash must be a SHA-256 hex digest")
|
||||
return normalized
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _category_matches_directory(self):
|
||||
if PurePosixPath(self.relative_dir).parts[0] != self.category:
|
||||
raise ValueError("skill category must match relative_dir")
|
||||
return self
|
||||
|
||||
|
||||
class SupportDirectoryManifestItem(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
relative_dir: str
|
||||
tree_hash: str
|
||||
required_by: list[str] = Field(default_factory=list, max_length=5000)
|
||||
|
||||
@field_validator("relative_dir")
|
||||
@classmethod
|
||||
def _relative_dir(cls, value: str) -> str:
|
||||
normalized = _safe_relative_path(value)
|
||||
if PurePosixPath(normalized).parts[0] not in {"public", "custom"}:
|
||||
raise ValueError("support relative_dir must begin with public/ or custom/")
|
||||
return normalized
|
||||
|
||||
@field_validator("tree_hash")
|
||||
@classmethod
|
||||
def _tree_hash(cls, value: str) -> str:
|
||||
normalized = value.lower()
|
||||
if not _TREE_HASH_RE.fullmatch(normalized):
|
||||
raise ValueError("tree_hash must be a SHA-256 hex digest")
|
||||
return normalized
|
||||
|
||||
|
||||
class MigrationManifest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
schema_version: Literal[1] = 1
|
||||
bundle_id: str = Field(min_length=1, max_length=128)
|
||||
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
agents: list[AgentManifestItem] = Field(default_factory=list, max_length=1000)
|
||||
skills: list[SkillManifestItem] = Field(default_factory=list, max_length=5000)
|
||||
support_directories: list[SupportDirectoryManifestItem] = Field(default_factory=list, max_length=1000)
|
||||
warnings: list[str] = Field(default_factory=list, max_length=1000)
|
||||
|
||||
|
||||
class ResourcePreflightItem(BaseModel):
|
||||
kind: Literal["agent", "skill", "support"]
|
||||
key: str
|
||||
relative_dir: str
|
||||
status: ResourceStatus
|
||||
message: str = ""
|
||||
|
||||
|
||||
class OwnerMappingItem(BaseModel):
|
||||
source_user_id: str | None = None
|
||||
email: str | None = None
|
||||
target_user_id: str | None = None
|
||||
status: OwnerStatus
|
||||
|
||||
|
||||
class MigrationPreflight(BaseModel):
|
||||
ready: bool
|
||||
agents: list[ResourcePreflightItem]
|
||||
skills: list[ResourcePreflightItem]
|
||||
support_directories: list[ResourcePreflightItem]
|
||||
owner_mappings: list[OwnerMappingItem]
|
||||
|
||||
|
||||
class MigrationApplyResult(BaseModel):
|
||||
bundle_id: str
|
||||
agents_applied: int
|
||||
skills_applied: int
|
||||
skills_skipped: int = 0
|
||||
skipped_skill_names: list[str] = Field(default_factory=list)
|
||||
tags_created: int
|
||||
fallback_owner_count: int
|
||||
|
||||
|
||||
def _resolved_child(root: Path, relative_dir: str) -> Path | None:
|
||||
root = root.resolve()
|
||||
child = root
|
||||
for part in PurePosixPath(relative_dir).parts:
|
||||
child /= part
|
||||
if child.is_symlink():
|
||||
return None
|
||||
child = child.resolve()
|
||||
try:
|
||||
child.relative_to(root)
|
||||
except ValueError:
|
||||
return None
|
||||
return child
|
||||
|
||||
|
||||
def _agent_identity(directory: Path) -> str | None:
|
||||
config_path = directory / "config.yaml"
|
||||
if not config_path.is_file():
|
||||
return None
|
||||
try:
|
||||
payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
|
||||
except (OSError, UnicodeError, yaml.YAMLError):
|
||||
return None
|
||||
return str(payload.get("id") or "").strip() or None if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
def _skill_identity(directory: Path) -> str | None:
|
||||
skill_path = directory / "SKILL.md"
|
||||
if not skill_path.is_file():
|
||||
return None
|
||||
try:
|
||||
text = skill_path.read_text(encoding="utf-8")
|
||||
if not text.startswith("---"):
|
||||
return None
|
||||
parts = text.split("---", 2)
|
||||
payload = yaml.safe_load(parts[1]) or {}
|
||||
except (OSError, UnicodeError, yaml.YAMLError, IndexError):
|
||||
return None
|
||||
return str(payload.get("name") or "").strip() or None if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
def _resource_preflight(
|
||||
*,
|
||||
kind: Literal["agent", "skill", "support"],
|
||||
key: str,
|
||||
root: Path,
|
||||
relative_dir: str,
|
||||
expected_hash: str,
|
||||
expected_identity: str | None = None,
|
||||
) -> ResourcePreflightItem:
|
||||
directory = _resolved_child(root, relative_dir)
|
||||
if directory is None:
|
||||
return ResourcePreflightItem(
|
||||
kind=kind,
|
||||
key=key,
|
||||
relative_dir=relative_dir,
|
||||
status="invalid_path",
|
||||
message="目录路径不安全、包含符号链接或超出运行根目录",
|
||||
)
|
||||
if not directory.is_dir():
|
||||
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="missing", message="目标运行目录中未找到该目录")
|
||||
if expected_identity is not None:
|
||||
identity_file = directory / ("config.yaml" if kind == "agent" else "SKILL.md")
|
||||
if not identity_file.is_file():
|
||||
return ResourcePreflightItem(
|
||||
kind=kind,
|
||||
key=key,
|
||||
relative_dir=relative_dir,
|
||||
status="missing",
|
||||
message=f"目录中缺少 {identity_file.name}",
|
||||
)
|
||||
try:
|
||||
actual_hash = compute_tree_hash(directory)
|
||||
except (OSError, ValueError) as exc:
|
||||
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="hash_mismatch", message=str(exc))
|
||||
if actual_hash != expected_hash:
|
||||
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="hash_mismatch", message="目录内容与导出清单不一致")
|
||||
if expected_identity is not None:
|
||||
actual_identity = _agent_identity(directory) if kind == "agent" else _skill_identity(directory)
|
||||
if actual_identity != expected_identity:
|
||||
return ResourcePreflightItem(
|
||||
kind=kind,
|
||||
key=key,
|
||||
relative_dir=relative_dir,
|
||||
status="identity_mismatch",
|
||||
message=f"目录内标识为 {actual_identity or '空'},清单标识为 {expected_identity}",
|
||||
)
|
||||
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="matched")
|
||||
|
||||
|
||||
def preflight_manifest(
|
||||
manifest: MigrationManifest,
|
||||
*,
|
||||
agents_root: Path,
|
||||
skills_root: Path,
|
||||
users_by_email: dict[str, str],
|
||||
) -> MigrationPreflight:
|
||||
"""Compare a manifest with target live roots without modifying anything."""
|
||||
|
||||
normalized_users = {email.strip().casefold(): user_id for email, user_id in users_by_email.items()}
|
||||
resource_specs: list[dict[str, Any]] = [
|
||||
{
|
||||
"kind": "agent",
|
||||
"key": item.id,
|
||||
"root": agents_root,
|
||||
"relative_dir": item.relative_dir,
|
||||
"expected_hash": item.tree_hash,
|
||||
"expected_identity": item.id,
|
||||
}
|
||||
for item in manifest.agents
|
||||
]
|
||||
resource_specs.extend(
|
||||
{
|
||||
"kind": "skill",
|
||||
"key": f"{item.category}:{item.name}",
|
||||
"root": skills_root,
|
||||
"relative_dir": item.relative_dir,
|
||||
"expected_hash": item.tree_hash,
|
||||
"expected_identity": item.name,
|
||||
}
|
||||
for item in manifest.skills
|
||||
)
|
||||
resource_specs.extend(
|
||||
{
|
||||
"kind": "support",
|
||||
"key": item.relative_dir,
|
||||
"root": skills_root,
|
||||
"relative_dir": item.relative_dir,
|
||||
"expected_hash": item.tree_hash,
|
||||
}
|
||||
for item in manifest.support_directories
|
||||
)
|
||||
|
||||
def _check_resource(spec: dict[str, Any]) -> ResourcePreflightItem:
|
||||
return _resource_preflight(**spec)
|
||||
|
||||
if len(resource_specs) > 1:
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=min(_TREE_HASH_MAX_WORKERS, len(resource_specs)),
|
||||
thread_name_prefix="skill-migration-hash",
|
||||
) as executor:
|
||||
resources = list(executor.map(_check_resource, resource_specs))
|
||||
else:
|
||||
resources = [_check_resource(spec) for spec in resource_specs]
|
||||
|
||||
agent_count = len(manifest.agents)
|
||||
skill_count = len(manifest.skills)
|
||||
agents = resources[:agent_count]
|
||||
skills = resources[agent_count : agent_count + skill_count]
|
||||
support = resources[agent_count + skill_count :]
|
||||
|
||||
owners: dict[tuple[str | None, str | None], OwnerMappingItem] = {}
|
||||
for resource in [*manifest.agents, *manifest.skills]:
|
||||
owner = resource.owner
|
||||
if owner is None or (owner.source_user_id is None and owner.email is None):
|
||||
continue
|
||||
email = owner.email.strip().casefold() if owner.email else None
|
||||
target_user_id = normalized_users.get(email) if email else None
|
||||
key = (owner.source_user_id, email)
|
||||
owners[key] = OwnerMappingItem(
|
||||
source_user_id=owner.source_user_id,
|
||||
email=email,
|
||||
target_user_id=target_user_id,
|
||||
status="matched" if target_user_id else "unresolved",
|
||||
)
|
||||
|
||||
owner_mappings = list(owners.values())
|
||||
ready = all(item.status == "matched" for item in [*agents, *skills, *support]) and all(item.status == "matched" for item in owner_mappings)
|
||||
return MigrationPreflight(
|
||||
ready=ready,
|
||||
agents=agents,
|
||||
skills=skills,
|
||||
support_directories=support,
|
||||
owner_mappings=owner_mappings,
|
||||
)
|
||||
|
||||
|
||||
def prepare_manifest_for_apply(
|
||||
manifest: MigrationManifest,
|
||||
preflight: MigrationPreflight,
|
||||
*,
|
||||
skip_invalid_skills: bool,
|
||||
) -> tuple[MigrationManifest, MigrationPreflight, list[str]]:
|
||||
"""Optionally remove invalid skills and support-directory dependants.
|
||||
|
||||
Skipping is intentionally limited to skill-only manifests. Agent manifests
|
||||
retain all-or-nothing semantics because their live ``config.yaml`` is the
|
||||
authoritative source for several related database caches.
|
||||
"""
|
||||
|
||||
if not skip_invalid_skills:
|
||||
return manifest, preflight, []
|
||||
if manifest.agents:
|
||||
raise ValueError("跳过异常条目仅支持独立技能清单")
|
||||
|
||||
failed_skill_dirs = {
|
||||
item.relative_dir for item in preflight.skills if item.status != "matched"
|
||||
}
|
||||
skipped_names = {
|
||||
item.name for item in manifest.skills if item.relative_dir in failed_skill_dirs
|
||||
}
|
||||
support_by_dir = {
|
||||
item.relative_dir: item for item in manifest.support_directories
|
||||
}
|
||||
for resource in preflight.support_directories:
|
||||
if resource.status == "matched":
|
||||
continue
|
||||
support = support_by_dir.get(resource.relative_dir)
|
||||
if support is not None:
|
||||
skipped_names.update(support.required_by)
|
||||
|
||||
ordered_skipped = [item.name for item in manifest.skills if item.name in skipped_names]
|
||||
kept_skills = [item for item in manifest.skills if item.name not in skipped_names]
|
||||
kept_names = {item.name for item in kept_skills}
|
||||
kept_support = [
|
||||
item
|
||||
for item in manifest.support_directories
|
||||
if any(name in kept_names for name in item.required_by)
|
||||
]
|
||||
kept_skill_dirs = {item.relative_dir for item in kept_skills}
|
||||
kept_support_dirs = {item.relative_dir for item in kept_support}
|
||||
|
||||
owner_keys = {
|
||||
(
|
||||
item.owner.source_user_id,
|
||||
item.owner.email.strip().casefold() if item.owner and item.owner.email else None,
|
||||
)
|
||||
for item in kept_skills
|
||||
if item.owner is not None
|
||||
and (item.owner.source_user_id is not None or item.owner.email is not None)
|
||||
}
|
||||
owner_mappings = [
|
||||
item
|
||||
for item in preflight.owner_mappings
|
||||
if (
|
||||
item.source_user_id,
|
||||
item.email.strip().casefold() if item.email else None,
|
||||
)
|
||||
in owner_keys
|
||||
]
|
||||
skill_resources = [
|
||||
item for item in preflight.skills if item.relative_dir in kept_skill_dirs
|
||||
]
|
||||
support_resources = [
|
||||
item
|
||||
for item in preflight.support_directories
|
||||
if item.relative_dir in kept_support_dirs
|
||||
]
|
||||
ready = all(
|
||||
item.status == "matched"
|
||||
for item in [*preflight.agents, *skill_resources, *support_resources]
|
||||
) and all(item.status == "matched" for item in owner_mappings)
|
||||
|
||||
effective_manifest = manifest.model_copy(
|
||||
update={
|
||||
"skills": kept_skills,
|
||||
"support_directories": kept_support,
|
||||
}
|
||||
)
|
||||
effective_preflight = preflight.model_copy(
|
||||
update={
|
||||
"ready": ready,
|
||||
"skills": skill_resources,
|
||||
"support_directories": support_resources,
|
||||
"owner_mappings": owner_mappings,
|
||||
}
|
||||
)
|
||||
return effective_manifest, effective_preflight, ordered_skipped
|
||||
|
||||
|
||||
def _owner_reference(user_id: str | None, users_by_id: dict[str, str]) -> OwnerReference | None:
|
||||
if user_id is None:
|
||||
return None
|
||||
return OwnerReference(source_user_id=user_id, email=users_by_id.get(user_id))
|
||||
|
||||
|
||||
def _load_agent_config(directory: Path) -> dict[str, Any]:
|
||||
try:
|
||||
payload = yaml.safe_load((directory / "config.yaml").read_text(encoding="utf-8")) or {}
|
||||
except (OSError, UnicodeError, yaml.YAMLError) as exc:
|
||||
raise ValueError(f"Cannot read agent config: {directory}") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"Agent config must be an object: {directory}")
|
||||
return payload
|
||||
|
||||
|
||||
def _manifest_tags(rows: list[dict[str, Any]]) -> list[TagManifestItem]:
|
||||
return [
|
||||
TagManifestItem(
|
||||
id=str(row.get("id") or ""),
|
||||
name=str(row.get("name") or ""),
|
||||
color=str(row.get("color") or "gray"),
|
||||
description=str(row.get("description") or ""),
|
||||
)
|
||||
for row in rows
|
||||
if row.get("id") and row.get("name")
|
||||
]
|
||||
|
||||
|
||||
async def build_agent_migration_manifest(
|
||||
agent_ids: list[str],
|
||||
*,
|
||||
agent_store: Any,
|
||||
tag_store: Any,
|
||||
agents_root: Path,
|
||||
users_by_id: dict[str, str],
|
||||
) -> MigrationManifest:
|
||||
"""Build an agent manifest without inspecting or exporting skill payloads."""
|
||||
|
||||
selected_ids = list(dict.fromkeys(agent_id.strip() for agent_id in agent_ids if agent_id.strip()))
|
||||
if not selected_ids:
|
||||
raise ValueError("Select at least one agent")
|
||||
rows_by_id = {str(row["id"]): row for row in await agent_store.list_all()}
|
||||
missing_agents = [agent_id for agent_id in selected_ids if agent_id not in rows_by_id]
|
||||
if missing_agents:
|
||||
raise ValueError(f"Unknown agents: {', '.join(missing_agents)}")
|
||||
|
||||
agent_tag_map = await tag_store.list_assignments("agent", selected_ids)
|
||||
agents: list[AgentManifestItem] = []
|
||||
warnings: list[str] = []
|
||||
for agent_id in selected_ids:
|
||||
row = rows_by_id[agent_id]
|
||||
directory = _resolved_child(agents_root, agent_id.lower())
|
||||
if directory is None or not directory.is_dir():
|
||||
raise ValueError(f"Agent directory is missing: {agent_id}")
|
||||
config = _load_agent_config(directory)
|
||||
if str(config.get("id") or "") != agent_id:
|
||||
raise ValueError(f"Agent config id does not match directory: {agent_id}")
|
||||
raw_skills = config.get("skills")
|
||||
selected_skills = list(dict.fromkeys(item.strip() for item in raw_skills if isinstance(item, str) and item.strip())) if isinstance(raw_skills, list) else []
|
||||
knowledge_ids = config.get("llmwiki_knowledge_base_ids")
|
||||
if isinstance(knowledge_ids, list) and any(str(item).strip() for item in knowledge_ids):
|
||||
warnings.append(f"智能体 {agent_id} 绑定了知识库;知识库数据不在本迁移清单范围内,请在目标机器单独确认")
|
||||
agents.append(
|
||||
AgentManifestItem(
|
||||
id=agent_id,
|
||||
relative_dir=directory.relative_to(agents_root.resolve()).as_posix(),
|
||||
tree_hash=await asyncio.to_thread(compute_tree_hash, directory),
|
||||
name=str(row.get("name") or config.get("name") or agent_id),
|
||||
description=str(row.get("description") or ""),
|
||||
owner=_owner_reference(row.get("user_id"), users_by_id),
|
||||
published=bool(row.get("published")),
|
||||
is_private=bool(row.get("is_private")),
|
||||
square_id=str(row.get("square_id") or ""),
|
||||
featured_order=row.get("featured_order"),
|
||||
created_at=row.get("created_at"),
|
||||
updated_at=row.get("updated_at"),
|
||||
skills=selected_skills,
|
||||
tags=_manifest_tags(agent_tag_map.get(agent_id, [])),
|
||||
)
|
||||
)
|
||||
|
||||
return MigrationManifest(
|
||||
bundle_id=uuid4().hex,
|
||||
agents=agents,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
async def build_skill_migration_manifest(
|
||||
skill_names: list[str],
|
||||
*,
|
||||
skill_store: Any,
|
||||
tag_store: Any,
|
||||
skills_root: Path,
|
||||
discovered_skills: list[Any],
|
||||
users_by_id: dict[str, str],
|
||||
usage_entries: dict[str, dict[str, Any]],
|
||||
) -> MigrationManifest:
|
||||
"""Build a metadata-only manifest for explicitly selected live skills."""
|
||||
|
||||
selected_names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
|
||||
if not selected_names:
|
||||
raise ValueError("Select at least one skill")
|
||||
|
||||
skill_rows = {str(row["name"]): row for row in await skill_store.list_all()}
|
||||
discovered = {str(skill.name): skill for skill in discovered_skills}
|
||||
missing_skills = [name for name in selected_names if name not in discovered]
|
||||
if missing_skills:
|
||||
raise ValueError(f"Skills are missing from the live runtime directory: {', '.join(missing_skills)}")
|
||||
|
||||
skill_tag_map = await tag_store.list_assignments("skill", selected_names)
|
||||
skill_directories = [Path(discovered[name].skill_dir).resolve() for name in selected_names]
|
||||
tree_hashes = await _compute_tree_hashes(skill_directories)
|
||||
skills: list[SkillManifestItem] = []
|
||||
for name, tree_hash in zip(selected_names, tree_hashes, strict=True):
|
||||
skill = discovered[name]
|
||||
category = str(skill.category)
|
||||
relative = PurePosixPath(category) / PurePosixPath(Path(skill.relative_path).as_posix())
|
||||
directory = Path(skill.skill_dir).resolve()
|
||||
row = skill_rows.get(name, {})
|
||||
usage = usage_entries.get(name, {})
|
||||
state = str(usage.get("state") or "active")
|
||||
if state not in {"active", "stale", "archived"}:
|
||||
state = "active"
|
||||
source = str(usage.get("source") or "upload")
|
||||
if source not in {"agent", "upload"}:
|
||||
source = "upload"
|
||||
skills.append(
|
||||
SkillManifestItem(
|
||||
name=name,
|
||||
category=category,
|
||||
relative_dir=relative.as_posix(),
|
||||
tree_hash=tree_hash,
|
||||
owner=_owner_reference(row.get("owner_user_id"), users_by_id),
|
||||
published=bool(row.get("published", category == "public")),
|
||||
detail=row.get("detail"),
|
||||
name_zh=row.get("name_zh"),
|
||||
always_on=bool(row.get("always_on", getattr(skill, "always_on", False))),
|
||||
square_id=str(row.get("square_id") or ""),
|
||||
enabled=bool(skill.enabled),
|
||||
state=state,
|
||||
pinned=bool(usage.get("pinned")),
|
||||
featured=bool(usage.get("featured")),
|
||||
source=source,
|
||||
created_at=row.get("created_at") or usage.get("created_at"),
|
||||
updated_at=row.get("updated_at"),
|
||||
tags=_manifest_tags(skill_tag_map.get(name, [])),
|
||||
)
|
||||
)
|
||||
|
||||
selected_set = set(selected_names)
|
||||
warnings: list[str] = []
|
||||
support_directories: list[SupportDirectoryManifestItem] = []
|
||||
support_candidates, support_warnings = await asyncio.to_thread(
|
||||
_find_support_directories,
|
||||
skills_root,
|
||||
selected_set,
|
||||
)
|
||||
warnings.extend(support_warnings)
|
||||
support_hashes = await _compute_tree_hashes(
|
||||
[directory for directory, _required_by in support_candidates]
|
||||
)
|
||||
for (directory, selected_required_by), tree_hash in zip(
|
||||
support_candidates,
|
||||
support_hashes,
|
||||
strict=True,
|
||||
):
|
||||
support_directories.append(
|
||||
SupportDirectoryManifestItem(
|
||||
relative_dir=directory.relative_to(skills_root.resolve()).as_posix(),
|
||||
tree_hash=tree_hash,
|
||||
required_by=selected_required_by,
|
||||
)
|
||||
)
|
||||
|
||||
return MigrationManifest(
|
||||
bundle_id=uuid4().hex,
|
||||
skills=skills,
|
||||
support_directories=support_directories,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
async def _compute_tree_hashes(directories: list[Path]) -> list[str]:
|
||||
if not directories:
|
||||
return []
|
||||
semaphore = asyncio.Semaphore(min(_TREE_HASH_MAX_WORKERS, len(directories)))
|
||||
|
||||
async def _compute(directory: Path) -> str:
|
||||
async with semaphore:
|
||||
return await asyncio.to_thread(compute_tree_hash, directory)
|
||||
|
||||
return list(await asyncio.gather(*(_compute(directory) for directory in directories)))
|
||||
|
||||
|
||||
def _find_support_directories(
|
||||
skills_root: Path,
|
||||
selected_names: set[str],
|
||||
) -> tuple[list[tuple[Path, list[str]]], list[str]]:
|
||||
candidates: list[tuple[Path, list[str]]] = []
|
||||
warnings: list[str] = []
|
||||
for support_manifest in sorted(skills_root.rglob(".deerflow-skill-support.json")):
|
||||
try:
|
||||
payload = json.loads(support_manifest.read_text(encoding="utf-8"))
|
||||
required_by = [str(item).strip() for item in payload.get("required_by", []) if str(item).strip()]
|
||||
except (OSError, UnicodeError, json.JSONDecodeError, AttributeError):
|
||||
warnings.append(f"无法读取技能支持目录清单:{support_manifest}")
|
||||
continue
|
||||
selected_required_by = [name for name in required_by if name in selected_names]
|
||||
if not selected_required_by:
|
||||
continue
|
||||
candidates.append((support_manifest.parent.resolve(), selected_required_by))
|
||||
return candidates, warnings
|
||||
|
||||
|
||||
def _apply_usage_metadata(name: str, metadata: dict[str, Any]) -> None:
|
||||
_apply_usage_metadata_batch({name: metadata})
|
||||
|
||||
|
||||
def _apply_usage_metadata_batch(entries: dict[str, dict[str, Any]]) -> None:
|
||||
from deerflow.skills.usage import set_migration_metadata_batch
|
||||
|
||||
set_migration_metadata_batch(entries)
|
||||
|
||||
|
||||
def _target_owner_id(
|
||||
owner: OwnerReference | None,
|
||||
*,
|
||||
users_by_email: dict[str, str],
|
||||
fallback_owner_user_id: str | None,
|
||||
) -> tuple[str | None, bool]:
|
||||
if owner is None or (owner.source_user_id is None and owner.email is None):
|
||||
return None, False
|
||||
target = users_by_email.get(owner.email.casefold()) if owner.email else None
|
||||
if target:
|
||||
return target, False
|
||||
if fallback_owner_user_id:
|
||||
return fallback_owner_user_id, True
|
||||
raise ValueError(f"Unresolved owner: {owner.email or owner.source_user_id}")
|
||||
|
||||
|
||||
async def _apply_tags(tag_store: Any, target_type: str, target_id: str, tags: list[TagManifestItem]) -> int:
|
||||
return await _apply_tags_bulk(tag_store, target_type, [(target_id, tags)])
|
||||
|
||||
|
||||
async def _apply_tags_bulk(
|
||||
tag_store: Any,
|
||||
target_type: str,
|
||||
targets: list[tuple[str, list[TagManifestItem]]],
|
||||
) -> int:
|
||||
if not targets:
|
||||
return 0
|
||||
existing = await tag_store.list_tags()
|
||||
by_name = {
|
||||
(str(row.get("tag_type")), str(row.get("name"))): row
|
||||
for row in existing
|
||||
}
|
||||
used_ids = {str(row.get("id")) for row in existing}
|
||||
assignments: dict[str, list[str]] = {}
|
||||
created = 0
|
||||
for target_id, tags in targets:
|
||||
tag_ids: list[str] = []
|
||||
for tag in tags:
|
||||
same_name = by_name.get((target_type, tag.name))
|
||||
if same_name is not None:
|
||||
tag_ids.append(str(same_name["id"]))
|
||||
continue
|
||||
tag_id = tag.id if tag.id not in used_ids else uuid4().hex
|
||||
created_tag = await tag_store.create_tag(
|
||||
{
|
||||
"id": tag_id,
|
||||
"name": tag.name,
|
||||
"tag_type": target_type,
|
||||
"color": tag.color,
|
||||
"description": tag.description,
|
||||
}
|
||||
)
|
||||
by_name[(target_type, tag.name)] = created_tag
|
||||
used_ids.add(tag_id)
|
||||
tag_ids.append(tag_id)
|
||||
created += 1
|
||||
assignments[target_id] = tag_ids
|
||||
await tag_store.set_target_tags_bulk(target_type, assignments)
|
||||
return created
|
||||
|
||||
|
||||
async def apply_migration_manifest(
|
||||
manifest: MigrationManifest,
|
||||
*,
|
||||
preflight: MigrationPreflight,
|
||||
agent_store: Any,
|
||||
skill_store: Any,
|
||||
tag_store: Any,
|
||||
agents_root: Path,
|
||||
users_by_email: dict[str, str],
|
||||
fallback_owner_user_id: str | None,
|
||||
extensions_config_path: Path | None,
|
||||
skip_invalid_skills: bool = False,
|
||||
) -> MigrationApplyResult:
|
||||
"""Idempotently reconcile metadata after all live folders passed preflight."""
|
||||
|
||||
manifest, preflight, skipped_skill_names = prepare_manifest_for_apply(
|
||||
manifest,
|
||||
preflight,
|
||||
skip_invalid_skills=skip_invalid_skills,
|
||||
)
|
||||
|
||||
bad_resources = [item for item in [*preflight.agents, *preflight.skills, *preflight.support_directories] if item.status != "matched"]
|
||||
if bad_resources:
|
||||
raise ValueError("Migration preflight contains missing or mismatched directories")
|
||||
if any(item.status == "unresolved" for item in preflight.owner_mappings) and not fallback_owner_user_id:
|
||||
raise ValueError("Migration preflight contains unresolved owners")
|
||||
|
||||
if manifest.skills:
|
||||
if extensions_config_path is None:
|
||||
raise ValueError("目标运行环境未找到 extensions_config.json")
|
||||
from deerflow.config.extensions_config import patch_extensions_config
|
||||
|
||||
enabled_by_name = {item.name: item.enabled for item in manifest.skills}
|
||||
|
||||
def _patch(document: dict[str, Any]) -> None:
|
||||
skills_document = document.setdefault("skills", {})
|
||||
if not isinstance(skills_document, dict):
|
||||
raise ValueError("extensions_config.json skills must be an object")
|
||||
for name, enabled in enabled_by_name.items():
|
||||
current = skills_document.get(name)
|
||||
entry = dict(current) if isinstance(current, dict) else {}
|
||||
entry["enabled"] = enabled
|
||||
skills_document[name] = entry
|
||||
|
||||
patch_extensions_config(extensions_config_path, _patch)
|
||||
|
||||
normalized_users = {email.strip().casefold(): user_id for email, user_id in users_by_email.items()}
|
||||
fallback_count = 0
|
||||
tags_created = 0
|
||||
skill_payloads: list[dict[str, Any]] = []
|
||||
skill_usage_entries: dict[str, dict[str, Any]] = {}
|
||||
for item in manifest.skills:
|
||||
owner_user_id, used_fallback = _target_owner_id(
|
||||
item.owner,
|
||||
users_by_email=normalized_users,
|
||||
fallback_owner_user_id=fallback_owner_user_id,
|
||||
)
|
||||
fallback_count += int(used_fallback)
|
||||
skill_payloads.append(
|
||||
{
|
||||
"name": item.name,
|
||||
"owner_user_id": owner_user_id,
|
||||
"published": item.published,
|
||||
"detail": item.detail,
|
||||
"name_zh": item.name_zh,
|
||||
"always_on": item.always_on,
|
||||
"square_id": item.square_id,
|
||||
"created_at": item.created_at,
|
||||
"updated_at": item.updated_at,
|
||||
}
|
||||
)
|
||||
skill_usage_entries[item.name] = {
|
||||
"state": item.state,
|
||||
"pinned": item.pinned,
|
||||
"featured": item.featured,
|
||||
"source": item.source,
|
||||
}
|
||||
|
||||
if skill_payloads:
|
||||
await skill_store.migration_upsert_many(skill_payloads)
|
||||
tags_created += await _apply_tags_bulk(
|
||||
tag_store,
|
||||
"skill",
|
||||
[(item.name, item.tags) for item in manifest.skills],
|
||||
)
|
||||
_apply_usage_metadata_batch(skill_usage_entries)
|
||||
|
||||
agent_tag_targets: list[tuple[str, list[TagManifestItem]]] = []
|
||||
for item in manifest.agents:
|
||||
owner_user_id, used_fallback = _target_owner_id(
|
||||
item.owner,
|
||||
users_by_email=normalized_users,
|
||||
fallback_owner_user_id=fallback_owner_user_id,
|
||||
)
|
||||
fallback_count += int(used_fallback)
|
||||
await agent_store.migration_upsert(
|
||||
{
|
||||
"id": item.id,
|
||||
"name": item.name,
|
||||
"description": item.description,
|
||||
"user_id": owner_user_id,
|
||||
"published": item.published,
|
||||
"is_private": item.is_private,
|
||||
"square_id": item.square_id,
|
||||
"featured_order": item.featured_order,
|
||||
"created_at": item.created_at,
|
||||
"updated_at": item.updated_at,
|
||||
}
|
||||
)
|
||||
config = _load_agent_config(agents_root / item.relative_dir)
|
||||
await agent_store.replace_skills(item.id, config.get("skills") if isinstance(config.get("skills"), list) else [])
|
||||
await agent_store.set_extras(
|
||||
item.id,
|
||||
tool_groups=config.get("tool_groups") if isinstance(config.get("tool_groups"), list) else None,
|
||||
model=str(config.get("model")) if config.get("model") else None,
|
||||
)
|
||||
agent_tag_targets.append((item.id, item.tags))
|
||||
|
||||
tags_created += await _apply_tags_bulk(tag_store, "agent", agent_tag_targets)
|
||||
|
||||
return MigrationApplyResult(
|
||||
bundle_id=manifest.bundle_id,
|
||||
agents_applied=len(manifest.agents),
|
||||
skills_applied=len(manifest.skills),
|
||||
skills_skipped=len(skipped_skill_names),
|
||||
skipped_skill_names=skipped_skill_names,
|
||||
tags_created=tags_created,
|
||||
fallback_owner_count=fallback_count,
|
||||
)
|
||||
Reference in new issue
Block a user