925 lines
34 KiB
Python
925 lines
34 KiB
Python
"""Agent/skill metadata migration manifests for already-copied live folders.
|
|
|
|
The manifest deliberately contains no agent or skill payload files. Operators
|
|
copy those directories with their normal offline transfer process; these
|
|
helpers only prove that the target runtime folders match the exported inventory
|
|
before database/configuration metadata is reconciled.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import Any, Literal
|
|
from uuid import uuid4
|
|
|
|
import yaml
|
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
|
|
ResourceStatus = Literal["matched", "missing", "hash_mismatch", "identity_mismatch", "invalid_path"]
|
|
OwnerStatus = Literal["matched", "unresolved"]
|
|
|
|
_TREE_HASH_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
_TREE_HASH_MAX_WORKERS = 4
|
|
|
|
|
|
def _safe_relative_path(value: str) -> str:
|
|
normalized = value.strip()
|
|
if not normalized or "\\" in normalized or ":" in normalized:
|
|
raise ValueError("relative_dir must be a non-empty portable relative path")
|
|
path = PurePosixPath(normalized)
|
|
if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
|
|
raise ValueError("relative_dir must stay inside the configured runtime root")
|
|
return path.as_posix()
|
|
|
|
|
|
def compute_tree_hash(root: Path) -> str:
|
|
"""Return a deterministic SHA-256 for a directory tree.
|
|
|
|
Relative paths and bytes are both signed. Symlinks are rejected so a
|
|
manifest cannot attest content outside the selected runtime directory.
|
|
"""
|
|
|
|
root = root.resolve()
|
|
if not root.is_dir():
|
|
raise FileNotFoundError(root)
|
|
digest = hashlib.sha256()
|
|
files = sorted((path for path in root.rglob("*") if path.is_file() or path.is_symlink()), key=lambda p: p.relative_to(root).as_posix())
|
|
for path in files:
|
|
if path.is_symlink():
|
|
raise ValueError(f"Symlinks are not supported in migration trees: {path}")
|
|
relative = path.relative_to(root).as_posix().encode("utf-8")
|
|
digest.update(len(relative).to_bytes(4, "big"))
|
|
digest.update(relative)
|
|
digest.update(path.stat().st_size.to_bytes(8, "big"))
|
|
with path.open("rb") as stream:
|
|
while chunk := stream.read(1024 * 1024):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
class OwnerReference(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
source_user_id: str | None = None
|
|
email: str | None = None
|
|
|
|
@field_validator("source_user_id", "email")
|
|
@classmethod
|
|
def _strip_optional(cls, value: str | None) -> str | None:
|
|
return value.strip() if value and value.strip() else None
|
|
|
|
|
|
class TagManifestItem(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
id: str = Field(min_length=1, max_length=64)
|
|
name: str = Field(min_length=1, max_length=128)
|
|
color: str = Field(default="gray", max_length=32)
|
|
description: str = Field(default="", max_length=1024)
|
|
|
|
|
|
class AgentManifestItem(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
id: str
|
|
relative_dir: str
|
|
tree_hash: str
|
|
name: str
|
|
description: str = ""
|
|
owner: OwnerReference | None = None
|
|
published: bool = False
|
|
is_private: bool = False
|
|
square_id: str = ""
|
|
featured_order: int | None = None
|
|
created_at: datetime | None = None
|
|
updated_at: datetime | None = None
|
|
skills: list[str] = Field(default_factory=list, max_length=5000)
|
|
tags: list[TagManifestItem] = Field(default_factory=list, max_length=200)
|
|
|
|
@field_validator("relative_dir")
|
|
@classmethod
|
|
def _relative_dir(cls, value: str) -> str:
|
|
return _safe_relative_path(value)
|
|
|
|
@field_validator("tree_hash")
|
|
@classmethod
|
|
def _tree_hash(cls, value: str) -> str:
|
|
normalized = value.lower()
|
|
if not _TREE_HASH_RE.fullmatch(normalized):
|
|
raise ValueError("tree_hash must be a SHA-256 hex digest")
|
|
return normalized
|
|
|
|
@model_validator(mode="after")
|
|
def _directory_matches_id(self):
|
|
if self.relative_dir != self.id.lower():
|
|
raise ValueError("agent relative_dir must equal the stable agent id")
|
|
return self
|
|
|
|
|
|
class SkillManifestItem(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
name: str
|
|
category: Literal["public", "custom"]
|
|
relative_dir: str
|
|
tree_hash: str
|
|
owner: OwnerReference | None = None
|
|
published: bool = False
|
|
detail: str | None = None
|
|
name_zh: str | None = None
|
|
always_on: bool = False
|
|
square_id: str = ""
|
|
enabled: bool = True
|
|
state: Literal["active", "stale", "archived"] = "active"
|
|
pinned: bool = False
|
|
featured: bool = False
|
|
source: Literal["agent", "upload"] = "upload"
|
|
created_at: datetime | None = None
|
|
updated_at: datetime | None = None
|
|
tags: list[TagManifestItem] = Field(default_factory=list, max_length=200)
|
|
|
|
@field_validator("relative_dir")
|
|
@classmethod
|
|
def _relative_dir(cls, value: str) -> str:
|
|
normalized = _safe_relative_path(value)
|
|
if PurePosixPath(normalized).parts[0] not in {"public", "custom"}:
|
|
raise ValueError("skill relative_dir must begin with public/ or custom/")
|
|
return normalized
|
|
|
|
@field_validator("tree_hash")
|
|
@classmethod
|
|
def _tree_hash(cls, value: str) -> str:
|
|
normalized = value.lower()
|
|
if not _TREE_HASH_RE.fullmatch(normalized):
|
|
raise ValueError("tree_hash must be a SHA-256 hex digest")
|
|
return normalized
|
|
|
|
@model_validator(mode="after")
|
|
def _category_matches_directory(self):
|
|
if PurePosixPath(self.relative_dir).parts[0] != self.category:
|
|
raise ValueError("skill category must match relative_dir")
|
|
return self
|
|
|
|
|
|
class SupportDirectoryManifestItem(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
relative_dir: str
|
|
tree_hash: str
|
|
required_by: list[str] = Field(default_factory=list, max_length=5000)
|
|
|
|
@field_validator("relative_dir")
|
|
@classmethod
|
|
def _relative_dir(cls, value: str) -> str:
|
|
normalized = _safe_relative_path(value)
|
|
if PurePosixPath(normalized).parts[0] not in {"public", "custom"}:
|
|
raise ValueError("support relative_dir must begin with public/ or custom/")
|
|
return normalized
|
|
|
|
@field_validator("tree_hash")
|
|
@classmethod
|
|
def _tree_hash(cls, value: str) -> str:
|
|
normalized = value.lower()
|
|
if not _TREE_HASH_RE.fullmatch(normalized):
|
|
raise ValueError("tree_hash must be a SHA-256 hex digest")
|
|
return normalized
|
|
|
|
|
|
class MigrationManifest(BaseModel):
|
|
model_config = ConfigDict(extra="forbid")
|
|
|
|
schema_version: Literal[1] = 1
|
|
bundle_id: str = Field(min_length=1, max_length=128)
|
|
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
agents: list[AgentManifestItem] = Field(default_factory=list, max_length=1000)
|
|
skills: list[SkillManifestItem] = Field(default_factory=list, max_length=5000)
|
|
support_directories: list[SupportDirectoryManifestItem] = Field(default_factory=list, max_length=1000)
|
|
warnings: list[str] = Field(default_factory=list, max_length=1000)
|
|
|
|
|
|
class ResourcePreflightItem(BaseModel):
|
|
kind: Literal["agent", "skill", "support"]
|
|
key: str
|
|
relative_dir: str
|
|
status: ResourceStatus
|
|
message: str = ""
|
|
|
|
|
|
class OwnerMappingItem(BaseModel):
|
|
source_user_id: str | None = None
|
|
email: str | None = None
|
|
target_user_id: str | None = None
|
|
status: OwnerStatus
|
|
|
|
|
|
class MigrationPreflight(BaseModel):
|
|
ready: bool
|
|
agents: list[ResourcePreflightItem]
|
|
skills: list[ResourcePreflightItem]
|
|
support_directories: list[ResourcePreflightItem]
|
|
owner_mappings: list[OwnerMappingItem]
|
|
|
|
|
|
class MigrationApplyResult(BaseModel):
|
|
bundle_id: str
|
|
agents_applied: int
|
|
skills_applied: int
|
|
skills_skipped: int = 0
|
|
skipped_skill_names: list[str] = Field(default_factory=list)
|
|
tags_created: int
|
|
fallback_owner_count: int
|
|
|
|
|
|
def _resolved_child(root: Path, relative_dir: str) -> Path | None:
|
|
root = root.resolve()
|
|
child = root
|
|
for part in PurePosixPath(relative_dir).parts:
|
|
child /= part
|
|
if child.is_symlink():
|
|
return None
|
|
child = child.resolve()
|
|
try:
|
|
child.relative_to(root)
|
|
except ValueError:
|
|
return None
|
|
return child
|
|
|
|
|
|
def _agent_identity(directory: Path) -> str | None:
|
|
config_path = directory / "config.yaml"
|
|
if not config_path.is_file():
|
|
return None
|
|
try:
|
|
payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
|
|
except (OSError, UnicodeError, yaml.YAMLError):
|
|
return None
|
|
return str(payload.get("id") or "").strip() or None if isinstance(payload, dict) else None
|
|
|
|
|
|
def _skill_identity(directory: Path) -> str | None:
|
|
skill_path = directory / "SKILL.md"
|
|
if not skill_path.is_file():
|
|
return None
|
|
try:
|
|
text = skill_path.read_text(encoding="utf-8")
|
|
if not text.startswith("---"):
|
|
return None
|
|
parts = text.split("---", 2)
|
|
payload = yaml.safe_load(parts[1]) or {}
|
|
except (OSError, UnicodeError, yaml.YAMLError, IndexError):
|
|
return None
|
|
return str(payload.get("name") or "").strip() or None if isinstance(payload, dict) else None
|
|
|
|
|
|
def _resource_preflight(
|
|
*,
|
|
kind: Literal["agent", "skill", "support"],
|
|
key: str,
|
|
root: Path,
|
|
relative_dir: str,
|
|
expected_hash: str,
|
|
expected_identity: str | None = None,
|
|
) -> ResourcePreflightItem:
|
|
directory = _resolved_child(root, relative_dir)
|
|
if directory is None:
|
|
return ResourcePreflightItem(
|
|
kind=kind,
|
|
key=key,
|
|
relative_dir=relative_dir,
|
|
status="invalid_path",
|
|
message="目录路径不安全、包含符号链接或超出运行根目录",
|
|
)
|
|
if not directory.is_dir():
|
|
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="missing", message="目标运行目录中未找到该目录")
|
|
if expected_identity is not None:
|
|
identity_file = directory / ("config.yaml" if kind == "agent" else "SKILL.md")
|
|
if not identity_file.is_file():
|
|
return ResourcePreflightItem(
|
|
kind=kind,
|
|
key=key,
|
|
relative_dir=relative_dir,
|
|
status="missing",
|
|
message=f"目录中缺少 {identity_file.name}",
|
|
)
|
|
try:
|
|
actual_hash = compute_tree_hash(directory)
|
|
except (OSError, ValueError) as exc:
|
|
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="hash_mismatch", message=str(exc))
|
|
if actual_hash != expected_hash:
|
|
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="hash_mismatch", message="目录内容与导出清单不一致")
|
|
if expected_identity is not None:
|
|
actual_identity = _agent_identity(directory) if kind == "agent" else _skill_identity(directory)
|
|
if actual_identity != expected_identity:
|
|
return ResourcePreflightItem(
|
|
kind=kind,
|
|
key=key,
|
|
relative_dir=relative_dir,
|
|
status="identity_mismatch",
|
|
message=f"目录内标识为 {actual_identity or '空'},清单标识为 {expected_identity}",
|
|
)
|
|
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="matched")
|
|
|
|
|
|
def preflight_manifest(
|
|
manifest: MigrationManifest,
|
|
*,
|
|
agents_root: Path,
|
|
skills_root: Path,
|
|
users_by_email: dict[str, str],
|
|
) -> MigrationPreflight:
|
|
"""Compare a manifest with target live roots without modifying anything."""
|
|
|
|
normalized_users = {email.strip().casefold(): user_id for email, user_id in users_by_email.items()}
|
|
resource_specs: list[dict[str, Any]] = [
|
|
{
|
|
"kind": "agent",
|
|
"key": item.id,
|
|
"root": agents_root,
|
|
"relative_dir": item.relative_dir,
|
|
"expected_hash": item.tree_hash,
|
|
"expected_identity": item.id,
|
|
}
|
|
for item in manifest.agents
|
|
]
|
|
resource_specs.extend(
|
|
{
|
|
"kind": "skill",
|
|
"key": f"{item.category}:{item.name}",
|
|
"root": skills_root,
|
|
"relative_dir": item.relative_dir,
|
|
"expected_hash": item.tree_hash,
|
|
"expected_identity": item.name,
|
|
}
|
|
for item in manifest.skills
|
|
)
|
|
resource_specs.extend(
|
|
{
|
|
"kind": "support",
|
|
"key": item.relative_dir,
|
|
"root": skills_root,
|
|
"relative_dir": item.relative_dir,
|
|
"expected_hash": item.tree_hash,
|
|
}
|
|
for item in manifest.support_directories
|
|
)
|
|
|
|
def _check_resource(spec: dict[str, Any]) -> ResourcePreflightItem:
|
|
return _resource_preflight(**spec)
|
|
|
|
if len(resource_specs) > 1:
|
|
with ThreadPoolExecutor(
|
|
max_workers=min(_TREE_HASH_MAX_WORKERS, len(resource_specs)),
|
|
thread_name_prefix="skill-migration-hash",
|
|
) as executor:
|
|
resources = list(executor.map(_check_resource, resource_specs))
|
|
else:
|
|
resources = [_check_resource(spec) for spec in resource_specs]
|
|
|
|
agent_count = len(manifest.agents)
|
|
skill_count = len(manifest.skills)
|
|
agents = resources[:agent_count]
|
|
skills = resources[agent_count : agent_count + skill_count]
|
|
support = resources[agent_count + skill_count :]
|
|
|
|
owners: dict[tuple[str | None, str | None], OwnerMappingItem] = {}
|
|
for resource in [*manifest.agents, *manifest.skills]:
|
|
owner = resource.owner
|
|
if owner is None or (owner.source_user_id is None and owner.email is None):
|
|
continue
|
|
email = owner.email.strip().casefold() if owner.email else None
|
|
target_user_id = normalized_users.get(email) if email else None
|
|
key = (owner.source_user_id, email)
|
|
owners[key] = OwnerMappingItem(
|
|
source_user_id=owner.source_user_id,
|
|
email=email,
|
|
target_user_id=target_user_id,
|
|
status="matched" if target_user_id else "unresolved",
|
|
)
|
|
|
|
owner_mappings = list(owners.values())
|
|
ready = all(item.status == "matched" for item in [*agents, *skills, *support]) and all(item.status == "matched" for item in owner_mappings)
|
|
return MigrationPreflight(
|
|
ready=ready,
|
|
agents=agents,
|
|
skills=skills,
|
|
support_directories=support,
|
|
owner_mappings=owner_mappings,
|
|
)
|
|
|
|
|
|
def prepare_manifest_for_apply(
|
|
manifest: MigrationManifest,
|
|
preflight: MigrationPreflight,
|
|
*,
|
|
skip_invalid_skills: bool,
|
|
) -> tuple[MigrationManifest, MigrationPreflight, list[str]]:
|
|
"""Optionally remove invalid skills and support-directory dependants.
|
|
|
|
Skipping is intentionally limited to skill-only manifests. Agent manifests
|
|
retain all-or-nothing semantics because their live ``config.yaml`` is the
|
|
authoritative source for several related database caches.
|
|
"""
|
|
|
|
if not skip_invalid_skills:
|
|
return manifest, preflight, []
|
|
if manifest.agents:
|
|
raise ValueError("跳过异常条目仅支持独立技能清单")
|
|
|
|
failed_skill_dirs = {
|
|
item.relative_dir for item in preflight.skills if item.status != "matched"
|
|
}
|
|
skipped_names = {
|
|
item.name for item in manifest.skills if item.relative_dir in failed_skill_dirs
|
|
}
|
|
support_by_dir = {
|
|
item.relative_dir: item for item in manifest.support_directories
|
|
}
|
|
for resource in preflight.support_directories:
|
|
if resource.status == "matched":
|
|
continue
|
|
support = support_by_dir.get(resource.relative_dir)
|
|
if support is not None:
|
|
skipped_names.update(support.required_by)
|
|
|
|
ordered_skipped = [item.name for item in manifest.skills if item.name in skipped_names]
|
|
kept_skills = [item for item in manifest.skills if item.name not in skipped_names]
|
|
kept_names = {item.name for item in kept_skills}
|
|
kept_support = [
|
|
item
|
|
for item in manifest.support_directories
|
|
if any(name in kept_names for name in item.required_by)
|
|
]
|
|
kept_skill_dirs = {item.relative_dir for item in kept_skills}
|
|
kept_support_dirs = {item.relative_dir for item in kept_support}
|
|
|
|
owner_keys = {
|
|
(
|
|
item.owner.source_user_id,
|
|
item.owner.email.strip().casefold() if item.owner and item.owner.email else None,
|
|
)
|
|
for item in kept_skills
|
|
if item.owner is not None
|
|
and (item.owner.source_user_id is not None or item.owner.email is not None)
|
|
}
|
|
owner_mappings = [
|
|
item
|
|
for item in preflight.owner_mappings
|
|
if (
|
|
item.source_user_id,
|
|
item.email.strip().casefold() if item.email else None,
|
|
)
|
|
in owner_keys
|
|
]
|
|
skill_resources = [
|
|
item for item in preflight.skills if item.relative_dir in kept_skill_dirs
|
|
]
|
|
support_resources = [
|
|
item
|
|
for item in preflight.support_directories
|
|
if item.relative_dir in kept_support_dirs
|
|
]
|
|
ready = all(
|
|
item.status == "matched"
|
|
for item in [*preflight.agents, *skill_resources, *support_resources]
|
|
) and all(item.status == "matched" for item in owner_mappings)
|
|
|
|
effective_manifest = manifest.model_copy(
|
|
update={
|
|
"skills": kept_skills,
|
|
"support_directories": kept_support,
|
|
}
|
|
)
|
|
effective_preflight = preflight.model_copy(
|
|
update={
|
|
"ready": ready,
|
|
"skills": skill_resources,
|
|
"support_directories": support_resources,
|
|
"owner_mappings": owner_mappings,
|
|
}
|
|
)
|
|
return effective_manifest, effective_preflight, ordered_skipped
|
|
|
|
|
|
def _owner_reference(user_id: str | None, users_by_id: dict[str, str]) -> OwnerReference | None:
|
|
if user_id is None:
|
|
return None
|
|
return OwnerReference(source_user_id=user_id, email=users_by_id.get(user_id))
|
|
|
|
|
|
def _load_agent_config(directory: Path) -> dict[str, Any]:
|
|
try:
|
|
payload = yaml.safe_load((directory / "config.yaml").read_text(encoding="utf-8")) or {}
|
|
except (OSError, UnicodeError, yaml.YAMLError) as exc:
|
|
raise ValueError(f"Cannot read agent config: {directory}") from exc
|
|
if not isinstance(payload, dict):
|
|
raise ValueError(f"Agent config must be an object: {directory}")
|
|
return payload
|
|
|
|
|
|
def _manifest_tags(rows: list[dict[str, Any]]) -> list[TagManifestItem]:
|
|
return [
|
|
TagManifestItem(
|
|
id=str(row.get("id") or ""),
|
|
name=str(row.get("name") or ""),
|
|
color=str(row.get("color") or "gray"),
|
|
description=str(row.get("description") or ""),
|
|
)
|
|
for row in rows
|
|
if row.get("id") and row.get("name")
|
|
]
|
|
|
|
|
|
async def build_agent_migration_manifest(
|
|
agent_ids: list[str],
|
|
*,
|
|
agent_store: Any,
|
|
tag_store: Any,
|
|
agents_root: Path,
|
|
users_by_id: dict[str, str],
|
|
) -> MigrationManifest:
|
|
"""Build an agent manifest without inspecting or exporting skill payloads."""
|
|
|
|
selected_ids = list(dict.fromkeys(agent_id.strip() for agent_id in agent_ids if agent_id.strip()))
|
|
if not selected_ids:
|
|
raise ValueError("Select at least one agent")
|
|
rows_by_id = {str(row["id"]): row for row in await agent_store.list_all()}
|
|
missing_agents = [agent_id for agent_id in selected_ids if agent_id not in rows_by_id]
|
|
if missing_agents:
|
|
raise ValueError(f"Unknown agents: {', '.join(missing_agents)}")
|
|
|
|
agent_tag_map = await tag_store.list_assignments("agent", selected_ids)
|
|
agents: list[AgentManifestItem] = []
|
|
warnings: list[str] = []
|
|
for agent_id in selected_ids:
|
|
row = rows_by_id[agent_id]
|
|
directory = _resolved_child(agents_root, agent_id.lower())
|
|
if directory is None or not directory.is_dir():
|
|
raise ValueError(f"Agent directory is missing: {agent_id}")
|
|
config = _load_agent_config(directory)
|
|
if str(config.get("id") or "") != agent_id:
|
|
raise ValueError(f"Agent config id does not match directory: {agent_id}")
|
|
raw_skills = config.get("skills")
|
|
selected_skills = list(dict.fromkeys(item.strip() for item in raw_skills if isinstance(item, str) and item.strip())) if isinstance(raw_skills, list) else []
|
|
knowledge_ids = config.get("llmwiki_knowledge_base_ids")
|
|
if isinstance(knowledge_ids, list) and any(str(item).strip() for item in knowledge_ids):
|
|
warnings.append(f"智能体 {agent_id} 绑定了知识库;知识库数据不在本迁移清单范围内,请在目标机器单独确认")
|
|
agents.append(
|
|
AgentManifestItem(
|
|
id=agent_id,
|
|
relative_dir=directory.relative_to(agents_root.resolve()).as_posix(),
|
|
tree_hash=await asyncio.to_thread(compute_tree_hash, directory),
|
|
name=str(row.get("name") or config.get("name") or agent_id),
|
|
description=str(row.get("description") or ""),
|
|
owner=_owner_reference(row.get("user_id"), users_by_id),
|
|
published=bool(row.get("published")),
|
|
is_private=bool(row.get("is_private")),
|
|
square_id=str(row.get("square_id") or ""),
|
|
featured_order=row.get("featured_order"),
|
|
created_at=row.get("created_at"),
|
|
updated_at=row.get("updated_at"),
|
|
skills=selected_skills,
|
|
tags=_manifest_tags(agent_tag_map.get(agent_id, [])),
|
|
)
|
|
)
|
|
|
|
return MigrationManifest(
|
|
bundle_id=uuid4().hex,
|
|
agents=agents,
|
|
warnings=warnings,
|
|
)
|
|
|
|
|
|
async def build_skill_migration_manifest(
|
|
skill_names: list[str],
|
|
*,
|
|
skill_store: Any,
|
|
tag_store: Any,
|
|
skills_root: Path,
|
|
discovered_skills: list[Any],
|
|
users_by_id: dict[str, str],
|
|
usage_entries: dict[str, dict[str, Any]],
|
|
) -> MigrationManifest:
|
|
"""Build a metadata-only manifest for explicitly selected live skills."""
|
|
|
|
selected_names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
|
|
if not selected_names:
|
|
raise ValueError("Select at least one skill")
|
|
|
|
skill_rows = {str(row["name"]): row for row in await skill_store.list_all()}
|
|
discovered = {str(skill.name): skill for skill in discovered_skills}
|
|
missing_skills = [name for name in selected_names if name not in discovered]
|
|
if missing_skills:
|
|
raise ValueError(f"Skills are missing from the live runtime directory: {', '.join(missing_skills)}")
|
|
|
|
skill_tag_map = await tag_store.list_assignments("skill", selected_names)
|
|
skill_directories = [Path(discovered[name].skill_dir).resolve() for name in selected_names]
|
|
tree_hashes = await _compute_tree_hashes(skill_directories)
|
|
skills: list[SkillManifestItem] = []
|
|
for name, tree_hash in zip(selected_names, tree_hashes, strict=True):
|
|
skill = discovered[name]
|
|
category = str(skill.category)
|
|
relative = PurePosixPath(category) / PurePosixPath(Path(skill.relative_path).as_posix())
|
|
directory = Path(skill.skill_dir).resolve()
|
|
row = skill_rows.get(name, {})
|
|
usage = usage_entries.get(name, {})
|
|
state = str(usage.get("state") or "active")
|
|
if state not in {"active", "stale", "archived"}:
|
|
state = "active"
|
|
source = str(usage.get("source") or "upload")
|
|
if source not in {"agent", "upload"}:
|
|
source = "upload"
|
|
skills.append(
|
|
SkillManifestItem(
|
|
name=name,
|
|
category=category,
|
|
relative_dir=relative.as_posix(),
|
|
tree_hash=tree_hash,
|
|
owner=_owner_reference(row.get("owner_user_id"), users_by_id),
|
|
published=bool(row.get("published", category == "public")),
|
|
detail=row.get("detail"),
|
|
name_zh=row.get("name_zh"),
|
|
always_on=bool(row.get("always_on", getattr(skill, "always_on", False))),
|
|
square_id=str(row.get("square_id") or ""),
|
|
enabled=bool(skill.enabled),
|
|
state=state,
|
|
pinned=bool(usage.get("pinned")),
|
|
featured=bool(usage.get("featured")),
|
|
source=source,
|
|
created_at=row.get("created_at") or usage.get("created_at"),
|
|
updated_at=row.get("updated_at"),
|
|
tags=_manifest_tags(skill_tag_map.get(name, [])),
|
|
)
|
|
)
|
|
|
|
selected_set = set(selected_names)
|
|
warnings: list[str] = []
|
|
support_directories: list[SupportDirectoryManifestItem] = []
|
|
support_candidates, support_warnings = await asyncio.to_thread(
|
|
_find_support_directories,
|
|
skills_root,
|
|
selected_set,
|
|
)
|
|
warnings.extend(support_warnings)
|
|
support_hashes = await _compute_tree_hashes(
|
|
[directory for directory, _required_by in support_candidates]
|
|
)
|
|
for (directory, selected_required_by), tree_hash in zip(
|
|
support_candidates,
|
|
support_hashes,
|
|
strict=True,
|
|
):
|
|
support_directories.append(
|
|
SupportDirectoryManifestItem(
|
|
relative_dir=directory.relative_to(skills_root.resolve()).as_posix(),
|
|
tree_hash=tree_hash,
|
|
required_by=selected_required_by,
|
|
)
|
|
)
|
|
|
|
return MigrationManifest(
|
|
bundle_id=uuid4().hex,
|
|
skills=skills,
|
|
support_directories=support_directories,
|
|
warnings=warnings,
|
|
)
|
|
|
|
|
|
async def _compute_tree_hashes(directories: list[Path]) -> list[str]:
|
|
if not directories:
|
|
return []
|
|
semaphore = asyncio.Semaphore(min(_TREE_HASH_MAX_WORKERS, len(directories)))
|
|
|
|
async def _compute(directory: Path) -> str:
|
|
async with semaphore:
|
|
return await asyncio.to_thread(compute_tree_hash, directory)
|
|
|
|
return list(await asyncio.gather(*(_compute(directory) for directory in directories)))
|
|
|
|
|
|
def _find_support_directories(
|
|
skills_root: Path,
|
|
selected_names: set[str],
|
|
) -> tuple[list[tuple[Path, list[str]]], list[str]]:
|
|
candidates: list[tuple[Path, list[str]]] = []
|
|
warnings: list[str] = []
|
|
for support_manifest in sorted(skills_root.rglob(".deerflow-skill-support.json")):
|
|
try:
|
|
payload = json.loads(support_manifest.read_text(encoding="utf-8"))
|
|
required_by = [str(item).strip() for item in payload.get("required_by", []) if str(item).strip()]
|
|
except (OSError, UnicodeError, json.JSONDecodeError, AttributeError):
|
|
warnings.append(f"无法读取技能支持目录清单:{support_manifest}")
|
|
continue
|
|
selected_required_by = [name for name in required_by if name in selected_names]
|
|
if not selected_required_by:
|
|
continue
|
|
candidates.append((support_manifest.parent.resolve(), selected_required_by))
|
|
return candidates, warnings
|
|
|
|
|
|
def _apply_usage_metadata(name: str, metadata: dict[str, Any]) -> None:
|
|
_apply_usage_metadata_batch({name: metadata})
|
|
|
|
|
|
def _apply_usage_metadata_batch(entries: dict[str, dict[str, Any]]) -> None:
|
|
from deerflow.skills.usage import set_migration_metadata_batch
|
|
|
|
set_migration_metadata_batch(entries)
|
|
|
|
|
|
def _target_owner_id(
|
|
owner: OwnerReference | None,
|
|
*,
|
|
users_by_email: dict[str, str],
|
|
fallback_owner_user_id: str | None,
|
|
) -> tuple[str | None, bool]:
|
|
if owner is None or (owner.source_user_id is None and owner.email is None):
|
|
return None, False
|
|
target = users_by_email.get(owner.email.casefold()) if owner.email else None
|
|
if target:
|
|
return target, False
|
|
if fallback_owner_user_id:
|
|
return fallback_owner_user_id, True
|
|
raise ValueError(f"Unresolved owner: {owner.email or owner.source_user_id}")
|
|
|
|
|
|
async def _apply_tags(tag_store: Any, target_type: str, target_id: str, tags: list[TagManifestItem]) -> int:
|
|
return await _apply_tags_bulk(tag_store, target_type, [(target_id, tags)])
|
|
|
|
|
|
async def _apply_tags_bulk(
|
|
tag_store: Any,
|
|
target_type: str,
|
|
targets: list[tuple[str, list[TagManifestItem]]],
|
|
) -> int:
|
|
if not targets:
|
|
return 0
|
|
existing = await tag_store.list_tags()
|
|
by_name = {
|
|
(str(row.get("tag_type")), str(row.get("name"))): row
|
|
for row in existing
|
|
}
|
|
used_ids = {str(row.get("id")) for row in existing}
|
|
assignments: dict[str, list[str]] = {}
|
|
created = 0
|
|
for target_id, tags in targets:
|
|
tag_ids: list[str] = []
|
|
for tag in tags:
|
|
same_name = by_name.get((target_type, tag.name))
|
|
if same_name is not None:
|
|
tag_ids.append(str(same_name["id"]))
|
|
continue
|
|
tag_id = tag.id if tag.id not in used_ids else uuid4().hex
|
|
created_tag = await tag_store.create_tag(
|
|
{
|
|
"id": tag_id,
|
|
"name": tag.name,
|
|
"tag_type": target_type,
|
|
"color": tag.color,
|
|
"description": tag.description,
|
|
}
|
|
)
|
|
by_name[(target_type, tag.name)] = created_tag
|
|
used_ids.add(tag_id)
|
|
tag_ids.append(tag_id)
|
|
created += 1
|
|
assignments[target_id] = tag_ids
|
|
await tag_store.set_target_tags_bulk(target_type, assignments)
|
|
return created
|
|
|
|
|
|
async def apply_migration_manifest(
|
|
manifest: MigrationManifest,
|
|
*,
|
|
preflight: MigrationPreflight,
|
|
agent_store: Any,
|
|
skill_store: Any,
|
|
tag_store: Any,
|
|
agents_root: Path,
|
|
users_by_email: dict[str, str],
|
|
fallback_owner_user_id: str | None,
|
|
extensions_config_path: Path | None,
|
|
skip_invalid_skills: bool = False,
|
|
) -> MigrationApplyResult:
|
|
"""Idempotently reconcile metadata after all live folders passed preflight."""
|
|
|
|
manifest, preflight, skipped_skill_names = prepare_manifest_for_apply(
|
|
manifest,
|
|
preflight,
|
|
skip_invalid_skills=skip_invalid_skills,
|
|
)
|
|
|
|
bad_resources = [item for item in [*preflight.agents, *preflight.skills, *preflight.support_directories] if item.status != "matched"]
|
|
if bad_resources:
|
|
raise ValueError("Migration preflight contains missing or mismatched directories")
|
|
if any(item.status == "unresolved" for item in preflight.owner_mappings) and not fallback_owner_user_id:
|
|
raise ValueError("Migration preflight contains unresolved owners")
|
|
|
|
if manifest.skills:
|
|
if extensions_config_path is None:
|
|
raise ValueError("目标运行环境未找到 extensions_config.json")
|
|
from deerflow.config.extensions_config import patch_extensions_config
|
|
|
|
enabled_by_name = {item.name: item.enabled for item in manifest.skills}
|
|
|
|
def _patch(document: dict[str, Any]) -> None:
|
|
skills_document = document.setdefault("skills", {})
|
|
if not isinstance(skills_document, dict):
|
|
raise ValueError("extensions_config.json skills must be an object")
|
|
for name, enabled in enabled_by_name.items():
|
|
current = skills_document.get(name)
|
|
entry = dict(current) if isinstance(current, dict) else {}
|
|
entry["enabled"] = enabled
|
|
skills_document[name] = entry
|
|
|
|
patch_extensions_config(extensions_config_path, _patch)
|
|
|
|
normalized_users = {email.strip().casefold(): user_id for email, user_id in users_by_email.items()}
|
|
fallback_count = 0
|
|
tags_created = 0
|
|
skill_payloads: list[dict[str, Any]] = []
|
|
skill_usage_entries: dict[str, dict[str, Any]] = {}
|
|
for item in manifest.skills:
|
|
owner_user_id, used_fallback = _target_owner_id(
|
|
item.owner,
|
|
users_by_email=normalized_users,
|
|
fallback_owner_user_id=fallback_owner_user_id,
|
|
)
|
|
fallback_count += int(used_fallback)
|
|
skill_payloads.append(
|
|
{
|
|
"name": item.name,
|
|
"owner_user_id": owner_user_id,
|
|
"published": item.published,
|
|
"detail": item.detail,
|
|
"name_zh": item.name_zh,
|
|
"always_on": item.always_on,
|
|
"square_id": item.square_id,
|
|
"created_at": item.created_at,
|
|
"updated_at": item.updated_at,
|
|
}
|
|
)
|
|
skill_usage_entries[item.name] = {
|
|
"state": item.state,
|
|
"pinned": item.pinned,
|
|
"featured": item.featured,
|
|
"source": item.source,
|
|
}
|
|
|
|
if skill_payloads:
|
|
await skill_store.migration_upsert_many(skill_payloads)
|
|
tags_created += await _apply_tags_bulk(
|
|
tag_store,
|
|
"skill",
|
|
[(item.name, item.tags) for item in manifest.skills],
|
|
)
|
|
_apply_usage_metadata_batch(skill_usage_entries)
|
|
|
|
agent_tag_targets: list[tuple[str, list[TagManifestItem]]] = []
|
|
for item in manifest.agents:
|
|
owner_user_id, used_fallback = _target_owner_id(
|
|
item.owner,
|
|
users_by_email=normalized_users,
|
|
fallback_owner_user_id=fallback_owner_user_id,
|
|
)
|
|
fallback_count += int(used_fallback)
|
|
await agent_store.migration_upsert(
|
|
{
|
|
"id": item.id,
|
|
"name": item.name,
|
|
"description": item.description,
|
|
"user_id": owner_user_id,
|
|
"published": item.published,
|
|
"is_private": item.is_private,
|
|
"square_id": item.square_id,
|
|
"featured_order": item.featured_order,
|
|
"created_at": item.created_at,
|
|
"updated_at": item.updated_at,
|
|
}
|
|
)
|
|
config = _load_agent_config(agents_root / item.relative_dir)
|
|
await agent_store.replace_skills(item.id, config.get("skills") if isinstance(config.get("skills"), list) else [])
|
|
await agent_store.set_extras(
|
|
item.id,
|
|
tool_groups=config.get("tool_groups") if isinstance(config.get("tool_groups"), list) else None,
|
|
model=str(config.get("model")) if config.get("model") else None,
|
|
)
|
|
agent_tag_targets.append((item.id, item.tags))
|
|
|
|
tags_created += await _apply_tags_bulk(tag_store, "agent", agent_tag_targets)
|
|
|
|
return MigrationApplyResult(
|
|
bundle_id=manifest.bundle_id,
|
|
agents_applied=len(manifest.agents),
|
|
skills_applied=len(manifest.skills),
|
|
skills_skipped=len(skipped_skill_names),
|
|
skipped_skill_names=skipped_skill_names,
|
|
tags_created=tags_created,
|
|
fallback_owner_count=fallback_count,
|
|
)
|