sync snapshot from LAN repo (source master-zg @ 40658c74)

This commit is contained in:
gozeros committed 2026-11-05 10:58:49 +08:00
1 parent ea9dcff9c0
commit 754b61572c
179 files changed
+26085 -8788

No files matched your search

@@ -0,0 +1,924 @@
"""Agent/skill metadata migration manifests for already-copied live folders.
The manifest deliberately contains no agent or skill payload files. Operators
copy those directories with their normal offline transfer process; these
helpers only prove that the target runtime folders match the exported inventory
before database/configuration metadata is reconciled.
"""
from __future__ import annotations
import asyncio
import hashlib
import json
import re
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath
from typing import Any, Literal
from uuid import uuid4
import yaml
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
ResourceStatus = Literal["matched", "missing", "hash_mismatch", "identity_mismatch", "invalid_path"]
OwnerStatus = Literal["matched", "unresolved"]
_TREE_HASH_RE = re.compile(r"^[0-9a-f]{64}$")
_TREE_HASH_MAX_WORKERS = 4
def _safe_relative_path(value: str) -> str:
normalized = value.strip()
if not normalized or "\\" in normalized or ":" in normalized:
raise ValueError("relative_dir must be a non-empty portable relative path")
path = PurePosixPath(normalized)
if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts):
raise ValueError("relative_dir must stay inside the configured runtime root")
return path.as_posix()
def compute_tree_hash(root: Path) -> str:
"""Return a deterministic SHA-256 for a directory tree.
Relative paths and bytes are both signed. Symlinks are rejected so a
manifest cannot attest content outside the selected runtime directory.
"""
root = root.resolve()
if not root.is_dir():
raise FileNotFoundError(root)
digest = hashlib.sha256()
files = sorted((path for path in root.rglob("*") if path.is_file() or path.is_symlink()), key=lambda p: p.relative_to(root).as_posix())
for path in files:
if path.is_symlink():
raise ValueError(f"Symlinks are not supported in migration trees: {path}")
relative = path.relative_to(root).as_posix().encode("utf-8")
digest.update(len(relative).to_bytes(4, "big"))
digest.update(relative)
digest.update(path.stat().st_size.to_bytes(8, "big"))
with path.open("rb") as stream:
while chunk := stream.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
class OwnerReference(BaseModel):
model_config = ConfigDict(extra="forbid")
source_user_id: str | None = None
email: str | None = None
@field_validator("source_user_id", "email")
@classmethod
def _strip_optional(cls, value: str | None) -> str | None:
return value.strip() if value and value.strip() else None
class TagManifestItem(BaseModel):
model_config = ConfigDict(extra="forbid")
id: str = Field(min_length=1, max_length=64)
name: str = Field(min_length=1, max_length=128)
color: str = Field(default="gray", max_length=32)
description: str = Field(default="", max_length=1024)
class AgentManifestItem(BaseModel):
model_config = ConfigDict(extra="forbid")
id: str
relative_dir: str
tree_hash: str
name: str
description: str = ""
owner: OwnerReference | None = None
published: bool = False
is_private: bool = False
square_id: str = ""
featured_order: int | None = None
created_at: datetime | None = None
updated_at: datetime | None = None
skills: list[str] = Field(default_factory=list, max_length=5000)
tags: list[TagManifestItem] = Field(default_factory=list, max_length=200)
@field_validator("relative_dir")
@classmethod
def _relative_dir(cls, value: str) -> str:
return _safe_relative_path(value)
@field_validator("tree_hash")
@classmethod
def _tree_hash(cls, value: str) -> str:
normalized = value.lower()
if not _TREE_HASH_RE.fullmatch(normalized):
raise ValueError("tree_hash must be a SHA-256 hex digest")
return normalized
@model_validator(mode="after")
def _directory_matches_id(self):
if self.relative_dir != self.id.lower():
raise ValueError("agent relative_dir must equal the stable agent id")
return self
class SkillManifestItem(BaseModel):
model_config = ConfigDict(extra="forbid")
name: str
category: Literal["public", "custom"]
relative_dir: str
tree_hash: str
owner: OwnerReference | None = None
published: bool = False
detail: str | None = None
name_zh: str | None = None
always_on: bool = False
square_id: str = ""
enabled: bool = True
state: Literal["active", "stale", "archived"] = "active"
pinned: bool = False
featured: bool = False
source: Literal["agent", "upload"] = "upload"
created_at: datetime | None = None
updated_at: datetime | None = None
tags: list[TagManifestItem] = Field(default_factory=list, max_length=200)
@field_validator("relative_dir")
@classmethod
def _relative_dir(cls, value: str) -> str:
normalized = _safe_relative_path(value)
if PurePosixPath(normalized).parts[0] not in {"public", "custom"}:
raise ValueError("skill relative_dir must begin with public/ or custom/")
return normalized
@field_validator("tree_hash")
@classmethod
def _tree_hash(cls, value: str) -> str:
normalized = value.lower()
if not _TREE_HASH_RE.fullmatch(normalized):
raise ValueError("tree_hash must be a SHA-256 hex digest")
return normalized
@model_validator(mode="after")
def _category_matches_directory(self):
if PurePosixPath(self.relative_dir).parts[0] != self.category:
raise ValueError("skill category must match relative_dir")
return self
class SupportDirectoryManifestItem(BaseModel):
model_config = ConfigDict(extra="forbid")
relative_dir: str
tree_hash: str
required_by: list[str] = Field(default_factory=list, max_length=5000)
@field_validator("relative_dir")
@classmethod
def _relative_dir(cls, value: str) -> str:
normalized = _safe_relative_path(value)
if PurePosixPath(normalized).parts[0] not in {"public", "custom"}:
raise ValueError("support relative_dir must begin with public/ or custom/")
return normalized
@field_validator("tree_hash")
@classmethod
def _tree_hash(cls, value: str) -> str:
normalized = value.lower()
if not _TREE_HASH_RE.fullmatch(normalized):
raise ValueError("tree_hash must be a SHA-256 hex digest")
return normalized
class MigrationManifest(BaseModel):
model_config = ConfigDict(extra="forbid")
schema_version: Literal[1] = 1
bundle_id: str = Field(min_length=1, max_length=128)
created_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
agents: list[AgentManifestItem] = Field(default_factory=list, max_length=1000)
skills: list[SkillManifestItem] = Field(default_factory=list, max_length=5000)
support_directories: list[SupportDirectoryManifestItem] = Field(default_factory=list, max_length=1000)
warnings: list[str] = Field(default_factory=list, max_length=1000)
class ResourcePreflightItem(BaseModel):
kind: Literal["agent", "skill", "support"]
key: str
relative_dir: str
status: ResourceStatus
message: str = ""
class OwnerMappingItem(BaseModel):
source_user_id: str | None = None
email: str | None = None
target_user_id: str | None = None
status: OwnerStatus
class MigrationPreflight(BaseModel):
ready: bool
agents: list[ResourcePreflightItem]
skills: list[ResourcePreflightItem]
support_directories: list[ResourcePreflightItem]
owner_mappings: list[OwnerMappingItem]
class MigrationApplyResult(BaseModel):
bundle_id: str
agents_applied: int
skills_applied: int
skills_skipped: int = 0
skipped_skill_names: list[str] = Field(default_factory=list)
tags_created: int
fallback_owner_count: int
def _resolved_child(root: Path, relative_dir: str) -> Path | None:
root = root.resolve()
child = root
for part in PurePosixPath(relative_dir).parts:
child /= part
if child.is_symlink():
return None
child = child.resolve()
try:
child.relative_to(root)
except ValueError:
return None
return child
def _agent_identity(directory: Path) -> str | None:
config_path = directory / "config.yaml"
if not config_path.is_file():
return None
try:
payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
except (OSError, UnicodeError, yaml.YAMLError):
return None
return str(payload.get("id") or "").strip() or None if isinstance(payload, dict) else None
def _skill_identity(directory: Path) -> str | None:
skill_path = directory / "SKILL.md"
if not skill_path.is_file():
return None
try:
text = skill_path.read_text(encoding="utf-8")
if not text.startswith("---"):
return None
parts = text.split("---", 2)
payload = yaml.safe_load(parts[1]) or {}
except (OSError, UnicodeError, yaml.YAMLError, IndexError):
return None
return str(payload.get("name") or "").strip() or None if isinstance(payload, dict) else None
def _resource_preflight(
*,
kind: Literal["agent", "skill", "support"],
key: str,
root: Path,
relative_dir: str,
expected_hash: str,
expected_identity: str | None = None,
) -> ResourcePreflightItem:
directory = _resolved_child(root, relative_dir)
if directory is None:
return ResourcePreflightItem(
kind=kind,
key=key,
relative_dir=relative_dir,
status="invalid_path",
message="目录路径不安全、包含符号链接或超出运行根目录",
)
if not directory.is_dir():
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="missing", message="目标运行目录中未找到该目录")
if expected_identity is not None:
identity_file = directory / ("config.yaml" if kind == "agent" else "SKILL.md")
if not identity_file.is_file():
return ResourcePreflightItem(
kind=kind,
key=key,
relative_dir=relative_dir,
status="missing",
message=f"目录中缺少 {identity_file.name}",
)
try:
actual_hash = compute_tree_hash(directory)
except (OSError, ValueError) as exc:
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="hash_mismatch", message=str(exc))
if actual_hash != expected_hash:
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="hash_mismatch", message="目录内容与导出清单不一致")
if expected_identity is not None:
actual_identity = _agent_identity(directory) if kind == "agent" else _skill_identity(directory)
if actual_identity != expected_identity:
return ResourcePreflightItem(
kind=kind,
key=key,
relative_dir=relative_dir,
status="identity_mismatch",
message=f"目录内标识为 {actual_identity or '空'},清单标识为 {expected_identity}",
)
return ResourcePreflightItem(kind=kind, key=key, relative_dir=relative_dir, status="matched")
def preflight_manifest(
manifest: MigrationManifest,
*,
agents_root: Path,
skills_root: Path,
users_by_email: dict[str, str],
) -> MigrationPreflight:
"""Compare a manifest with target live roots without modifying anything."""
normalized_users = {email.strip().casefold(): user_id for email, user_id in users_by_email.items()}
resource_specs: list[dict[str, Any]] = [
{
"kind": "agent",
"key": item.id,
"root": agents_root,
"relative_dir": item.relative_dir,
"expected_hash": item.tree_hash,
"expected_identity": item.id,
}
for item in manifest.agents
]
resource_specs.extend(
{
"kind": "skill",
"key": f"{item.category}:{item.name}",
"root": skills_root,
"relative_dir": item.relative_dir,
"expected_hash": item.tree_hash,
"expected_identity": item.name,
}
for item in manifest.skills
)
resource_specs.extend(
{
"kind": "support",
"key": item.relative_dir,
"root": skills_root,
"relative_dir": item.relative_dir,
"expected_hash": item.tree_hash,
}
for item in manifest.support_directories
)
def _check_resource(spec: dict[str, Any]) -> ResourcePreflightItem:
return _resource_preflight(**spec)
if len(resource_specs) > 1:
with ThreadPoolExecutor(
max_workers=min(_TREE_HASH_MAX_WORKERS, len(resource_specs)),
thread_name_prefix="skill-migration-hash",
) as executor:
resources = list(executor.map(_check_resource, resource_specs))
else:
resources = [_check_resource(spec) for spec in resource_specs]
agent_count = len(manifest.agents)
skill_count = len(manifest.skills)
agents = resources[:agent_count]
skills = resources[agent_count : agent_count + skill_count]
support = resources[agent_count + skill_count :]
owners: dict[tuple[str | None, str | None], OwnerMappingItem] = {}
for resource in [*manifest.agents, *manifest.skills]:
owner = resource.owner
if owner is None or (owner.source_user_id is None and owner.email is None):
continue
email = owner.email.strip().casefold() if owner.email else None
target_user_id = normalized_users.get(email) if email else None
key = (owner.source_user_id, email)
owners[key] = OwnerMappingItem(
source_user_id=owner.source_user_id,
email=email,
target_user_id=target_user_id,
status="matched" if target_user_id else "unresolved",
)
owner_mappings = list(owners.values())
ready = all(item.status == "matched" for item in [*agents, *skills, *support]) and all(item.status == "matched" for item in owner_mappings)
return MigrationPreflight(
ready=ready,
agents=agents,
skills=skills,
support_directories=support,
owner_mappings=owner_mappings,
)
def prepare_manifest_for_apply(
manifest: MigrationManifest,
preflight: MigrationPreflight,
*,
skip_invalid_skills: bool,
) -> tuple[MigrationManifest, MigrationPreflight, list[str]]:
"""Optionally remove invalid skills and support-directory dependants.
Skipping is intentionally limited to skill-only manifests. Agent manifests
retain all-or-nothing semantics because their live ``config.yaml`` is the
authoritative source for several related database caches.
"""
if not skip_invalid_skills:
return manifest, preflight, []
if manifest.agents:
raise ValueError("跳过异常条目仅支持独立技能清单")
failed_skill_dirs = {
item.relative_dir for item in preflight.skills if item.status != "matched"
}
skipped_names = {
item.name for item in manifest.skills if item.relative_dir in failed_skill_dirs
}
support_by_dir = {
item.relative_dir: item for item in manifest.support_directories
}
for resource in preflight.support_directories:
if resource.status == "matched":
continue
support = support_by_dir.get(resource.relative_dir)
if support is not None:
skipped_names.update(support.required_by)
ordered_skipped = [item.name for item in manifest.skills if item.name in skipped_names]
kept_skills = [item for item in manifest.skills if item.name not in skipped_names]
kept_names = {item.name for item in kept_skills}
kept_support = [
item
for item in manifest.support_directories
if any(name in kept_names for name in item.required_by)
]
kept_skill_dirs = {item.relative_dir for item in kept_skills}
kept_support_dirs = {item.relative_dir for item in kept_support}
owner_keys = {
(
item.owner.source_user_id,
item.owner.email.strip().casefold() if item.owner and item.owner.email else None,
)
for item in kept_skills
if item.owner is not None
and (item.owner.source_user_id is not None or item.owner.email is not None)
}
owner_mappings = [
item
for item in preflight.owner_mappings
if (
item.source_user_id,
item.email.strip().casefold() if item.email else None,
)
in owner_keys
]
skill_resources = [
item for item in preflight.skills if item.relative_dir in kept_skill_dirs
]
support_resources = [
item
for item in preflight.support_directories
if item.relative_dir in kept_support_dirs
]
ready = all(
item.status == "matched"
for item in [*preflight.agents, *skill_resources, *support_resources]
) and all(item.status == "matched" for item in owner_mappings)
effective_manifest = manifest.model_copy(
update={
"skills": kept_skills,
"support_directories": kept_support,
}
)
effective_preflight = preflight.model_copy(
update={
"ready": ready,
"skills": skill_resources,
"support_directories": support_resources,
"owner_mappings": owner_mappings,
}
)
return effective_manifest, effective_preflight, ordered_skipped
def _owner_reference(user_id: str | None, users_by_id: dict[str, str]) -> OwnerReference | None:
if user_id is None:
return None
return OwnerReference(source_user_id=user_id, email=users_by_id.get(user_id))
def _load_agent_config(directory: Path) -> dict[str, Any]:
try:
payload = yaml.safe_load((directory / "config.yaml").read_text(encoding="utf-8")) or {}
except (OSError, UnicodeError, yaml.YAMLError) as exc:
raise ValueError(f"Cannot read agent config: {directory}") from exc
if not isinstance(payload, dict):
raise ValueError(f"Agent config must be an object: {directory}")
return payload
def _manifest_tags(rows: list[dict[str, Any]]) -> list[TagManifestItem]:
return [
TagManifestItem(
id=str(row.get("id") or ""),
name=str(row.get("name") or ""),
color=str(row.get("color") or "gray"),
description=str(row.get("description") or ""),
)
for row in rows
if row.get("id") and row.get("name")
]
async def build_agent_migration_manifest(
agent_ids: list[str],
*,
agent_store: Any,
tag_store: Any,
agents_root: Path,
users_by_id: dict[str, str],
) -> MigrationManifest:
"""Build an agent manifest without inspecting or exporting skill payloads."""
selected_ids = list(dict.fromkeys(agent_id.strip() for agent_id in agent_ids if agent_id.strip()))
if not selected_ids:
raise ValueError("Select at least one agent")
rows_by_id = {str(row["id"]): row for row in await agent_store.list_all()}
missing_agents = [agent_id for agent_id in selected_ids if agent_id not in rows_by_id]
if missing_agents:
raise ValueError(f"Unknown agents: {', '.join(missing_agents)}")
agent_tag_map = await tag_store.list_assignments("agent", selected_ids)
agents: list[AgentManifestItem] = []
warnings: list[str] = []
for agent_id in selected_ids:
row = rows_by_id[agent_id]
directory = _resolved_child(agents_root, agent_id.lower())
if directory is None or not directory.is_dir():
raise ValueError(f"Agent directory is missing: {agent_id}")
config = _load_agent_config(directory)
if str(config.get("id") or "") != agent_id:
raise ValueError(f"Agent config id does not match directory: {agent_id}")
raw_skills = config.get("skills")
selected_skills = list(dict.fromkeys(item.strip() for item in raw_skills if isinstance(item, str) and item.strip())) if isinstance(raw_skills, list) else []
knowledge_ids = config.get("llmwiki_knowledge_base_ids")
if isinstance(knowledge_ids, list) and any(str(item).strip() for item in knowledge_ids):
warnings.append(f"智能体 {agent_id} 绑定了知识库;知识库数据不在本迁移清单范围内,请在目标机器单独确认")
agents.append(
AgentManifestItem(
id=agent_id,
relative_dir=directory.relative_to(agents_root.resolve()).as_posix(),
tree_hash=await asyncio.to_thread(compute_tree_hash, directory),
name=str(row.get("name") or config.get("name") or agent_id),
description=str(row.get("description") or ""),
owner=_owner_reference(row.get("user_id"), users_by_id),
published=bool(row.get("published")),
is_private=bool(row.get("is_private")),
square_id=str(row.get("square_id") or ""),
featured_order=row.get("featured_order"),
created_at=row.get("created_at"),
updated_at=row.get("updated_at"),
skills=selected_skills,
tags=_manifest_tags(agent_tag_map.get(agent_id, [])),
)
)
return MigrationManifest(
bundle_id=uuid4().hex,
agents=agents,
warnings=warnings,
)
async def build_skill_migration_manifest(
skill_names: list[str],
*,
skill_store: Any,
tag_store: Any,
skills_root: Path,
discovered_skills: list[Any],
users_by_id: dict[str, str],
usage_entries: dict[str, dict[str, Any]],
) -> MigrationManifest:
"""Build a metadata-only manifest for explicitly selected live skills."""
selected_names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
if not selected_names:
raise ValueError("Select at least one skill")
skill_rows = {str(row["name"]): row for row in await skill_store.list_all()}
discovered = {str(skill.name): skill for skill in discovered_skills}
missing_skills = [name for name in selected_names if name not in discovered]
if missing_skills:
raise ValueError(f"Skills are missing from the live runtime directory: {', '.join(missing_skills)}")
skill_tag_map = await tag_store.list_assignments("skill", selected_names)
skill_directories = [Path(discovered[name].skill_dir).resolve() for name in selected_names]
tree_hashes = await _compute_tree_hashes(skill_directories)
skills: list[SkillManifestItem] = []
for name, tree_hash in zip(selected_names, tree_hashes, strict=True):
skill = discovered[name]
category = str(skill.category)
relative = PurePosixPath(category) / PurePosixPath(Path(skill.relative_path).as_posix())
directory = Path(skill.skill_dir).resolve()
row = skill_rows.get(name, {})
usage = usage_entries.get(name, {})
state = str(usage.get("state") or "active")
if state not in {"active", "stale", "archived"}:
state = "active"
source = str(usage.get("source") or "upload")
if source not in {"agent", "upload"}:
source = "upload"
skills.append(
SkillManifestItem(
name=name,
category=category,
relative_dir=relative.as_posix(),
tree_hash=tree_hash,
owner=_owner_reference(row.get("owner_user_id"), users_by_id),
published=bool(row.get("published", category == "public")),
detail=row.get("detail"),
name_zh=row.get("name_zh"),
always_on=bool(row.get("always_on", getattr(skill, "always_on", False))),
square_id=str(row.get("square_id") or ""),
enabled=bool(skill.enabled),
state=state,
pinned=bool(usage.get("pinned")),
featured=bool(usage.get("featured")),
source=source,
created_at=row.get("created_at") or usage.get("created_at"),
updated_at=row.get("updated_at"),
tags=_manifest_tags(skill_tag_map.get(name, [])),
)
)
selected_set = set(selected_names)
warnings: list[str] = []
support_directories: list[SupportDirectoryManifestItem] = []
support_candidates, support_warnings = await asyncio.to_thread(
_find_support_directories,
skills_root,
selected_set,
)
warnings.extend(support_warnings)
support_hashes = await _compute_tree_hashes(
[directory for directory, _required_by in support_candidates]
)
for (directory, selected_required_by), tree_hash in zip(
support_candidates,
support_hashes,
strict=True,
):
support_directories.append(
SupportDirectoryManifestItem(
relative_dir=directory.relative_to(skills_root.resolve()).as_posix(),
tree_hash=tree_hash,
required_by=selected_required_by,
)
)
return MigrationManifest(
bundle_id=uuid4().hex,
skills=skills,
support_directories=support_directories,
warnings=warnings,
)
async def _compute_tree_hashes(directories: list[Path]) -> list[str]:
if not directories:
return []
semaphore = asyncio.Semaphore(min(_TREE_HASH_MAX_WORKERS, len(directories)))
async def _compute(directory: Path) -> str:
async with semaphore:
return await asyncio.to_thread(compute_tree_hash, directory)
return list(await asyncio.gather(*(_compute(directory) for directory in directories)))
def _find_support_directories(
skills_root: Path,
selected_names: set[str],
) -> tuple[list[tuple[Path, list[str]]], list[str]]:
candidates: list[tuple[Path, list[str]]] = []
warnings: list[str] = []
for support_manifest in sorted(skills_root.rglob(".deerflow-skill-support.json")):
try:
payload = json.loads(support_manifest.read_text(encoding="utf-8"))
required_by = [str(item).strip() for item in payload.get("required_by", []) if str(item).strip()]
except (OSError, UnicodeError, json.JSONDecodeError, AttributeError):
warnings.append(f"无法读取技能支持目录清单:{support_manifest}")
continue
selected_required_by = [name for name in required_by if name in selected_names]
if not selected_required_by:
continue
candidates.append((support_manifest.parent.resolve(), selected_required_by))
return candidates, warnings
def _apply_usage_metadata(name: str, metadata: dict[str, Any]) -> None:
_apply_usage_metadata_batch({name: metadata})
def _apply_usage_metadata_batch(entries: dict[str, dict[str, Any]]) -> None:
from deerflow.skills.usage import set_migration_metadata_batch
set_migration_metadata_batch(entries)
def _target_owner_id(
owner: OwnerReference | None,
*,
users_by_email: dict[str, str],
fallback_owner_user_id: str | None,
) -> tuple[str | None, bool]:
if owner is None or (owner.source_user_id is None and owner.email is None):
return None, False
target = users_by_email.get(owner.email.casefold()) if owner.email else None
if target:
return target, False
if fallback_owner_user_id:
return fallback_owner_user_id, True
raise ValueError(f"Unresolved owner: {owner.email or owner.source_user_id}")
async def _apply_tags(tag_store: Any, target_type: str, target_id: str, tags: list[TagManifestItem]) -> int:
return await _apply_tags_bulk(tag_store, target_type, [(target_id, tags)])
async def _apply_tags_bulk(
tag_store: Any,
target_type: str,
targets: list[tuple[str, list[TagManifestItem]]],
) -> int:
if not targets:
return 0
existing = await tag_store.list_tags()
by_name = {
(str(row.get("tag_type")), str(row.get("name"))): row
for row in existing
}
used_ids = {str(row.get("id")) for row in existing}
assignments: dict[str, list[str]] = {}
created = 0
for target_id, tags in targets:
tag_ids: list[str] = []
for tag in tags:
same_name = by_name.get((target_type, tag.name))
if same_name is not None:
tag_ids.append(str(same_name["id"]))
continue
tag_id = tag.id if tag.id not in used_ids else uuid4().hex
created_tag = await tag_store.create_tag(
{
"id": tag_id,
"name": tag.name,
"tag_type": target_type,
"color": tag.color,
"description": tag.description,
}
)
by_name[(target_type, tag.name)] = created_tag
used_ids.add(tag_id)
tag_ids.append(tag_id)
created += 1
assignments[target_id] = tag_ids
await tag_store.set_target_tags_bulk(target_type, assignments)
return created
async def apply_migration_manifest(
manifest: MigrationManifest,
*,
preflight: MigrationPreflight,
agent_store: Any,
skill_store: Any,
tag_store: Any,
agents_root: Path,
users_by_email: dict[str, str],
fallback_owner_user_id: str | None,
extensions_config_path: Path | None,
skip_invalid_skills: bool = False,
) -> MigrationApplyResult:
"""Idempotently reconcile metadata after all live folders passed preflight."""
manifest, preflight, skipped_skill_names = prepare_manifest_for_apply(
manifest,
preflight,
skip_invalid_skills=skip_invalid_skills,
)
bad_resources = [item for item in [*preflight.agents, *preflight.skills, *preflight.support_directories] if item.status != "matched"]
if bad_resources:
raise ValueError("Migration preflight contains missing or mismatched directories")
if any(item.status == "unresolved" for item in preflight.owner_mappings) and not fallback_owner_user_id:
raise ValueError("Migration preflight contains unresolved owners")
if manifest.skills:
if extensions_config_path is None:
raise ValueError("目标运行环境未找到 extensions_config.json")
from deerflow.config.extensions_config import patch_extensions_config
enabled_by_name = {item.name: item.enabled for item in manifest.skills}
def _patch(document: dict[str, Any]) -> None:
skills_document = document.setdefault("skills", {})
if not isinstance(skills_document, dict):
raise ValueError("extensions_config.json skills must be an object")
for name, enabled in enabled_by_name.items():
current = skills_document.get(name)
entry = dict(current) if isinstance(current, dict) else {}
entry["enabled"] = enabled
skills_document[name] = entry
patch_extensions_config(extensions_config_path, _patch)
normalized_users = {email.strip().casefold(): user_id for email, user_id in users_by_email.items()}
fallback_count = 0
tags_created = 0
skill_payloads: list[dict[str, Any]] = []
skill_usage_entries: dict[str, dict[str, Any]] = {}
for item in manifest.skills:
owner_user_id, used_fallback = _target_owner_id(
item.owner,
users_by_email=normalized_users,
fallback_owner_user_id=fallback_owner_user_id,
)
fallback_count += int(used_fallback)
skill_payloads.append(
{
"name": item.name,
"owner_user_id": owner_user_id,
"published": item.published,
"detail": item.detail,
"name_zh": item.name_zh,
"always_on": item.always_on,
"square_id": item.square_id,
"created_at": item.created_at,
"updated_at": item.updated_at,
}
)
skill_usage_entries[item.name] = {
"state": item.state,
"pinned": item.pinned,
"featured": item.featured,
"source": item.source,
}
if skill_payloads:
await skill_store.migration_upsert_many(skill_payloads)
tags_created += await _apply_tags_bulk(
tag_store,
"skill",
[(item.name, item.tags) for item in manifest.skills],
)
_apply_usage_metadata_batch(skill_usage_entries)
agent_tag_targets: list[tuple[str, list[TagManifestItem]]] = []
for item in manifest.agents:
owner_user_id, used_fallback = _target_owner_id(
item.owner,
users_by_email=normalized_users,
fallback_owner_user_id=fallback_owner_user_id,
)
fallback_count += int(used_fallback)
await agent_store.migration_upsert(
{
"id": item.id,
"name": item.name,
"description": item.description,
"user_id": owner_user_id,
"published": item.published,
"is_private": item.is_private,
"square_id": item.square_id,
"featured_order": item.featured_order,
"created_at": item.created_at,
"updated_at": item.updated_at,
}
)
config = _load_agent_config(agents_root / item.relative_dir)
await agent_store.replace_skills(item.id, config.get("skills") if isinstance(config.get("skills"), list) else [])
await agent_store.set_extras(
item.id,
tool_groups=config.get("tool_groups") if isinstance(config.get("tool_groups"), list) else None,
model=str(config.get("model")) if config.get("model") else None,
)
agent_tag_targets.append((item.id, item.tags))
tags_created += await _apply_tags_bulk(tag_store, "agent", agent_tag_targets)
return MigrationApplyResult(
bundle_id=manifest.bundle_id,
agents_applied=len(manifest.agents),
skills_applied=len(manifest.skills),
skills_skipped=len(skipped_skill_names),
skipped_skill_names=skipped_skill_names,
tags_created=tags_created,
fallback_owner_count=fallback_count,
)