sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -0,0 +1,29 @@
"""Web App 产物模式纯引擎:站点发现 / 构建 / 预览签名。
不 import app.*(harness 边界),路由层只做 HTTP 适配。
所有入口在能力不可用时返回结构化的"不可用"结果,绝不抛出影响主流程的异常。
"""
from deerflow.webapp.builder import SiteBuildRunner, get_site_build_runner
from deerflow.webapp.engine import (
WebAppCapability,
capability_status,
find_sites,
guess_site_mime,
read_site_manifest,
resolve_template_dir,
)
from deerflow.webapp.security import mint_preview_token, verify_preview_token
__all__ = [
"SiteBuildRunner",
"get_site_build_runner",
"WebAppCapability",
"capability_status",
"find_sites",
"guess_site_mime",
"read_site_manifest",
"resolve_template_dir",
"mint_preview_token",
"verify_preview_token",
]
@@ -0,0 +1,136 @@
"""站点构建执行器:子进程隔离 + 每站点并发锁 + 任务注册表。
构建失败/超时只影响本次任务状态,绝不影响 gateway 主链路。
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import subprocess
import sys
import time
import uuid
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from deerflow.config import get_app_config
from deerflow.webapp.engine import resolve_template_dir
logger = logging.getLogger(__name__)
MAX_OUTPUT_CHARS = 64_000
@dataclass
class BuildJob:
job_id: str
thread_id: str
site_name: str
status: str # running | success | failed
started_at: float = field(default_factory=time.time)
finished_at: float | None = None
result: dict[str, Any] = field(default_factory=dict)
error: str = ""
class SiteBuildRunner:
"""每个 (thread_id, site_name) 同时只允许一个构建任务。"""
def __init__(self) -> None:
self._jobs: dict[tuple[str, str], BuildJob] = {}
self._locks: dict[tuple[str, str], asyncio.Lock] = {}
def get_job(self, thread_id: str, site_name: str) -> BuildJob | None:
return self._jobs.get((thread_id, site_name))
async def start_build(self, thread_id: str, site_name: str, site_dir: Path) -> BuildJob:
"""启动构建;同站点已有运行中任务时直接返回该任务(幂等)。"""
key = (thread_id, site_name)
lock = self._locks.setdefault(key, asyncio.Lock())
if lock.locked():
existing = self._jobs.get(key)
if existing is not None and existing.status == "running":
return existing
await lock.acquire()
job = BuildJob(
job_id=uuid.uuid4().hex[:12],
thread_id=thread_id,
site_name=site_name,
status="running",
)
self._jobs[key] = job
asyncio.create_task(self._run_build(job, site_dir, lock))
return job
async def _run_build(self, job: BuildJob, site_dir: Path, lock: asyncio.Lock) -> None:
try:
result = await asyncio.to_thread(self._run_build_sync, job, site_dir)
job.result = result
job.status = "success" if result.get("success") else "failed"
if not result.get("success"):
job.error = str(result.get("error") or result.get("stderr_tail") or "构建失败")
except Exception as exc: # noqa: BLE001 - 构建异常全部收进任务状态
logger.warning("web-app build job %s crashed: %s", job.job_id, exc)
job.status = "failed"
job.error = f"构建执行异常: {exc}"
job.result = {"success": False, "error": job.error}
finally:
job.finished_at = time.time()
lock.release()
def _run_build_sync(self, _job: BuildJob, site_dir: Path) -> dict[str, Any]:
config = get_app_config()
template_dir = resolve_template_dir()
script = template_dir / "scripts" / "build_site.py"
if not script.is_file():
return {"success": False, "error": f"构建脚本不存在: {script}"}
env = dict(os.environ)
env["DEERFLOW_WEBAPP_TEMPLATE"] = str(template_dir)
env["DEERFLOW_WEBAPP_TIMEOUT"] = str(config.web_app.build_timeout_seconds)
node_override = config.web_app.node_path.strip()
if node_override:
env["DEERFLOW_WEBAPP_NODE"] = node_override
timeout = config.web_app.build_timeout_seconds + 30 # 脚本自身超时 + 通信余量
try:
proc = subprocess.run(
[sys.executable, str(script), "build", str(site_dir)],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
timeout=timeout,
env=env,
cwd=str(site_dir.parent),
)
except subprocess.TimeoutExpired:
return {"success": False, "error": f"构建超时(>{timeout}s)已终止"}
payload: dict[str, Any] = {"success": False, "error": f"构建脚本退出码 {proc.returncode}"}
# stdout 是一行 JSON(取最后一个非空行,容忍脚本前置告警输出)
for line in reversed((proc.stdout or "").strip().splitlines()):
line = line.strip()
if line.startswith("{"):
try:
parsed = json.loads(line)
if isinstance(parsed, dict):
payload = parsed
break
except ValueError:
continue
payload.setdefault("success", False)
payload["stdout_tail"] = (proc.stdout or "")[-MAX_OUTPUT_CHARS:]
payload["stderr_tail"] = (proc.stderr or "")[-MAX_OUTPUT_CHARS:]
return payload
_runner: SiteBuildRunner | None = None
def get_site_build_runner() -> SiteBuildRunner:
global _runner
if _runner is None:
_runner = SiteBuildRunner()
return _runner
@@ -0,0 +1,163 @@
"""站点发现、能力探测与状态读取(纯函数,无副作用)。"""
from __future__ import annotations
import json
import mimetypes
import re
import shutil
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from deerflow.config import get_app_config
from deerflow.config.runtime_paths import resolve_path
SITE_MANIFEST_NAME = "site.json"
BUILD_RESULT_NAME = ".build-result.json"
# 站点目录名约束(与 build_site.py 的 SITE_NAME_RE 一致)
SITE_NAME_RE_OK = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
# dist/ 静态托管的 MIME 修正表
_EXTRA_MIME_TYPES = {
".js": "text/javascript; charset=utf-8",
".mjs": "text/javascript; charset=utf-8",
".css": "text/css; charset=utf-8",
".json": "application/json; charset=utf-8",
".geojson": "application/json; charset=utf-8",
".wasm": "application/wasm",
".html": "text/html; charset=utf-8",
".htm": "text/html; charset=utf-8",
".svg": "image/svg+xml",
".woff": "font/woff",
".woff2": "font/woff2",
}
@dataclass
class WebAppCapability:
"""能力探测结果:任何一项缺失都只是让功能优雅不可用。"""
available: bool = False
enabled: bool = False
checks: dict[str, dict[str, Any]] = field(default_factory=dict)
template_dir: str = ""
node_bin: str = ""
def resolve_template_dir() -> Path:
"""解析站点模板目录(相对 project root 或绝对路径)。"""
config = get_app_config()
return resolve_path(config.web_app.template_dir)
def capability_status() -> WebAppCapability:
"""检查配置开关、模板、预置依赖与 node 二进制。永不抛异常。"""
checks: dict[str, dict[str, Any]] = {}
try:
config = get_app_config()
enabled = bool(config.web_app.enabled)
checks["enabled"] = {"ok": enabled, "detail": "config.yaml web_app.enabled"}
template_dir = resolve_template_dir()
template_ok = template_dir.is_dir() and (template_dir / SITE_MANIFEST_NAME).is_file()
checks["template"] = {"ok": template_ok, "detail": str(template_dir)}
vite_bin = template_dir / "node_modules" / "vite" / "bin" / "vite.js"
deps_ok = vite_bin.is_file()
checks["node_modules"] = {"ok": deps_ok, "detail": str(vite_bin)}
node_path = config.web_app.node_path.strip()
node_bin = node_path if node_path and Path(node_path).exists() else (shutil.which("node") or "")
checks["node"] = {"ok": bool(node_bin), "detail": node_bin}
available = all(item["ok"] for item in checks.values())
return WebAppCapability(
available=available,
enabled=enabled,
checks=checks,
template_dir=str(template_dir),
node_bin=node_bin,
)
except Exception as exc: # noqa: BLE001 - 能力探测永不抛出
checks["internal"] = {"ok": False, "detail": str(exc)}
return WebAppCapability(available=False, enabled=False, checks=checks)
def read_site_manifest(site_dir: Path) -> dict[str, Any]:
"""读取 site.json(损坏时返回空 dict,由调用方决定如何呈现)。"""
manifest_path = site_dir / SITE_MANIFEST_NAME
try:
data = json.loads(manifest_path.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else {}
except (OSError, ValueError):
return {}
def _dist_info(site_dir: Path) -> dict[str, Any]:
dist = site_dir / "dist"
if not dist.is_dir():
return {"exists": False}
try:
entries = [p for p in dist.rglob("*") if p.is_file()]
latest = max((p.stat().st_mtime for p in entries), default=None)
return {
"exists": True,
"file_count": len(entries),
"total_bytes": sum(p.stat().st_size for p in entries),
"mtime": latest,
}
except OSError:
return {"exists": True, "file_count": 0, "total_bytes": 0, "mtime": None}
def _last_build(site_dir: Path) -> dict[str, Any] | None:
result_path = site_dir / BUILD_RESULT_NAME
try:
data = json.loads(result_path.read_text(encoding="utf-8"))
return data if isinstance(data, dict) else None
except (OSError, ValueError):
return None
def find_sites(outputs_dir: Path) -> list[dict[str, Any]]:
"""扫描 outputs/ 一级目录,凡带 site.json 的即站点。永不抛异常。"""
sites: list[dict[str, Any]] = []
try:
candidates = sorted(outputs_dir.iterdir(), key=lambda p: (p.name.lower()))
except OSError:
return sites
for entry in candidates:
if not entry.is_dir() or not SITE_NAME_RE_OK.match(entry.name):
continue
if not (entry / SITE_MANIFEST_NAME).is_file():
continue
manifest = read_site_manifest(entry)
try:
mtime = (entry / SITE_MANIFEST_NAME).stat().st_mtime
except OSError:
mtime = None
sites.append(
{
"name": entry.name,
"title": str(manifest.get("title") or entry.name),
"description": str(manifest.get("description") or ""),
"nav": manifest.get("nav") if isinstance(manifest.get("nav"), list) else [],
"map": manifest.get("map") if isinstance(manifest.get("map"), dict) else {},
"manifest_mtime": mtime,
"dist": _dist_info(entry),
"last_build": _last_build(entry),
}
)
sites.sort(key=lambda s: (-(s["manifest_mtime"] or 0), str(s["name"])))
return sites
def guess_site_mime(path: Path) -> str:
"""dist/ 静态文件 MIME(带编码修正;未知扩展名回退 octet-stream)。"""
suffix = path.suffix.lower()
if suffix in _EXTRA_MIME_TYPES:
return _EXTRA_MIME_TYPES[suffix]
guessed, _ = mimetypes.guess_type(path.name)
return guessed or "application/octet-stream"
@@ -0,0 +1,94 @@
"""站点预览签名 URL:HMAC 短时效令牌,密钥首次使用时生成并持久化。
令牌只授予「该站点 dist/ 目录静态文件读」这一项能力,不携带任何其他 API 权限。
"""
from __future__ import annotations
import base64
import hashlib
import hmac
import secrets
import time
from pathlib import Path
from deerflow.config.runtime_paths import runtime_home
SECRET_FILE_NAME = ".web-app-secret"
_DEFAULT_TTL_SECONDS = 7200
_secret_cache: bytes | None = None
def _secret_path() -> Path:
return runtime_home() / SECRET_FILE_NAME
def get_preview_secret() -> bytes:
"""加载或生成实例级 HMAC 密钥(0600 权限持久化在 .deer-flow 下)。"""
global _secret_cache
if _secret_cache is not None:
return _secret_cache
path = _secret_path()
try:
if path.is_file():
data = path.read_bytes().strip()
if len(data) >= 32:
_secret_cache = data
return _secret_cache
path.parent.mkdir(parents=True, exist_ok=True)
data = secrets.token_hex(32).encode("ascii")
path.write_bytes(data)
try:
path.chmod(0o600)
except OSError: # Windows 无 POSIX 权限,忽略
pass
_secret_cache = data
return _secret_cache
except OSError:
# 持久化失败时退化为进程内随机密钥(重启后旧令牌失效,可接受)
_secret_cache = secrets.token_hex(32).encode("ascii")
return _secret_cache
def mint_preview_token(thread_id: str, site_name: str, ttl_seconds: int = _DEFAULT_TTL_SECONDS) -> tuple[str, int]:
"""签发预览令牌,返回 (token, expires_at_epoch)。"""
expires_at = int(time.time()) + max(60, ttl_seconds)
payload = f"{thread_id}|{site_name}|{expires_at}"
encoded = base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=")
signature = hmac.new(get_preview_secret(), payload.encode("utf-8"), hashlib.sha256).hexdigest()
return f"{encoded}.{signature}", expires_at
def verify_preview_token(token: str) -> tuple[str, str]:
"""校验令牌,返回 (thread_id, site_name)。
Raises:
ValueError: 令牌格式非法、签名不符或已过期。
"""
if not token or token.count(".") != 1:
raise ValueError("malformed token")
encoded, signature = token.split(".", 1)
if not encoded or not signature:
raise ValueError("malformed token")
padding = "=" * (-len(encoded) % 4)
try:
payload = base64.urlsafe_b64decode(encoded + padding).decode("utf-8")
except (ValueError, UnicodeDecodeError) as exc:
raise ValueError("malformed token") from exc
parts = payload.split("|")
if len(parts) != 3:
raise ValueError("malformed token")
thread_id, site_name, exp_raw = parts
expected = hmac.new(get_preview_secret(), payload.encode("utf-8"), hashlib.sha256).hexdigest()
if not hmac.compare_digest(expected, signature):
raise ValueError("bad signature")
try:
expires_at = int(exp_raw)
except ValueError as exc:
raise ValueError("malformed token") from exc
if time.time() > expires_at:
raise ValueError("token expired")
if not thread_id or not site_name:
raise ValueError("malformed token")
return thread_id, site_name