sync update from LAN repo (source dhk-1 @ 3e87be55)
This commit is contained in:
1 parent
ab6068a7c1
commit
42f2a26fa8
1482 files changed
+227759
-9584
No files matched your search
@@ -0,0 +1,29 @@
|
||||
"""Web App 产物模式纯引擎:站点发现 / 构建 / 预览签名。
|
||||
|
||||
不 import app.*(harness 边界),路由层只做 HTTP 适配。
|
||||
所有入口在能力不可用时返回结构化的"不可用"结果,绝不抛出影响主流程的异常。
|
||||
"""
|
||||
|
||||
from deerflow.webapp.builder import SiteBuildRunner, get_site_build_runner
|
||||
from deerflow.webapp.engine import (
|
||||
WebAppCapability,
|
||||
capability_status,
|
||||
find_sites,
|
||||
guess_site_mime,
|
||||
read_site_manifest,
|
||||
resolve_template_dir,
|
||||
)
|
||||
from deerflow.webapp.security import mint_preview_token, verify_preview_token
|
||||
|
||||
__all__ = [
|
||||
"SiteBuildRunner",
|
||||
"get_site_build_runner",
|
||||
"WebAppCapability",
|
||||
"capability_status",
|
||||
"find_sites",
|
||||
"guess_site_mime",
|
||||
"read_site_manifest",
|
||||
"resolve_template_dir",
|
||||
"mint_preview_token",
|
||||
"verify_preview_token",
|
||||
]
|
||||
@@ -0,0 +1,136 @@
|
||||
"""站点构建执行器:子进程隔离 + 每站点并发锁 + 任务注册表。
|
||||
|
||||
构建失败/超时只影响本次任务状态,绝不影响 gateway 主链路。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.webapp.engine import resolve_template_dir
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_OUTPUT_CHARS = 64_000
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildJob:
|
||||
job_id: str
|
||||
thread_id: str
|
||||
site_name: str
|
||||
status: str # running | success | failed
|
||||
started_at: float = field(default_factory=time.time)
|
||||
finished_at: float | None = None
|
||||
result: dict[str, Any] = field(default_factory=dict)
|
||||
error: str = ""
|
||||
|
||||
|
||||
class SiteBuildRunner:
|
||||
"""每个 (thread_id, site_name) 同时只允许一个构建任务。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._jobs: dict[tuple[str, str], BuildJob] = {}
|
||||
self._locks: dict[tuple[str, str], asyncio.Lock] = {}
|
||||
|
||||
def get_job(self, thread_id: str, site_name: str) -> BuildJob | None:
|
||||
return self._jobs.get((thread_id, site_name))
|
||||
|
||||
async def start_build(self, thread_id: str, site_name: str, site_dir: Path) -> BuildJob:
|
||||
"""启动构建;同站点已有运行中任务时直接返回该任务(幂等)。"""
|
||||
key = (thread_id, site_name)
|
||||
lock = self._locks.setdefault(key, asyncio.Lock())
|
||||
if lock.locked():
|
||||
existing = self._jobs.get(key)
|
||||
if existing is not None and existing.status == "running":
|
||||
return existing
|
||||
await lock.acquire()
|
||||
job = BuildJob(
|
||||
job_id=uuid.uuid4().hex[:12],
|
||||
thread_id=thread_id,
|
||||
site_name=site_name,
|
||||
status="running",
|
||||
)
|
||||
self._jobs[key] = job
|
||||
asyncio.create_task(self._run_build(job, site_dir, lock))
|
||||
return job
|
||||
|
||||
async def _run_build(self, job: BuildJob, site_dir: Path, lock: asyncio.Lock) -> None:
|
||||
try:
|
||||
result = await asyncio.to_thread(self._run_build_sync, job, site_dir)
|
||||
job.result = result
|
||||
job.status = "success" if result.get("success") else "failed"
|
||||
if not result.get("success"):
|
||||
job.error = str(result.get("error") or result.get("stderr_tail") or "构建失败")
|
||||
except Exception as exc: # noqa: BLE001 - 构建异常全部收进任务状态
|
||||
logger.warning("web-app build job %s crashed: %s", job.job_id, exc)
|
||||
job.status = "failed"
|
||||
job.error = f"构建执行异常: {exc}"
|
||||
job.result = {"success": False, "error": job.error}
|
||||
finally:
|
||||
job.finished_at = time.time()
|
||||
lock.release()
|
||||
|
||||
def _run_build_sync(self, _job: BuildJob, site_dir: Path) -> dict[str, Any]:
|
||||
config = get_app_config()
|
||||
template_dir = resolve_template_dir()
|
||||
script = template_dir / "scripts" / "build_site.py"
|
||||
if not script.is_file():
|
||||
return {"success": False, "error": f"构建脚本不存在: {script}"}
|
||||
env = dict(os.environ)
|
||||
env["DEERFLOW_WEBAPP_TEMPLATE"] = str(template_dir)
|
||||
env["DEERFLOW_WEBAPP_TIMEOUT"] = str(config.web_app.build_timeout_seconds)
|
||||
node_override = config.web_app.node_path.strip()
|
||||
if node_override:
|
||||
env["DEERFLOW_WEBAPP_NODE"] = node_override
|
||||
timeout = config.web_app.build_timeout_seconds + 30 # 脚本自身超时 + 通信余量
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(script), "build", str(site_dir)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=timeout,
|
||||
env=env,
|
||||
cwd=str(site_dir.parent),
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"success": False, "error": f"构建超时(>{timeout}s)已终止"}
|
||||
payload: dict[str, Any] = {"success": False, "error": f"构建脚本退出码 {proc.returncode}"}
|
||||
# stdout 是一行 JSON(取最后一个非空行,容忍脚本前置告警输出)
|
||||
for line in reversed((proc.stdout or "").strip().splitlines()):
|
||||
line = line.strip()
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
parsed = json.loads(line)
|
||||
if isinstance(parsed, dict):
|
||||
payload = parsed
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
payload.setdefault("success", False)
|
||||
payload["stdout_tail"] = (proc.stdout or "")[-MAX_OUTPUT_CHARS:]
|
||||
payload["stderr_tail"] = (proc.stderr or "")[-MAX_OUTPUT_CHARS:]
|
||||
return payload
|
||||
|
||||
|
||||
_runner: SiteBuildRunner | None = None
|
||||
|
||||
|
||||
def get_site_build_runner() -> SiteBuildRunner:
|
||||
global _runner
|
||||
if _runner is None:
|
||||
_runner = SiteBuildRunner()
|
||||
return _runner
|
||||
@@ -0,0 +1,163 @@
|
||||
"""站点发现、能力探测与状态读取(纯函数,无副作用)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import mimetypes
|
||||
import re
|
||||
import shutil
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.config.runtime_paths import resolve_path
|
||||
|
||||
SITE_MANIFEST_NAME = "site.json"
|
||||
BUILD_RESULT_NAME = ".build-result.json"
|
||||
|
||||
# 站点目录名约束(与 build_site.py 的 SITE_NAME_RE 一致)
|
||||
SITE_NAME_RE_OK = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
||||
|
||||
# dist/ 静态托管的 MIME 修正表
|
||||
_EXTRA_MIME_TYPES = {
|
||||
".js": "text/javascript; charset=utf-8",
|
||||
".mjs": "text/javascript; charset=utf-8",
|
||||
".css": "text/css; charset=utf-8",
|
||||
".json": "application/json; charset=utf-8",
|
||||
".geojson": "application/json; charset=utf-8",
|
||||
".wasm": "application/wasm",
|
||||
".html": "text/html; charset=utf-8",
|
||||
".htm": "text/html; charset=utf-8",
|
||||
".svg": "image/svg+xml",
|
||||
".woff": "font/woff",
|
||||
".woff2": "font/woff2",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class WebAppCapability:
|
||||
"""能力探测结果:任何一项缺失都只是让功能优雅不可用。"""
|
||||
|
||||
available: bool = False
|
||||
enabled: bool = False
|
||||
checks: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
template_dir: str = ""
|
||||
node_bin: str = ""
|
||||
|
||||
|
||||
def resolve_template_dir() -> Path:
|
||||
"""解析站点模板目录(相对 project root 或绝对路径)。"""
|
||||
config = get_app_config()
|
||||
return resolve_path(config.web_app.template_dir)
|
||||
|
||||
|
||||
def capability_status() -> WebAppCapability:
|
||||
"""检查配置开关、模板、预置依赖与 node 二进制。永不抛异常。"""
|
||||
checks: dict[str, dict[str, Any]] = {}
|
||||
try:
|
||||
config = get_app_config()
|
||||
enabled = bool(config.web_app.enabled)
|
||||
checks["enabled"] = {"ok": enabled, "detail": "config.yaml web_app.enabled"}
|
||||
|
||||
template_dir = resolve_template_dir()
|
||||
template_ok = template_dir.is_dir() and (template_dir / SITE_MANIFEST_NAME).is_file()
|
||||
checks["template"] = {"ok": template_ok, "detail": str(template_dir)}
|
||||
|
||||
vite_bin = template_dir / "node_modules" / "vite" / "bin" / "vite.js"
|
||||
deps_ok = vite_bin.is_file()
|
||||
checks["node_modules"] = {"ok": deps_ok, "detail": str(vite_bin)}
|
||||
|
||||
node_path = config.web_app.node_path.strip()
|
||||
node_bin = node_path if node_path and Path(node_path).exists() else (shutil.which("node") or "")
|
||||
checks["node"] = {"ok": bool(node_bin), "detail": node_bin}
|
||||
|
||||
available = all(item["ok"] for item in checks.values())
|
||||
return WebAppCapability(
|
||||
available=available,
|
||||
enabled=enabled,
|
||||
checks=checks,
|
||||
template_dir=str(template_dir),
|
||||
node_bin=node_bin,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - 能力探测永不抛出
|
||||
checks["internal"] = {"ok": False, "detail": str(exc)}
|
||||
return WebAppCapability(available=False, enabled=False, checks=checks)
|
||||
|
||||
|
||||
def read_site_manifest(site_dir: Path) -> dict[str, Any]:
|
||||
"""读取 site.json(损坏时返回空 dict,由调用方决定如何呈现)。"""
|
||||
manifest_path = site_dir / SITE_MANIFEST_NAME
|
||||
try:
|
||||
data = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else {}
|
||||
except (OSError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def _dist_info(site_dir: Path) -> dict[str, Any]:
|
||||
dist = site_dir / "dist"
|
||||
if not dist.is_dir():
|
||||
return {"exists": False}
|
||||
try:
|
||||
entries = [p for p in dist.rglob("*") if p.is_file()]
|
||||
latest = max((p.stat().st_mtime for p in entries), default=None)
|
||||
return {
|
||||
"exists": True,
|
||||
"file_count": len(entries),
|
||||
"total_bytes": sum(p.stat().st_size for p in entries),
|
||||
"mtime": latest,
|
||||
}
|
||||
except OSError:
|
||||
return {"exists": True, "file_count": 0, "total_bytes": 0, "mtime": None}
|
||||
|
||||
|
||||
def _last_build(site_dir: Path) -> dict[str, Any] | None:
|
||||
result_path = site_dir / BUILD_RESULT_NAME
|
||||
try:
|
||||
data = json.loads(result_path.read_text(encoding="utf-8"))
|
||||
return data if isinstance(data, dict) else None
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def find_sites(outputs_dir: Path) -> list[dict[str, Any]]:
|
||||
"""扫描 outputs/ 一级目录,凡带 site.json 的即站点。永不抛异常。"""
|
||||
sites: list[dict[str, Any]] = []
|
||||
try:
|
||||
candidates = sorted(outputs_dir.iterdir(), key=lambda p: (p.name.lower()))
|
||||
except OSError:
|
||||
return sites
|
||||
for entry in candidates:
|
||||
if not entry.is_dir() or not SITE_NAME_RE_OK.match(entry.name):
|
||||
continue
|
||||
if not (entry / SITE_MANIFEST_NAME).is_file():
|
||||
continue
|
||||
manifest = read_site_manifest(entry)
|
||||
try:
|
||||
mtime = (entry / SITE_MANIFEST_NAME).stat().st_mtime
|
||||
except OSError:
|
||||
mtime = None
|
||||
sites.append(
|
||||
{
|
||||
"name": entry.name,
|
||||
"title": str(manifest.get("title") or entry.name),
|
||||
"description": str(manifest.get("description") or ""),
|
||||
"nav": manifest.get("nav") if isinstance(manifest.get("nav"), list) else [],
|
||||
"map": manifest.get("map") if isinstance(manifest.get("map"), dict) else {},
|
||||
"manifest_mtime": mtime,
|
||||
"dist": _dist_info(entry),
|
||||
"last_build": _last_build(entry),
|
||||
}
|
||||
)
|
||||
sites.sort(key=lambda s: (-(s["manifest_mtime"] or 0), str(s["name"])))
|
||||
return sites
|
||||
|
||||
|
||||
def guess_site_mime(path: Path) -> str:
|
||||
"""dist/ 静态文件 MIME(带编码修正;未知扩展名回退 octet-stream)。"""
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in _EXTRA_MIME_TYPES:
|
||||
return _EXTRA_MIME_TYPES[suffix]
|
||||
guessed, _ = mimetypes.guess_type(path.name)
|
||||
return guessed or "application/octet-stream"
|
||||
@@ -0,0 +1,94 @@
|
||||
"""站点预览签名 URL:HMAC 短时效令牌,密钥首次使用时生成并持久化。
|
||||
|
||||
令牌只授予「该站点 dist/ 目录静态文件读」这一项能力,不携带任何其他 API 权限。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
|
||||
SECRET_FILE_NAME = ".web-app-secret"
|
||||
_DEFAULT_TTL_SECONDS = 7200
|
||||
|
||||
_secret_cache: bytes | None = None
|
||||
|
||||
|
||||
def _secret_path() -> Path:
|
||||
return runtime_home() / SECRET_FILE_NAME
|
||||
|
||||
|
||||
def get_preview_secret() -> bytes:
|
||||
"""加载或生成实例级 HMAC 密钥(0600 权限持久化在 .deer-flow 下)。"""
|
||||
global _secret_cache
|
||||
if _secret_cache is not None:
|
||||
return _secret_cache
|
||||
path = _secret_path()
|
||||
try:
|
||||
if path.is_file():
|
||||
data = path.read_bytes().strip()
|
||||
if len(data) >= 32:
|
||||
_secret_cache = data
|
||||
return _secret_cache
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
data = secrets.token_hex(32).encode("ascii")
|
||||
path.write_bytes(data)
|
||||
try:
|
||||
path.chmod(0o600)
|
||||
except OSError: # Windows 无 POSIX 权限,忽略
|
||||
pass
|
||||
_secret_cache = data
|
||||
return _secret_cache
|
||||
except OSError:
|
||||
# 持久化失败时退化为进程内随机密钥(重启后旧令牌失效,可接受)
|
||||
_secret_cache = secrets.token_hex(32).encode("ascii")
|
||||
return _secret_cache
|
||||
|
||||
|
||||
def mint_preview_token(thread_id: str, site_name: str, ttl_seconds: int = _DEFAULT_TTL_SECONDS) -> tuple[str, int]:
|
||||
"""签发预览令牌,返回 (token, expires_at_epoch)。"""
|
||||
expires_at = int(time.time()) + max(60, ttl_seconds)
|
||||
payload = f"{thread_id}|{site_name}|{expires_at}"
|
||||
encoded = base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=")
|
||||
signature = hmac.new(get_preview_secret(), payload.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
return f"{encoded}.{signature}", expires_at
|
||||
|
||||
|
||||
def verify_preview_token(token: str) -> tuple[str, str]:
|
||||
"""校验令牌,返回 (thread_id, site_name)。
|
||||
|
||||
Raises:
|
||||
ValueError: 令牌格式非法、签名不符或已过期。
|
||||
"""
|
||||
if not token or token.count(".") != 1:
|
||||
raise ValueError("malformed token")
|
||||
encoded, signature = token.split(".", 1)
|
||||
if not encoded or not signature:
|
||||
raise ValueError("malformed token")
|
||||
padding = "=" * (-len(encoded) % 4)
|
||||
try:
|
||||
payload = base64.urlsafe_b64decode(encoded + padding).decode("utf-8")
|
||||
except (ValueError, UnicodeDecodeError) as exc:
|
||||
raise ValueError("malformed token") from exc
|
||||
parts = payload.split("|")
|
||||
if len(parts) != 3:
|
||||
raise ValueError("malformed token")
|
||||
thread_id, site_name, exp_raw = parts
|
||||
expected = hmac.new(get_preview_secret(), payload.encode("utf-8"), hashlib.sha256).hexdigest()
|
||||
if not hmac.compare_digest(expected, signature):
|
||||
raise ValueError("bad signature")
|
||||
try:
|
||||
expires_at = int(exp_raw)
|
||||
except ValueError as exc:
|
||||
raise ValueError("malformed token") from exc
|
||||
if time.time() > expires_at:
|
||||
raise ValueError("token expired")
|
||||
if not thread_id or not site_name:
|
||||
raise ValueError("malformed token")
|
||||
return thread_id, site_name
|
||||
Reference in new issue
Block a user