sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
@@ -0,0 +1,136 @@
|
||||
"""站点构建执行器:子进程隔离 + 每站点并发锁 + 任务注册表。
|
||||
|
||||
构建失败/超时只影响本次任务状态,绝不影响 gateway 主链路。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config import get_app_config
|
||||
from deerflow.webapp.engine import resolve_template_dir
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_OUTPUT_CHARS = 64_000
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildJob:
|
||||
job_id: str
|
||||
thread_id: str
|
||||
site_name: str
|
||||
status: str # running | success | failed
|
||||
started_at: float = field(default_factory=time.time)
|
||||
finished_at: float | None = None
|
||||
result: dict[str, Any] = field(default_factory=dict)
|
||||
error: str = ""
|
||||
|
||||
|
||||
class SiteBuildRunner:
|
||||
"""每个 (thread_id, site_name) 同时只允许一个构建任务。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._jobs: dict[tuple[str, str], BuildJob] = {}
|
||||
self._locks: dict[tuple[str, str], asyncio.Lock] = {}
|
||||
|
||||
def get_job(self, thread_id: str, site_name: str) -> BuildJob | None:
|
||||
return self._jobs.get((thread_id, site_name))
|
||||
|
||||
async def start_build(self, thread_id: str, site_name: str, site_dir: Path) -> BuildJob:
|
||||
"""启动构建;同站点已有运行中任务时直接返回该任务(幂等)。"""
|
||||
key = (thread_id, site_name)
|
||||
lock = self._locks.setdefault(key, asyncio.Lock())
|
||||
if lock.locked():
|
||||
existing = self._jobs.get(key)
|
||||
if existing is not None and existing.status == "running":
|
||||
return existing
|
||||
await lock.acquire()
|
||||
job = BuildJob(
|
||||
job_id=uuid.uuid4().hex[:12],
|
||||
thread_id=thread_id,
|
||||
site_name=site_name,
|
||||
status="running",
|
||||
)
|
||||
self._jobs[key] = job
|
||||
asyncio.create_task(self._run_build(job, site_dir, lock))
|
||||
return job
|
||||
|
||||
async def _run_build(self, job: BuildJob, site_dir: Path, lock: asyncio.Lock) -> None:
|
||||
try:
|
||||
result = await asyncio.to_thread(self._run_build_sync, job, site_dir)
|
||||
job.result = result
|
||||
job.status = "success" if result.get("success") else "failed"
|
||||
if not result.get("success"):
|
||||
job.error = str(result.get("error") or result.get("stderr_tail") or "构建失败")
|
||||
except Exception as exc: # noqa: BLE001 - 构建异常全部收进任务状态
|
||||
logger.warning("web-app build job %s crashed: %s", job.job_id, exc)
|
||||
job.status = "failed"
|
||||
job.error = f"构建执行异常: {exc}"
|
||||
job.result = {"success": False, "error": job.error}
|
||||
finally:
|
||||
job.finished_at = time.time()
|
||||
lock.release()
|
||||
|
||||
def _run_build_sync(self, _job: BuildJob, site_dir: Path) -> dict[str, Any]:
|
||||
config = get_app_config()
|
||||
template_dir = resolve_template_dir()
|
||||
script = template_dir / "scripts" / "build_site.py"
|
||||
if not script.is_file():
|
||||
return {"success": False, "error": f"构建脚本不存在: {script}"}
|
||||
env = dict(os.environ)
|
||||
env["DEERFLOW_WEBAPP_TEMPLATE"] = str(template_dir)
|
||||
env["DEERFLOW_WEBAPP_TIMEOUT"] = str(config.web_app.build_timeout_seconds)
|
||||
node_override = config.web_app.node_path.strip()
|
||||
if node_override:
|
||||
env["DEERFLOW_WEBAPP_NODE"] = node_override
|
||||
timeout = config.web_app.build_timeout_seconds + 30 # 脚本自身超时 + 通信余量
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(script), "build", str(site_dir)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=timeout,
|
||||
env=env,
|
||||
cwd=str(site_dir.parent),
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"success": False, "error": f"构建超时(>{timeout}s)已终止"}
|
||||
payload: dict[str, Any] = {"success": False, "error": f"构建脚本退出码 {proc.returncode}"}
|
||||
# stdout 是一行 JSON(取最后一个非空行,容忍脚本前置告警输出)
|
||||
for line in reversed((proc.stdout or "").strip().splitlines()):
|
||||
line = line.strip()
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
parsed = json.loads(line)
|
||||
if isinstance(parsed, dict):
|
||||
payload = parsed
|
||||
break
|
||||
except ValueError:
|
||||
continue
|
||||
payload.setdefault("success", False)
|
||||
payload["stdout_tail"] = (proc.stdout or "")[-MAX_OUTPUT_CHARS:]
|
||||
payload["stderr_tail"] = (proc.stderr or "")[-MAX_OUTPUT_CHARS:]
|
||||
return payload
|
||||
|
||||
|
||||
_runner: SiteBuildRunner | None = None
|
||||
|
||||
|
||||
def get_site_build_runner() -> SiteBuildRunner:
|
||||
global _runner
|
||||
if _runner is None:
|
||||
_runner = SiteBuildRunner()
|
||||
return _runner
|
||||
Reference in new issue
Block a user