diff --git a/.gitignore b/.gitignore index 35e457d6..5c8cac46 100644 --- a/.gitignore +++ b/.gitignore @@ -65,7 +65,10 @@ offline-backend-20260512/data/ offline-backend-20260512/deerflow-backend-offline-image-20260505.tar offline-backend-20260512/extensions_config.json offline-backend-20260512/skills/ -offline-backend-20260512/wheelhouse/ +# Canonical DeerFlow-only offline release assets. The base wheelhouse is +# versioned so a clean checkout can reproduce the fully offline bundle. +!offline-backend-20260512/wheelhouse/ +!offline-backend-20260512/wheelhouse/** offline-backend-20260512/backend/.deer-flow/data/*.db offline-backend-20260512/backend/.deer-flow/data/*.db-* offline-backend-20260512/backend/.deer-flow/* @@ -119,7 +122,19 @@ offline-backend-20260512/backend/tests/* !offline-backend-20260512/backend/tests/test_enterprise_research_tasks.py # Position-roundtable per-turn write/present cap (weak-model duplicate artifacts). !offline-backend-20260512/backend/tests/test_position_artifact_cap.py -deploy +# Keep only the canonical DeerFlow-only deployment implementation and the +# three shared build inputs it consumes. Other historical deploy variants +# remain local-only. +deploy/* +!deploy/linux-docker-offline-deerflow-only/ +!deploy/linux-docker-offline-deerflow-only/** +deploy/linux-docker-offline-deerflow-only/**/__pycache__/ +deploy/linux-docker-offline-deerflow-only/**/*.pyc +!deploy/linux-docker-offline/ +deploy/linux-docker-offline/* +!deploy/linux-docker-offline/Dockerfile.backend +!deploy/linux-docker-offline/Dockerfile.frontend +!deploy/linux-docker-offline/sanitize_release_sources.py .history # Cloned third-party repos (local only) diff --git a/AGENTS.md b/AGENTS.md index fda68ac5..b1accdee 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,20 @@ This is a customized offline deployment of **DeerFlow** — a LangGraph-based AI Managed together via shell scripts in `scripts/`. +## Canonical Offline Release + +All future Linux AMD64 offline releases must use +`deploy/linux-docker-offline-deerflow-only/build-release.sh`. This is the +repository's canonical DeerFlow-only deployment path; do not package the +historical DeerFlow + WeKnora deployment variant. The canonical bundle uses +`config/deploy.env` as its only hand-edited configuration source and includes +the `aiomysql` database adapter, persistent `source/wheelhouse` extension +support, multi-model configuration, and the Office document runtime. + +Follow `docs/DeerFlow纯离线标准发版与部署.md` for source synchronization, +building, verification, installation, source-only updates, rollback, and +artifact retention. + ## Commands ### Full Application (from repo root) diff --git a/deploy/linux-docker-offline-deerflow-only/Dockerfile.backend-code-refresh b/deploy/linux-docker-offline-deerflow-only/Dockerfile.backend-code-refresh new file mode 100644 index 00000000..2e3526f3 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/Dockerfile.backend-code-refresh @@ -0,0 +1,13 @@ +# Fast, deterministic code refresh for a previously built runtime whose +# dependency manifests are byte-identical to the current source snapshot. +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +USER root +RUN find /app/backend -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + +COPY . /app/backend +WORKDIR /app/backend + +# Keep the base image's PATH, PYTHONPATH, ENTRYPOINT and CMD. The caller must +# compare pyproject.toml, uv.lock, requirements-offline.txt and the harness +# pyproject before using this path. diff --git a/deploy/linux-docker-offline-deerflow-only/Dockerfile.backend-source-overlay b/deploy/linux-docker-offline-deerflow-only/Dockerfile.backend-source-overlay new file mode 100644 index 00000000..deee3a3b --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/Dockerfile.backend-source-overlay @@ -0,0 +1,10 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +USER root +RUN find /app/backend -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + +COPY . /app/backend +WORKDIR /app/backend + +# Python、Office、浏览器和系统依赖全部继承自固定基线镜像。 +# build-source-update.sh 与 APPLY_SOURCE_UPDATE.sh 会双重校验依赖清单。 diff --git a/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-assets b/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-assets new file mode 100644 index 00000000..d848294d --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-assets @@ -0,0 +1,16 @@ +# syntax=docker/dockerfile:1.7 +FROM node:22-bookworm-slim AS builder + +WORKDIR /workspace/frontend-web +RUN corepack enable && corepack prepare pnpm@10.26.2 --activate + +COPY frontend-web/package.json frontend-web/pnpm-lock.yaml ./ +RUN --mount=type=cache,target=/root/.local/share/pnpm/store \ + pnpm install --frozen-lockfile + +COPY frontend-web/ ./ +ENV VITE_ASSET_BASE=/ +RUN pnpm build + +FROM scratch AS artifact +COPY --from=builder /workspace/frontend-web/dist/ / diff --git a/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-dist-overlay b/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-dist-overlay new file mode 100644 index 00000000..ae5b88be --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-dist-overlay @@ -0,0 +1,9 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +USER root +RUN find /usr/share/nginx/html -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + +COPY frontend-dist/ /usr/share/nginx/html/ +COPY nginx/deerflow.conf /etc/nginx/conf.d/default.conf + +# Nginx 运行时继承自固定基线镜像;这里只替换编译后的前端静态文件。 diff --git a/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-runtime-config-refresh b/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-runtime-config-refresh new file mode 100644 index 00000000..8b1b8d19 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-runtime-config-refresh @@ -0,0 +1,6 @@ +# Refresh the unmounted fallback runtime-config.js in an already compiled +# frontend image. The compiled application assets must come from the exact same +# source snapshot; only this plain runtime file is replaced after sanitization. +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +COPY frontend-web/public/runtime-config.js /usr/share/nginx/html/runtime-config.js diff --git a/deploy/linux-docker-offline-deerflow-only/README_OFFLINE.md b/deploy/linux-docker-offline-deerflow-only/README_OFFLINE.md new file mode 100644 index 00000000..d5dc6096 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/README_OFFLINE.md @@ -0,0 +1,49 @@ +# DeerFlow 纯项目完全离线一键部署包 + +本包面向 Linux `x86_64/amd64`,只包含构建时工作区中的 DeerFlow 前端、后端镜像、命名智能体、技能和一键部署文件。版本来源、镜像标签和构建时间见 `VERSION.json`。 + +明确不包含:WeKnora UI/App、ParadeDB、Redis、DocReader、独立远程 Sandbox 镜像及 WeKnora 预置 Skills。DeerFlow 默认使用后端容器内的本地沙箱;该沙箱已经内置 Word、Excel、PowerPoint、OpenDocument 和 PDF 处理依赖。DeerFlow 中依赖 WeKnora 的 LLMWiki 连接和本地 Wiki 同步会被运行配置强制关闭。 + +“完全离线”是指解压、Docker 镜像加载和应用启动不访问镜像仓库、npm、pip 或 Git。实际问答仍需服务器能访问你在 `config/deploy.env` 中填写的外部 MySQL 和模型服务;模型权重不在本包内。 + +## 最短部署流程 + +```bash +sha256sum -c deerflow-linux-amd64-offline-*.tar.gz.sha256 +tar -xzf deerflow-linux-amd64-offline-*.tar.gz +cd deerflow-linux-amd64-offline-* +vi config/deploy.env +sudo ./OFFLINE_INSTALL.sh --start +``` + +唯一需要人工修改的运行配置文件是 `config/deploy.env`。必须填写访问地址、外部 MySQL 和至少一个 DeerFlow 模型服务;支持用连续编号配置 1~20 个模型。新增离线 Python 依赖时只需向 `source/wheelhouse` 放入 wheel,并在该目录的 `requirements.txt` 增加一行。 + +```bash +sudo ./OFFLINE_INSTALL.sh --start +sudo ./OFFLINE_INSTALL.sh --stop +sudo ./OFFLINE_INSTALL.sh --restart +sudo ./OFFLINE_INSTALL.sh --status +sudo ./OFFLINE_INSTALL.sh --logs +sudo ./OFFLINE_INSTALL.sh --logs deerflow-backend +sudo ./OFFLINE_INSTALL.sh --office-check +``` + +`--stop` 只停止容器,不删除镜像、容器、网络或数据卷。再次启动会检查两个镜像,已存在时跳过 `docker load`。 + +MySQL ORM 固定使用 `aiomysql` 兼容组。`--start` 与 `--restart` 都只从包内 wheelhouse 安装依赖,不访问 pip 仓库;依赖安装在持久卷中,容器重建后仍保留。 + +详细文档按顺序阅读: + +1. `docs/01-部署前准备.md` +2. `docs/02-deploy.env配置说明.md` +3. `docs/03-一键部署与启停.md` +4. `docs/04-技能与智能体位置.md` +5. `docs/05-数据备份与恢复.md` +6. `docs/06-故障排查与校验.md` +7. `docs/07-升级与版本回退.md` +8. `docs/08-离线验收清单.md` +9. `docs/09-后续增量更新发版手册.md` +10. `docs/10-Office文档处理与自检.md` +11. `docs/11-只替换前后端源码.md` +12. `docs/12-直接覆盖源码并重启.md` +13. `docs/13-wheelhouse离线新增依赖与多模型.md` diff --git a/deploy/linux-docker-offline-deerflow-only/build-release.sh b/deploy/linux-docker-offline-deerflow-only/build-release.sh new file mode 100644 index 00000000..744e5989 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/build-release.sh @@ -0,0 +1,278 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# 在可联网的 Linux amd64 构建机上执行: +# ./deploy/linux-docker-offline-deerflow-only/build-release.sh WORKSPACE BUILD_ROOT + +WORKSPACE="${1:?workspace path is required}" +BUILD_ROOT="${2:?build root is required}" +WORKSPACE="$(realpath "$WORKSPACE")" +BUILD_ROOT="$(realpath -m "$BUILD_ROOT")" + +[[ -f "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/release-template/OFFLINE_INSTALL.sh" ]] || { + echo "invalid DeerFlow workspace: $WORKSPACE" >&2 + exit 2 +} +[[ -f "$WORKSPACE/offline-backend-20260512/backend/pyproject.toml" ]] || exit 2 +[[ -f "$WORKSPACE/frontend-web/package.json" ]] || exit 2 + +case "$BUILD_ROOT" in + /|/opt|/opt/codex-build|"$WORKSPACE") + echo "unsafe build root: $BUILD_ROOT" >&2 + exit 2 + ;; +esac + +STAMP="${RELEASE_STAMP:-$(date +%Y%m%d-%H%M%S)}" +DEERFLOW_COMMIT="${DEERFLOW_COMMIT_OVERRIDE:-}" +if [[ -z "$DEERFLOW_COMMIT" ]] && git -C "$WORKSPACE" rev-parse HEAD >/dev/null 2>&1; then + DEERFLOW_COMMIT="$(git -C "$WORKSPACE" rev-parse HEAD)" +fi +[[ "$DEERFLOW_COMMIT" =~ ^[0-9a-fA-F]{40}$ ]] || { + echo "set DEERFLOW_COMMIT_OVERRIDE to the 40-character source commit" >&2 + exit 2 +} +DEERFLOW_SHORT="${DEERFLOW_COMMIT:0:8}" +SOURCE_DIRTY="${DEERFLOW_DIRTY_OVERRIDE:-false}" +[[ "$SOURCE_DIRTY" == "true" || "$SOURCE_DIRTY" == "false" ]] || exit 2 + +RELEASE_NAME="deerflow-linux-amd64-offline-${STAMP}" +STAGE="$BUILD_ROOT/stage/$RELEASE_NAME" +DIST="$BUILD_ROOT/dist" +CONTEXT_ROOT="$BUILD_ROOT/context/$RELEASE_NAME" +BACKEND_CONTEXT="$CONTEXT_ROOT/backend" +FRONTEND_CONTEXT="$CONTEXT_ROOT/frontend-root" + +for target in "$STAGE" "$CONTEXT_ROOT"; do + if [[ -e "$target" ]]; then + resolved="$(realpath "$target")" + case "$resolved" in + "$BUILD_ROOT/stage/$RELEASE_NAME"|"$BUILD_ROOT/context/$RELEASE_NAME") rm -rf -- "$resolved" ;; + *) echo "refusing to replace unexpected path: $resolved" >&2; exit 2 ;; + esac + fi +done +mkdir -p "$STAGE" "$DIST" "$BACKEND_CONTEXT" "$FRONTEND_CONTEXT/frontend-web" \ + "$FRONTEND_CONTEXT/deploy/linux-docker-offline/nginx" + +echo "[1/7] Assemble sanitized DeerFlow-only release" +cp -a "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/release-template/." "$STAGE/" +mkdir -p \ + "$STAGE/config/deerflow" \ + "$STAGE/config/frontend" \ + "$STAGE/agents-skills/deerflow-agents" \ + "$STAGE/agents-skills/deerflow-skills" \ + "$STAGE/agents-skills/deerflow-knowledge" \ + "$STAGE/images" \ + "$STAGE/source/wheelhouse" \ + "$STAGE/data/generated" \ + "$STAGE/data/state" + +cp -a "$WORKSPACE/offline-backend-20260512/backend/." "$BACKEND_CONTEXT/" +cp -a "$WORKSPACE/frontend-web/." "$FRONTEND_CONTEXT/frontend-web/" +cp -a "$WORKSPACE/deploy/linux-docker-offline/nginx/." \ + "$FRONTEND_CONTEXT/deploy/linux-docker-offline/nginx/" +rm -rf -- \ + "$BACKEND_CONTEXT/.venv" \ + "$BACKEND_CONTEXT/.deer-flow" \ + "$BACKEND_CONTEXT/.pytest_cache" \ + "$BACKEND_CONTEXT/.ruff_cache" \ + "$FRONTEND_CONTEXT/frontend-web/node_modules" \ + "$FRONTEND_CONTEXT/frontend-web/dist" \ + "$FRONTEND_CONTEXT/frontend-web/.vite" +find "$BACKEND_CONTEXT" "$FRONTEND_CONTEXT" -type d -name __pycache__ -prune -exec rm -rf -- {} + + +python3 "$WORKSPACE/deploy/linux-docker-offline/sanitize_release_sources.py" backend "$BACKEND_CONTEXT" +python3 "$WORKSPACE/deploy/linux-docker-offline/sanitize_release_sources.py" frontend "$FRONTEND_CONTEXT/frontend-web" + +# 运行时直接挂载的后端源码。用户后续只需覆盖这三个目录并重启。 +mkdir -p "$STAGE/source/backend" "$STAGE/source/frontend/dist" +cp -a "$BACKEND_CONTEXT/app" "$STAGE/source/backend/app" +cp -a "$BACKEND_CONTEXT/packages" "$STAGE/source/backend/packages" +cp -a "$BACKEND_CONTEXT/tests" "$STAGE/source/backend/tests" + +cp -a "$BACKEND_CONTEXT/config.yaml" "$STAGE/config/deerflow/config.base.yaml" +cp -a "$BACKEND_CONTEXT/extensions_config.json" "$STAGE/config/deerflow/extensions_config.json" +cp -a "$FRONTEND_CONTEXT/frontend-web/public/runtime-config.js" "$STAGE/config/frontend/runtime-config.base.js" +cp -a "$WORKSPACE/offline-backend-20260512/backend/skills/." "$STAGE/agents-skills/deerflow-skills/" +cp -a "$WORKSPACE/offline-backend-20260512/wheelhouse/." "$STAGE/source/wheelhouse/" +[[ -f "$STAGE/source/wheelhouse/aiomysql-0.3.2-py3-none-any.whl" ]] || { + echo "aiomysql wheel is missing from source/wheelhouse" >&2 + exit 2 +} + +# 预置项目自带的命名智能体;32 位十六进制目录是用户运行数据,不迁移。 +AGENT_ROOT="$WORKSPACE/offline-backend-20260512/backend/.deer-flow/agents" +if [[ -d "$AGENT_ROOT" ]]; then + while IFS= read -r -d '' agent_dir; do + agent_name="$(basename "$agent_dir")" + if [[ ! "$agent_name" =~ ^[0-9a-fA-F]{32}$ ]]; then + cp -a "$agent_dir" "$STAGE/agents-skills/deerflow-agents/" + fi + done < <(find "$AGENT_ROOT" -mindepth 1 -maxdepth 1 -type d -print0) +fi + +cp -a "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/docs/." "$STAGE/docs/" +cp -a "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/README_OFFLINE.md" "$STAGE/README_OFFLINE.md" +printf '%s\n' '把需要预置的本地知识文件放在此目录;本包不导入旧部署业务数据。' \ + > "$STAGE/agents-skills/deerflow-knowledge/README.txt" +find "$STAGE" -type d -name __pycache__ -prune -exec rm -rf -- {} + + +echo "[2/7] Build current DeerFlow linux/amd64 images" +BACKEND_IMAGE="cmzs/deerflow-backend-offline:${STAMP}-${DEERFLOW_SHORT}" +FRONTEND_IMAGE="cmzs/deerflow-frontend-offline:${STAMP}-${DEERFLOW_SHORT}" +if [[ -n "${DEERFLOW_BACKEND_RUNTIME_BASE:-}" ]]; then + [[ -n "${DEERFLOW_BASE_SOURCE_ROOT:-}" ]] || { + echo "DEERFLOW_BASE_SOURCE_ROOT is required with DEERFLOW_BACKEND_RUNTIME_BASE" >&2 + exit 2 + } + DEERFLOW_BASE_SOURCE_ROOT="$(realpath "$DEERFLOW_BASE_SOURCE_ROOT")" + for manifest in pyproject.toml uv.lock requirements-offline.txt packages/harness/pyproject.toml; do + cmp -s "$BACKEND_CONTEXT/$manifest" "$DEERFLOW_BASE_SOURCE_ROOT/$manifest" || { + echo "dependency manifest differs from runtime base: $manifest" >&2 + exit 2 + } + done + docker image inspect "$DEERFLOW_BACKEND_RUNTIME_BASE" >/dev/null + docker build --platform linux/amd64 --network host --provenance=false \ + --build-arg "BASE_IMAGE=$DEERFLOW_BACKEND_RUNTIME_BASE" \ + -f "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/Dockerfile.backend-code-refresh" \ + -t "$BACKEND_IMAGE" "$BACKEND_CONTEXT" +else + docker build --platform linux/amd64 --network host --provenance=false \ + -f "$WORKSPACE/deploy/linux-docker-offline/Dockerfile.backend" \ + -t "$BACKEND_IMAGE" "$BACKEND_CONTEXT" +fi + +if [[ -n "${DEERFLOW_FRONTEND_RUNTIME_BASE:-}" ]]; then + docker image inspect "$DEERFLOW_FRONTEND_RUNTIME_BASE" >/dev/null + docker build --platform linux/amd64 --network host --provenance=false \ + --build-arg "BASE_IMAGE=$DEERFLOW_FRONTEND_RUNTIME_BASE" \ + -f "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-runtime-config-refresh" \ + -t "$FRONTEND_IMAGE" "$FRONTEND_CONTEXT" +else + docker build --platform linux/amd64 --network host --provenance=false \ + -f "$WORKSPACE/deploy/linux-docker-offline/Dockerfile.frontend" \ + -t "$FRONTEND_IMAGE" "$FRONTEND_CONTEXT" +fi + +# 从刚构建的前端镜像取出与镜像完全一致的 dist,作为宿主机挂载初始内容。 +FRONTEND_EXPORT_CONTAINER="$(docker create "$FRONTEND_IMAGE")" +cleanup_frontend_export() { + docker rm -f "$FRONTEND_EXPORT_CONTAINER" >/dev/null 2>&1 || true +} +trap cleanup_frontend_export EXIT +docker cp "$FRONTEND_EXPORT_CONTAINER:/usr/share/nginx/html/." \ + "$STAGE/source/frontend/dist/" +docker rm "$FRONTEND_EXPORT_CONTAINER" >/dev/null +FRONTEND_EXPORT_CONTAINER="" +trap - EXIT + +echo "[3/7] Validate image contents and architecture" +docker image inspect "$BACKEND_IMAGE" "$FRONTEND_IMAGE" >/dev/null +[[ "$(docker image inspect -f '{{.Architecture}}' "$BACKEND_IMAGE")" == "amd64" ]] || exit 2 +[[ "$(docker image inspect -f '{{.Architecture}}' "$FRONTEND_IMAGE")" == "amd64" ]] || exit 2 +docker run --rm --network none "$BACKEND_IMAGE" python -c \ + 'import app.gateway.app, deerflow; print("backend import smoke passed")' +docker run --rm --network none "$BACKEND_IMAGE" \ + /opt/venv/bin/python /app/backend/scripts/office_runtime_selfcheck.py + +echo "[4/7] Write version metadata" +cat > "$STAGE/images/deerflow-image-list.txt" < "$STAGE/VERSION.json" < \ + "$STAGE/images/deerflow-images-linux-amd64.tar.gz" +gzip -t "$STAGE/images/deerflow-images-linux-amd64.tar.gz" + +echo "[6/7] Generate checksums and run static release checks" +( + cd "$STAGE" + sha256sum images/deerflow-images-linux-amd64.tar.gz VERSION.json SOURCE_BASELINE.json > SHA256SUMS +) +chmod 0755 "$STAGE/OFFLINE_INSTALL.sh" "$STAGE/UPDATE_INSTALL.sh" \ + "$STAGE/scripts/"*.py +find "$STAGE/data" -mindepth 1 -type f -delete +[[ "$(grep -cvE '^[[:space:]]*(#|$)' "$STAGE/images/deerflow-image-list.txt")" == "2" ]] || exit 2 +python3 "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/test_release_renderer.py" "$WORKSPACE" +# source/ 是供用户直接覆盖的 DeerFlow 项目源码,可能包含未启用的可选 +# WeKnora 适配代码;纯部署校验只禁止实际服务、镜像和部署文件。 +if find "$STAGE" -path "$STAGE/source" -prune -o \ + \( -iname '*weknora*' -o -iname '*paradedb*' -o -iname '*docreader*' \) \ + -print | grep -q .; then + echo "WeKnora-related deployment artifact found in pure DeerFlow stage" >&2 + exit 2 +fi +if grep -RIlE --exclude-dir=source \ + 'wechatopenai/weknora|paradedb/paradedb|redis:7\.0-alpine' "$STAGE" | grep -q .; then + echo "WeKnora image/service reference found in pure DeerFlow stage" >&2 + exit 2 +fi + +echo "[7/7] Create final archive" +FINAL="$DIST/$RELEASE_NAME.tar.gz" +tar -C "$(dirname "$STAGE")" -cf - "$(basename "$STAGE")" | gzip -1 > "$FINAL" +gzip -t "$FINAL" +( + cd "$DIST" + sha256sum "$(basename "$FINAL")" > "$(basename "$FINAL").sha256" +) +printf 'FINAL=%s\n' "$FINAL" +printf 'SIZE_BYTES=%s\n' "$(stat -c %s "$FINAL")" +printf 'SHA256=%s\n' "$(sha256sum "$FINAL" | awk '{print $1}')" diff --git a/deploy/linux-docker-offline-deerflow-only/build-source-update.sh b/deploy/linux-docker-offline-deerflow-only/build-source-update.sh new file mode 100644 index 00000000..9af9a188 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/build-source-update.sh @@ -0,0 +1,308 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# 在 Linux amd64 发版机上执行: +# ./build-source-update.sh WORKSPACE CURRENT_DEPLOY_DIR OUTPUT_ROOT [all|backend|frontend] +# +# CURRENT_DEPLOY_DIR 必须是当前正在使用的完整包解压目录(VERSION.json 会随每次 +# 源码更新向前推进)。首次使用当前 20260827 基线时,还需把同目录下保存的 +# SOURCE_BASELINE.json 放进该部署目录。 + +WORKSPACE="${1:?workspace path is required}" +BASE_BUNDLE="${2:?current DeerFlow deployment directory is required}" +OUTPUT_ROOT="${3:?output root is required}" +COMPONENTS="${4:-all}" + +WORKSPACE="$(realpath "$WORKSPACE")" +BASE_BUNDLE="$(realpath "$BASE_BUNDLE")" +OUTPUT_ROOT="$(realpath -m "$OUTPUT_ROOT")" + +case "$COMPONENTS" in + all|backend|frontend) ;; + *) echo "components must be all, backend or frontend" >&2; exit 2 ;; +esac +[[ -f "$WORKSPACE/offline-backend-20260512/backend/pyproject.toml" ]] || exit 2 +[[ -f "$WORKSPACE/frontend-web/package.json" ]] || exit 2 +[[ -f "$BASE_BUNDLE/VERSION.json" ]] || { echo "base bundle lacks VERSION.json" >&2; exit 2; } +[[ -f "$BASE_BUNDLE/SOURCE_BASELINE.json" ]] || { + echo "base bundle lacks SOURCE_BASELINE.json; use the immutable baseline sidecar" >&2 + exit 2 +} + +case "$OUTPUT_ROOT" in + /|/opt|/opt/codex-build|"$WORKSPACE"|"$BASE_BUNDLE") + echo "unsafe output root: $OUTPUT_ROOT" >&2 + exit 2 + ;; +esac + +mapfile -t BASE_VALUES < <(python3 - "$BASE_BUNDLE/VERSION.json" <<'PY' +import json, sys +d = json.load(open(sys.argv[1], encoding="utf-8")) +print(d["release"]) +print(d["images"]["deerflow_backend"]) +print(d["images"]["deerflow_frontend"]) +PY +) +BASE_RELEASE="${BASE_VALUES[0]}" +BASE_BACKEND_IMAGE="${BASE_VALUES[1]}" +BASE_FRONTEND_IMAGE="${BASE_VALUES[2]}" + +for value in "$BASE_RELEASE" "$BASE_BACKEND_IMAGE" "$BASE_FRONTEND_IMAGE"; do + [[ "$value" =~ ^[A-Za-z0-9._:/,+-]+$ ]] || { echo "unsafe base metadata value: $value" >&2; exit 2; } +done + +ensure_base_images() { + if docker image inspect "$BASE_BACKEND_IMAGE" "$BASE_FRONTEND_IMAGE" >/dev/null 2>&1; then + return + fi + local archive="$BASE_BUNDLE/images/deerflow-images-linux-amd64.tar.gz" + [[ -f "$archive" ]] || { + echo "base images are not loaded and full image archive is unavailable: $archive" >&2 + exit 2 + } + echo "Loading saved base images once..." + gzip -dc "$archive" | docker load + docker image inspect "$BASE_BACKEND_IMAGE" "$BASE_FRONTEND_IMAGE" >/dev/null +} + +ensure_base_images +[[ "$(docker image inspect -f '{{.Architecture}}' "$BASE_BACKEND_IMAGE")" == "amd64" ]] || exit 2 +[[ "$(docker image inspect -f '{{.Architecture}}' "$BASE_FRONTEND_IMAGE")" == "amd64" ]] || exit 2 + +echo "[1/7] Verify immutable dependency baseline" +python3 - "$WORKSPACE" "$BASE_BUNDLE/SOURCE_BASELINE.json" <<'PY' +from __future__ import annotations +import hashlib, json, sys +from pathlib import Path + +workspace = Path(sys.argv[1]).resolve() +baseline = json.load(open(sys.argv[2], encoding="utf-8")) +if baseline.get("format_version") != 1: + raise SystemExit("unsupported SOURCE_BASELINE.json format") +for relative, expected in baseline.get("locked_files", {}).items(): + path = (workspace / relative).resolve() + try: + path.relative_to(workspace) + except ValueError: + raise SystemExit(f"unsafe locked path: {relative}") + if not path.is_file(): + raise SystemExit(f"runtime dependency file is missing: {relative}") + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected: + raise SystemExit( + f"runtime dependency changed: {relative}\n" + "source-only update is refused; create a new full dependency baseline" + ) +print("immutable runtime dependency baseline passed") +PY + +STAMP="${RELEASE_STAMP:-$(date +%Y%m%d-%H%M%S)}" +SOURCE_COMMIT="${DEERFLOW_COMMIT_OVERRIDE:-}" +if [[ -z "$SOURCE_COMMIT" ]] && git -C "$WORKSPACE" rev-parse HEAD >/dev/null 2>&1; then + SOURCE_COMMIT="$(git -C "$WORKSPACE" rev-parse HEAD)" +fi +[[ "$SOURCE_COMMIT" =~ ^[0-9a-fA-F]{40}$ ]] || { + echo "set DEERFLOW_COMMIT_OVERRIDE to the 40-character source commit" >&2 + exit 2 +} +SOURCE_SHORT="${SOURCE_COMMIT:0:8}" +if [[ -n "$(git -C "$WORKSPACE" status --porcelain --untracked-files=normal -- \ + offline-backend-20260512/backend frontend-web 2>/dev/null || true)" ]]; then + SOURCE_DIRTY=true +else + SOURCE_DIRTY=false +fi +if [[ -n "${DEERFLOW_DIRTY_OVERRIDE:-}" ]]; then + SOURCE_DIRTY="$DEERFLOW_DIRTY_OVERRIDE" +fi +[[ "$SOURCE_DIRTY" == "true" || "$SOURCE_DIRTY" == "false" ]] || exit 2 + +UPDATE_NAME="deerflow-source-update-${STAMP}-${SOURCE_SHORT}" +TARGET_RELEASE="deerflow-source-${STAMP}-${SOURCE_SHORT}" +UPDATE_DIR="$OUTPUT_ROOT/stage/$UPDATE_NAME" +CONTEXT_ROOT="$OUTPUT_ROOT/context/$UPDATE_NAME" +DIST="$OUTPUT_ROOT/dist" + +for target in "$UPDATE_DIR" "$CONTEXT_ROOT"; do + if [[ -e "$target" ]]; then + resolved="$(realpath "$target")" + case "$resolved" in + "$OUTPUT_ROOT/stage/$UPDATE_NAME"|"$OUTPUT_ROOT/context/$UPDATE_NAME") rm -rf -- "$resolved" ;; + *) echo "refusing to replace unexpected path: $resolved" >&2; exit 2 ;; + esac + fi +done +mkdir -p "$UPDATE_DIR/docker" "$UPDATE_DIR/payload/images" "$UPDATE_DIR/payload/config/deerflow" \ + "$UPDATE_DIR/payload/config/frontend" "$UPDATE_DIR/source" "$UPDATE_DIR/runtime" \ + "$CONTEXT_ROOT" "$DIST" + +BACKEND_CHANGED=false +FRONTEND_CHANGED=false +[[ "$COMPONENTS" == "all" || "$COMPONENTS" == "backend" ]] && BACKEND_CHANGED=true +[[ "$COMPONENTS" == "all" || "$COMPONENTS" == "frontend" ]] && FRONTEND_CHANGED=true + +TARGET_BACKEND_IMAGE="$BASE_BACKEND_IMAGE" +TARGET_FRONTEND_IMAGE="$BASE_FRONTEND_IMAGE" +[[ "$BACKEND_CHANGED" == "false" ]] || TARGET_BACKEND_IMAGE="cmzs/deerflow-backend-offline:src-${STAMP}-${SOURCE_SHORT}" +[[ "$FRONTEND_CHANGED" == "false" ]] || TARGET_FRONTEND_IMAGE="cmzs/deerflow-frontend-offline:src-${STAMP}-${SOURCE_SHORT}" +BASE_BACKEND_IMAGE_ID="$(docker image inspect -f '{{.Id}}' "$BASE_BACKEND_IMAGE")" +BASE_FRONTEND_IMAGE_ID="$(docker image inspect -f '{{.Id}}' "$BASE_FRONTEND_IMAGE")" + +echo "[2/7] Assemble sanitized source payload" +BACKEND_CONTEXT="$CONTEXT_ROOT/backend" +FRONTEND_CONTEXT_ROOT="$CONTEXT_ROOT/frontend-root" +mkdir -p "$BACKEND_CONTEXT" "$FRONTEND_CONTEXT_ROOT/frontend-web" +cp -a "$WORKSPACE/offline-backend-20260512/backend/." "$BACKEND_CONTEXT/" +cp -a "$WORKSPACE/frontend-web/." "$FRONTEND_CONTEXT_ROOT/frontend-web/" +rm -rf -- "$BACKEND_CONTEXT/.venv" "$BACKEND_CONTEXT/.deer-flow" \ + "$BACKEND_CONTEXT/.pytest_cache" "$BACKEND_CONTEXT/.ruff_cache" \ + "$BACKEND_CONTEXT/skills/custom" \ + "$FRONTEND_CONTEXT_ROOT/frontend-web/node_modules" "$FRONTEND_CONTEXT_ROOT/frontend-web/dist" \ + "$FRONTEND_CONTEXT_ROOT/frontend-web/.vite" +mkdir -p "$BACKEND_CONTEXT/skills/custom" +find "$BACKEND_CONTEXT" "$FRONTEND_CONTEXT_ROOT" -type d -name __pycache__ -prune -exec rm -rf -- {} + +python3 "$WORKSPACE/deploy/linux-docker-offline/sanitize_release_sources.py" backend "$BACKEND_CONTEXT" +python3 "$WORKSPACE/deploy/linux-docker-offline/sanitize_release_sources.py" frontend \ + "$FRONTEND_CONTEXT_ROOT/frontend-web" + +compare_backend_manifests() { + local cid temp manifest failed=false + temp="$(mktemp -d)" + cid="$(docker create "$BASE_BACKEND_IMAGE")" + for manifest in pyproject.toml uv.lock requirements-offline.txt packages/harness/pyproject.toml; do + mkdir -p "$temp/$(dirname "$manifest")" + if ! docker cp "$cid:/app/backend/$manifest" "$temp/$manifest" >/dev/null \ + || ! cmp -s "$BACKEND_CONTEXT/$manifest" "$temp/$manifest"; then + echo "backend dependency manifest differs from active base: $manifest" >&2 + failed=true + fi + done + docker rm -f "$cid" >/dev/null 2>&1 || true + rm -rf -- "$temp" + [[ "$failed" == "false" ]] || { + echo "source-only update refused; create a new full dependency baseline" >&2 + exit 2 + } +} +[[ "$BACKEND_CHANGED" == "false" ]] || compare_backend_manifests + +if [[ "$BACKEND_CHANGED" == "true" ]]; then + cp -a "$BACKEND_CONTEXT" "$UPDATE_DIR/source/backend" +fi +if [[ "$FRONTEND_CHANGED" == "true" ]]; then + cp -a "$FRONTEND_CONTEXT_ROOT/frontend-web" "$UPDATE_DIR/source/frontend" +fi + +if [[ "$BACKEND_CHANGED" == "true" ]]; then + cp -a "$BACKEND_CONTEXT/config.yaml" "$UPDATE_DIR/payload/config/deerflow/config.base.yaml" + cp -a "$BACKEND_CONTEXT/extensions_config.json" "$UPDATE_DIR/payload/config/deerflow/extensions_config.json" +else + cp -a "$BASE_BUNDLE/config/deerflow/config.base.yaml" "$UPDATE_DIR/payload/config/deerflow/config.base.yaml" + cp -a "$BASE_BUNDLE/config/deerflow/extensions_config.json" \ + "$UPDATE_DIR/payload/config/deerflow/extensions_config.json" +fi +if [[ "$FRONTEND_CHANGED" == "true" ]]; then + cp -a "$FRONTEND_CONTEXT_ROOT/frontend-web/public/runtime-config.js" \ + "$UPDATE_DIR/payload/config/frontend/runtime-config.base.js" +else + cp -a "$BASE_BUNDLE/config/frontend/runtime-config.base.js" \ + "$UPDATE_DIR/payload/config/frontend/runtime-config.base.js" +fi + +echo "[3/7] Compile frontend assets when requested" +if [[ "$FRONTEND_CHANGED" == "true" ]]; then + mkdir -p "$UPDATE_DIR/runtime/frontend-dist" "$UPDATE_DIR/runtime/nginx" + DOCKER_BUILDKIT=1 docker build --platform linux/amd64 --network host --provenance=false \ + --target artifact --output "type=local,dest=$UPDATE_DIR/runtime/frontend-dist" \ + -f "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-assets" \ + "$FRONTEND_CONTEXT_ROOT" + [[ -s "$UPDATE_DIR/runtime/frontend-dist/index.html" ]] || { + echo "frontend build did not produce index.html" >&2 + exit 2 + } + cp -a "$WORKSPACE/deploy/linux-docker-offline/nginx/deerflow.conf" \ + "$UPDATE_DIR/runtime/nginx/deerflow.conf" +fi + +echo "[4/7] Add offline overlay builders and target metadata" +cp -a "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/Dockerfile.backend-source-overlay" \ + "$UPDATE_DIR/docker/" +cp -a "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/Dockerfile.frontend-dist-overlay" \ + "$UPDATE_DIR/docker/" +cp -a "$WORKSPACE/deploy/linux-docker-offline-deerflow-only/source-update-template/APPLY_SOURCE_UPDATE.sh" \ + "$UPDATE_DIR/APPLY_SOURCE_UPDATE.sh" + +cat > "$UPDATE_DIR/SOURCE_UPDATE_MANIFEST.env" < "$UPDATE_DIR/payload/VERSION.json" < "$UPDATE_DIR/payload/images/deerflow-image-list.txt" + +echo "[5/7] Validate source update structure" +chmod 0755 "$UPDATE_DIR/APPLY_SOURCE_UPDATE.sh" +if find "$UPDATE_DIR" -type l -print -quit | grep -q .; then + echo "source update stage contains a symbolic link" >&2 + exit 2 +fi +grep -Fq "$TARGET_BACKEND_IMAGE" "$UPDATE_DIR/payload/VERSION.json" +grep -Fq "$TARGET_FRONTEND_IMAGE" "$UPDATE_DIR/payload/VERSION.json" +[[ "$(grep -cvE '^[[:space:]]*(#|$)' "$UPDATE_DIR/payload/images/deerflow-image-list.txt")" == "2" ]] + +echo "[6/7] Generate content checksums" +( + cd "$UPDATE_DIR" + find . -type f ! -name SHA256SUMS -print0 | LC_ALL=C sort -z | xargs -0 sha256sum > SHA256SUMS + sha256sum -c SHA256SUMS >/dev/null +) + +echo "[7/7] Create source-only update archive" +FINAL="$DIST/$UPDATE_NAME.tar.gz" +tar -C "$(dirname "$UPDATE_DIR")" -cf - "$(basename "$UPDATE_DIR")" | gzip -1 > "$FINAL" +gzip -t "$FINAL" +( + cd "$DIST" + sha256sum "$(basename "$FINAL")" > "$(basename "$FINAL").sha256" +) +printf 'FINAL=%s\n' "$FINAL" +printf 'SIZE_BYTES=%s\n' "$(stat -c %s "$FINAL")" +printf 'SHA256=%s\n' "$(sha256sum "$FINAL" | awk '{print $1}')" diff --git a/deploy/linux-docker-offline-deerflow-only/docs/01-部署前准备.md b/deploy/linux-docker-offline-deerflow-only/docs/01-部署前准备.md new file mode 100644 index 00000000..1deaa1a4 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/01-部署前准备.md @@ -0,0 +1,45 @@ +# 01 部署前准备 + +## 服务器要求 + +- Linux x86_64/amd64。 +- Docker Engine 和独立命令 `docker-compose` 已安装且运行正常;脚本优先使用它,并兼容 Compose v2 插件作为回退。 +- 建议至少 8 核 CPU、16 GiB 内存、20 GiB 可用磁盘;镜像加载期间需要同时容纳压缩包、解压目录和 Docker 镜像层。 +- 可访问外部 MySQL 8.x;部署账号需要连接、建库和建表权限。 +- 可从 Docker 容器访问 OpenAI 兼容的 DeerFlow 对话模型接口。 + +目标机部署过程不需要访问互联网、镜像仓库、npm 或 pip。`PUBLIC_HOST` 对应的前端和后端端口需要在防火墙放行。 + +本包不包含 WeKnora 以及其 PostgreSQL、Redis、DocReader、独立远程 Sandbox 镜像,也不包含模型权重。默认本地沙箱位于 DeerFlow 后端容器内,Office/PDF 依赖已经随该镜像离线交付。 + +## 部署前检查 + +```bash +uname -m # 应为 x86_64 +docker version +docker-compose version # 目标服务器使用此命令 +docker info +df -h +``` + +还需要系统命令:`bash`、`tar`、`gzip`、`sha256sum`、`curl`。部署用户可以直接使用 Docker;否则命令前加 `sudo`。 + +准备并确认以下网络方向: + +- 浏览器 → `FRONTEND_PORT`。 +- 浏览器 → `BACKEND_PORT`(前端运行时会直连 Gateway)。 +- DeerFlow 后端容器 → 外部 MySQL。 +- DeerFlow 后端容器 → OpenAI 兼容模型接口。 +- 如果启用 Tavily、Serper 或业务接口,还需按实际调用方向访问相应地址。 + +同一服务器上的服务不能从容器里使用 `127.0.0.1` 访问宿主机。应使用宿主机内网 IP;Docker 支持 `host-gateway` 时也可使用 `host.docker.internal`。 + +## 文件校验 + +把压缩包和同名 `.sha256` 放在同一目录: + +```bash +sha256sum -c deerflow-linux-amd64-offline-*.tar.gz.sha256 +``` + +显示 `OK` 后再解压。校验失败不要继续部署,应重新传输文件。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/02-deploy.env配置说明.md b/deploy/linux-docker-offline-deerflow-only/docs/02-deploy.env配置说明.md new file mode 100644 index 00000000..6dbc2da0 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/02-deploy.env配置说明.md @@ -0,0 +1,86 @@ +# 02 deploy.env 配置说明 + +唯一人工配置源: + +```text +config/deploy.env +``` + +`data/generated/` 下的 Compose 环境、DeerFlow `config.yaml` 和前端 `runtime-config.js` 每次启动都会自动生成,不要修改。 + +必须修改: + +```dotenv +PUBLIC_HOST=10.10.10.10 +FRONTEND_PORT=5174 +BACKEND_PORT=8001 + +MYSQL_HOST=10.10.10.20 +MYSQL_PORT=3306 +MYSQL_DATABASE=deerflow +MYSQL_USER=deerflow +MYSQL_PASSWORD=your-real-password + +DEERFLOW_MODEL_COUNT=2 + +DEERFLOW_MODEL_1_NAME=qwen3-32b +DEERFLOW_MODEL_1_DISPLAY_NAME=通用模型 +DEERFLOW_MODEL_1_MODEL=qwen3-32b +DEERFLOW_MODEL_1_BASE_URL=http://10.10.10.25:8000/v1 +DEERFLOW_MODEL_1_API_KEY=your-real-key-1 +DEERFLOW_MODEL_1_SUPPORTS_VISION=false +DEERFLOW_MODEL_1_SUPPORTS_THINKING=false + +DEERFLOW_MODEL_2_NAME=qwen-vl +DEERFLOW_MODEL_2_DISPLAY_NAME=视觉模型 +DEERFLOW_MODEL_2_MODEL=qwen-vl +DEERFLOW_MODEL_2_BASE_URL=http://10.10.10.26:8000/v1 +DEERFLOW_MODEL_2_API_KEY=your-real-key-2 +DEERFLOW_MODEL_2_SUPPORTS_VISION=true +DEERFLOW_MODEL_2_SUPPORTS_THINKING=false +``` + +同宿主机 MySQL 不要填 `127.0.0.1`;应填宿主机内网 IP,或在 Docker 的 `host-gateway` 正常时填写 `host.docker.internal`。 + +字段解释: + +| 字段 | 必填 | 说明 | +|---|---:|---| +| `PUBLIC_HOST` | 是 | 用户访问的服务器 IP 或域名,只填主机名,不带协议和路径 | +| `FRONTEND_PORT` | 是 | 页面端口,默认示例 `5174` | +| `BACKEND_PORT` | 是 | Gateway 端口,默认示例 `8001`,不能与前端端口相同 | +| `MYSQL_HOST` | 是 | 外部 MySQL 主机名、内网 IP 或 `host.docker.internal` | +| `MYSQL_PORT` | 是 | MySQL 监听端口 | +| `MYSQL_DATABASE` | 是 | 仅允许字母、数字和下划线;账号需有建库/建表权限 | +| `MYSQL_USER` / `MYSQL_PASSWORD` | 是 | DeerFlow 数据库账号和密码 | +| `DEERFLOW_MODEL_COUNT` | 是 | 启用的模型数量,范围 1~20;第 1 个模型是系统默认模型 | +| `DEERFLOW_MODEL_N_NAME` | 是 | DeerFlow 内部唯一名称,不能与其他模型重复 | +| `DEERFLOW_MODEL_N_DISPLAY_NAME` | 否 | 页面显示名称,留空时使用 `NAME` | +| `DEERFLOW_MODEL_N_MODEL` | 否 | 发给模型服务的真实 model id,留空时使用 `NAME` | +| `DEERFLOW_MODEL_N_BASE_URL` | 是 | OpenAI 兼容根地址,通常以 `/v1` 结尾 | +| `DEERFLOW_MODEL_N_API_KEY` | 是 | 对应模型接口 Key;内网无鉴权时也需填写接口接受的占位值 | +| `DEERFLOW_MODEL_N_SUPPORTS_VISION` | 否 | 是否支持图片输入,默认 `false` | +| `DEERFLOW_MODEL_N_SUPPORTS_THINKING` | 否 | 是否支持 thinking,默认 `false` | +| `DEERFLOW_MODEL_N_SUPPORTS_REASONING_EFFORT` | 否 | 是否接受 reasoning effort,默认 `false` | +| `DEERFLOW_MODEL_N_REQUEST_TIMEOUT` | 否 | 单次请求超时秒数,默认 `600`,范围 1~3600 | +| `DEERFLOW_MODEL_N_MAX_RETRIES` | 否 | 模型请求重试次数,默认 `2`,范围 0~10 | +| `DEERFLOW_MODEL_N_STREAM_USAGE` | 否 | 是否在流式响应中统计用量,默认 `true` | +| `DEERFLOW_MODEL_N_MAX_TOKENS` | 否 | 最大输出 token;留空表示由接口默认值决定 | +| `DEERFLOW_MODEL_N_TEMPERATURE` | 否 | 温度,范围 0~2;留空表示由接口默认值决定 | +| `DEERFLOW_MODEL_N_USE_RESPONSES_API` | 否 | OpenAI Responses API 开关,普通兼容接口保持 `false` | + +`N` 从 1 开始连续编号。例如 `DEERFLOW_MODEL_COUNT=3` 时,必须完整填写模型 1、2、3;模型 4 之后即使保留示例也会被忽略。要增加模型,复制最后一个模型块、把编号加 1,并同步修改 `DEERFLOW_MODEL_COUNT`。要删除模型,按连续编号整理后减小 `COUNT`。 + +所有模型默认使用 `langchain_openai:ChatOpenAI`。如果确实需要自定义 provider,可增加 `DEERFLOW_MODEL_N_USE=包路径:类名`;对应 Python 包必须已经内置,或通过 `source/wheelhouse` 离线安装。 + +Tavily、Serper、任务深链和所有 `VITE_*` 业务接口均为可选项,不使用就保持空值。示例保留地址和 `example` 密钥会被启动脚本拒绝,防止误部署。 + +任务深链不接真实系统时保持 `TASK_DEEPLINK_MOCK_MODE=true`。接真实系统时改为 `false`,并填写对应的 `TASK_*_URL` 与前端业务 `VITE_*` 地址。URL 必须是完整 `http://` 或 `https://` 地址,不要在 URL 内嵌账号密码,也不要附带查询参数。 + +`deploy.env` 是简单 `KEY=VALUE` 文件:一个配置占一行,不要在等号两侧加空格,不支持多行值;值包含 `#` 时不要写行尾注释。修改后执行 `--restart`,脚本会重新渲染并覆盖 `data/generated/`,然后重建前后端容器,因此不要直接编辑生成文件。`--restart` 不执行数据库预检、不加载镜像;首次部署仍使用 `--start`。 + +运行时生成的 MySQL SQLAlchemy 地址固定为 `mysql+aiomysql://...`。用户名和密码会自动 URL 编码,不要自己在 `MYSQL_PASSWORD` 中把 `@`、`#` 等字符改成 `%xx`。 + +DeerFlow 登录 JWT 密钥首次启动自动随机生成到 `data/state/runtime-secrets.json`,无需人工配置;该文件应随数据库和数据目录一起备份。 + +本包没有 WeKnora 配置项。运行配置始终把 WeKnora 地址置空,并关闭 LLMWiki 本地同步和向量索引。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/03-一键部署与启停.md b/deploy/linux-docker-offline-deerflow-only/docs/03-一键部署与启停.md new file mode 100644 index 00000000..c3d5a395 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/03-一键部署与启停.md @@ -0,0 +1,46 @@ +# 03 一键部署与启停 + +```bash +tar -xzf deerflow-linux-amd64-offline-*.tar.gz +cd deerflow-linux-amd64-offline-* +cp -a config/deploy.env config/deploy.env.before-edit +vi config/deploy.env +sudo ./OFFLINE_INSTALL.sh --start +``` + +启动会依次执行:检查镜像、必要时校验并加载离线镜像、校验 `deploy.env`、生成运行配置、使用 aiomysql 预检 MySQL、从 `source/wheelhouse` 安装已声明的离线依赖、使用 `docker-compose up -d --no-build` 启动、等待前后端健康。脚本不使用 Compose v2 专属的 `--pull never`;镜像缺失时会先从包内加载,因此不会现场构建、联网下载 pip 包或拉取镜像。 + +第一次加载镜像耗时取决于磁盘速度。出现“所需 DeerFlow 镜像已存在”表示标签完全匹配,脚本会跳过 `docker load`。启动完成后会打印页面地址,也可以手工检查: + +```bash +curl -f http://127.0.0.1:8001/health +curl -I http://127.0.0.1:5174/ +sudo ./OFFLINE_INSTALL.sh --status +``` + +如果修改过端口,请把示例端口替换成 `deploy.env` 中的值。 + +```bash +# 只停止,保留镜像和数据 +sudo ./OFFLINE_INSTALL.sh --stop + +# 修改 deploy.env 后重新渲染并启动 +sudo ./OFFLINE_INSTALL.sh --restart + +sudo ./OFFLINE_INSTALL.sh --status +sudo ./OFFLINE_INSTALL.sh --logs +sudo ./OFFLINE_INSTALL.sh --logs deerflow-backend + +# 不连接数据库,单独验证沙箱中的 Word/Excel/PPT/PDF 依赖 +sudo ./OFFLINE_INSTALL.sh --office-check +``` + +`--logs` 会持续跟踪日志,按 `Ctrl+C` 只退出日志查看,不会停止服务。前端服务名为 `deerflow-frontend`,后端服务名为 `deerflow-backend`。 + +`--restart` 会重新校验/渲染 `deploy.env`、检查 wheelhouse 指纹、验证挂载源码,然后强制重建前后端容器并等待健康。它不执行 MySQL 预检,不执行 `docker load/build/pull/rmi`,不删除网络和卷。容器重建是为了让新增挂载、依赖和环境变量可靠生效;数据库、用户文件、浏览器上下文和 wheelhouse 安装结果都在持久卷中保留。仅修改智能体、技能或知识文件后也使用 `--restart`。 + +脚本没有删除命令。需要卸载时应先完整备份,然后由运维人员明确执行 Compose `down` 或卷清理,不要把它与日常停止混用。 + +## 开机自启 + +Compose 服务设置为 `restart: unless-stopped`。手工执行 `--stop` 后 Docker 不会自动拉起它,需要恢复时执行 `--start`;如果只是重启服务器且此前没有手工停止,Docker 服务启动后通常会自动恢复容器。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/04-技能与智能体位置.md b/deploy/linux-docker-offline-deerflow-only/docs/04-技能与智能体位置.md new file mode 100644 index 00000000..439ffde3 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/04-技能与智能体位置.md @@ -0,0 +1,62 @@ +# 04 技能与智能体位置 + +## 智能体 + +```text +agents-skills/deerflow-agents//config.yaml +agents-skills/deerflow-agents//SOUL.md +``` + +运行时挂载到 `/app/backend/.deer-flow/agents`。 + +每个智能体使用独立、可读的目录名。`config.yaml` 定义名称、模型、工具和技能等配置,`SOUL.md` 定义角色提示词。额外模板或资源放在同一目录内并使用相对路径。不要使用 32 位十六进制目录名,这类名称保留给运行时用户智能体。 + +```text +agents-skills/deerflow-agents/my-agent/ +├── config.yaml +└── SOUL.md +``` + +## 技能 + +```text +agents-skills/deerflow-skills/public//SKILL.md +agents-skills/deerflow-skills/custom//SKILL.md +``` + +技能需要脚本时放在对应技能目录的 `scripts/` 下。运行时整个目录挂载到 `/app/backend/skills`。 + +推荐结构: + +```text +agents-skills/deerflow-skills/custom/my-skill/ +├── SKILL.md +├── scripts/ +├── references/ +└── assets/ +``` + +`SKILL.md` 是入口说明;脚本放 `scripts/`,资料放 `references/`,静态模板放 `assets/`。技能中如果写死了环境 URL/IP,应在迁移前替换成目标环境地址。 + +## 本地知识文件 + +```text +agents-skills/deerflow-knowledge/ +``` + +运行时挂载到 `/app/backend/knowledge`。修改上述目录后执行 `--restart`。不要进入镜像内部改文件;升级前备份整个 `agents-skills/`。 + +本地知识目录只是随容器挂载的文件位置,不等同于 WeKnora 知识库,也不会自动创建向量库。具体智能体或技能需要通过自身配置读取这些文件。 + +更新后检查: + +```bash +find agents-skills/deerflow-agents -maxdepth 2 -type f | sort +find agents-skills/deerflow-skills -maxdepth 3 -type f | sort +sudo ./OFFLINE_INSTALL.sh --restart +sudo ./OFFLINE_INSTALL.sh --logs deerflow-backend +``` + +YAML 或 Markdown 格式错误通常会出现在后端日志中。上线前先复制原目录作为回退副本,不要在运行中的容器文件系统里直接修改,重建容器会丢失该修改。 + +本发行包没有 `agents-skills/weknora-skills/`,也不会启动 WeKnora。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/05-数据备份与恢复.md b/deploy/linux-docker-offline-deerflow-only/docs/05-数据备份与恢复.md new file mode 100644 index 00000000..2a7f61d9 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/05-数据备份与恢复.md @@ -0,0 +1,32 @@ +# 05 数据备份与恢复 + +需要成套备份: + +- `config/deploy.env`。 +- `data/state/runtime-secrets.json`。 +- `agents-skills/`。 +- 外部 MySQL 中的 DeerFlow 数据库。 +- Docker 卷 `cmzs-deerflow_deerflow-data`、`cmzs-deerflow_deerflow-browser-context`、`cmzs-deerflow_deerflow-users`。 + +备份前执行 `sudo ./OFFLINE_INSTALL.sh --stop`,数据库和卷恢复完成后再执行 `--start`。不要重新生成丢失的 JWT 密钥后继续使用旧登录数据。 + +## 卷备份示例 + +先确认实际卷名: + +```bash +docker volume ls --filter name=cmzs-deerflow +``` + +以 `deerflow-data` 为例导出到当前目录: + +```bash +docker run --rm \ + -v cmzs-deerflow_deerflow-data:/source:ro \ + -v "$PWD":/backup \ + alpine:3.20 sh -c 'cd /source && tar -czf /backup/deerflow-data.tar.gz .' +``` + +纯离线目标机不一定有 `alpine:3.20`,该命令仅在镜像已存在时使用;否则使用运维环境已有的备份工具。MySQL 应使用 `mysqldump` 或单位数据库备份平台独立备份。 + +恢复时必须保证应用已停止,先恢复数据库,再恢复三个卷、`agents-skills/`、`deploy.env` 和 `runtime-secrets.json`,最后执行 `--start`。恢复后检查用户登录、智能体列表、历史会话和文件访问。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/06-故障排查与校验.md b/deploy/linux-docker-offline-deerflow-only/docs/06-故障排查与校验.md new file mode 100644 index 00000000..05f69792 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/06-故障排查与校验.md @@ -0,0 +1,54 @@ +# 06 故障排查与校验 + +外层包校验: + +```bash +sha256sum -c deerflow-linux-amd64-offline-*.tar.gz.sha256 +``` + +包内镜像校验: + +```bash +sha256sum -c SHA256SUMS +``` + +常用排查: + +```bash +sudo ./OFFLINE_INSTALL.sh --status +sudo ./OFFLINE_INSTALL.sh --logs deerflow-backend +docker image inspect "$(head -n 1 images/deerflow-image-list.txt)" +``` + +查看两个容器的退出码和健康状态: + +```bash +docker-compose --project-name cmzs-deerflow \ + --env-file data/generated/compose.env \ + -f compose/docker-compose.yml ps -a +``` + +- MySQL 预检失败:检查监听地址、账号来源网段、建库权限和防火墙。生成地址应以 `mysql+aiomysql://` 开头,可用 `grep '^MYSQL_DATABASE_URL=' data/generated/compose.env` 确认(不要把含密码的结果发给他人)。 +- 日志出现 `No module named aiomysql`:确认 `source/wheelhouse/aiomysql-0.3.2-py3-none-any.whl` 和 `requirements.txt` 中的 aiomysql 兼容组三行都存在,然后执行 `--restart`。 +- 日志出现 `ping() missing ... reconnect`:不要只升级 aiomysql;恢复 `aiomysql==0.3.2`、`PyMySQL==1.2.0`、`SQLAlchemy==2.0.51` 三行及对应 wheel,再执行 `--restart`。 +- wheelhouse 报 `No matching distribution found`:缺少传递依赖 wheel,或 wheel 不是 Python 3.12/Linux amd64 版本;按文档 13 补齐。 +- Docker 创建网络失败并提示 `iptables` 不存在:先修复宿主机 Docker/iptables 安装,不要修改包内文件绕过。 +- 模型调用失败:确认模型地址能从容器访问;宿主机服务不要填 `127.0.0.1`。 +- 再次启动仍执行 `docker load`:说明镜像标签缺失或镜像清单不匹配。 +- 页面能打开但请求失败:确认浏览器能访问 `BACKEND_PORT`,并检查 `data/generated/frontend/runtime-config.js` 中由 `PUBLIC_HOST` 生成的地址。 +- 端口已占用:用 `ss -lntp` 查占用,修改两个端口后执行 `--restart`。 +- 容器反复重启:查看后端最后 300 行日志,重点检查 MySQL 表创建、配置 YAML 和模型配置。 +- 技能/智能体没有出现:核对目录层级、文件权限和名称,再重启后端。 +- Word、Excel 或 PPT 上传后没有生成 Markdown:先执行 `sudo ./OFFLINE_INSTALL.sh --office-check`;通过后再检查后端上传日志和 `uploads.auto_convert_documents` 是否为 `true`。 + +`--stop` 后镜像应继续存在;脚本不会执行 `docker rmi` 或删除数据卷。 + +## 安全和内容检查 + +`config/deploy.env` 和 `data/state/runtime-secrets.json` 权限应为 `600`。不要把解压后的真实配置目录上传到公共文件服务器,也不要把运行时生成文件重新制作成分发包。 + +```bash +grep -vE '^[[:space:]]*(#|$)' images/deerflow-image-list.txt +``` + +清单应严格只有 DeerFlow Backend 和 Frontend,不应看到 WeKnora、ParadeDB、Redis 或 DocReader 镜像。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/07-升级与版本回退.md b/deploy/linux-docker-offline-deerflow-only/docs/07-升级与版本回退.md new file mode 100644 index 00000000..290ecfb6 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/07-升级与版本回退.md @@ -0,0 +1,51 @@ +# 07 升级与版本回退 + +日常只修改 `.py`、`.tsx`、`.ts`、样式和页面资源时,优先使用 `11-只替换前后端源码.md` 的源码覆盖层机制。它不传输 Docker 基础层,在部署服务器上基于已保存镜像离线生成很薄的新镜像。 + +涉及 Compose、部署控制文件或普通镜像增量包时,使用本包自带的通用增量机制,不再解压一套完整基线包: + +```bash +tar -xzf deerflow-update-*.tar.gz +cd /当前正在使用的DeerFlow部署目录 +sudo ./UPDATE_INSTALL.sh --apply /增量包解压目录 +``` + +增量更新只加载发生变化的镜像并更新白名单控制文件,保留现有 `config/deploy.env`、JWT 密钥、智能体、custom 技能、知识文件、Docker 卷和旧镜像。详细制作规范、变更分类及校验清单见 `09-后续增量更新发版手册.md`。 + +应用时会把旧控制文件备份到 `data/updates/backups/`。需要回退时: + +```bash +sudo ./UPDATE_INSTALL.sh --rollback \ + /当前部署目录/data/updates/backups/<对应备份目录> +``` + +该操作恢复旧镜像引用和控制文件;不会反向恢复数据库。若新版本执行过不兼容数据库迁移,必须同时恢复升级前数据库备份。 + +## 完整基线包切换(仅用于跨大版本或基线损坏) + +不同版本完整离线包解压到不同目录,不要覆盖旧目录。升级前备份旧包的 `config/deploy.env`、`data/state/runtime-secrets.json`、`agents-skills/`、外部 MySQL 和三个 Docker 卷。 + +推荐步骤: + +```bash +cd /旧包目录 +sudo ./OFFLINE_INSTALL.sh --stop + +cd /新包目录 +# 按旧配置逐项填写新版本 deploy.env;不要直接覆盖,以免漏掉新增字段 +vi config/deploy.env +# 复制经人工确认的智能体、技能、知识文件 +sudo ./OFFLINE_INSTALL.sh --start +``` + +同一个 Compose 项目名会复用现有数据卷。新版本首次启动可能自动创建或迁移数据库表,必须先完成数据库备份。不要在没有数据库备份的情况下来回切换应用版本。 + +回退时停止新包,在数据库兼容且已备份的前提下启动旧包。若新版本执行了不可逆数据库迁移,应先恢复升级前数据库和卷快照,再启动旧包。 + +```bash +cat VERSION.json +cat images/deerflow-image-list.txt +sha256sum -c SHA256SUMS +``` + +旧镜像不会被 `--stop` 删除,因此通常可以直接回退;只有运维人员明确清理镜像后才需要重新加载旧包镜像。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/08-离线验收清单.md b/deploy/linux-docker-offline-deerflow-only/docs/08-离线验收清单.md new file mode 100644 index 00000000..3b736069 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/08-离线验收清单.md @@ -0,0 +1,43 @@ +# 08 离线验收清单 + +## 安装前 + +- 外层 `.sha256` 校验为 `OK`。 +- `uname -m` 为 `x86_64`。 +- Docker Engine、`docker-compose` 正常。 +- 外部 MySQL 和模型接口可从服务器/容器网络访问。 +- `config/deploy.env` 中没有 `192.0.2.*` 或 `example` 示例值。 + +## 首次启动 + +```bash +sudo ./OFFLINE_INSTALL.sh --start +sudo ./OFFLINE_INSTALL.sh --status +curl -f http://127.0.0.1:/health +curl -I http://127.0.0.1:/ +sudo ./OFFLINE_INSTALL.sh --office-check +``` + +确认两个容器均为运行/健康状态,浏览器能打开页面,登录和一次最小模型问答成功。 + +## 停止与再次启动 + +```bash +sudo ./OFFLINE_INSTALL.sh --stop +docker image inspect "$(sed -n '1p' images/deerflow-image-list.txt)" +docker image inspect "$(sed -n '2p' images/deerflow-image-list.txt)" +sudo ./OFFLINE_INSTALL.sh --start +``` + +停止后两个镜像仍能 inspect;再次启动日志应显示跳过 `docker load`,容器重新健康,历史数据仍在。 + +## 功能与内容 + +- 智能体目录可见,命名智能体能加载。 +- 自定义技能放入 `agents-skills/deerflow-skills/custom/` 后重启可见。 +- 前后端端口与 `deploy.env` 一致。 +- 未配置的业务接口保持空值,不意外访问旧 UAT/生产地址。 +- 镜像清单只有两个 DeerFlow 镜像,不存在 WeKnora 相关服务。 +- Office 自检输出 `OFFICE_RUNTIME_SELF_CHECK=PASS`,Word、Excel、PowerPoint 样例均能生成并重新解析。 + +验收完成后保留:离线包、同名 `.sha256`、实际 `deploy.env` 的受控备份、数据库/卷备份和验收记录。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/09-后续增量更新发版手册.md b/deploy/linux-docker-offline-deerflow-only/docs/09-后续增量更新发版手册.md new file mode 100644 index 00000000..1d99f424 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/09-后续增量更新发版手册.md @@ -0,0 +1,141 @@ +# 09 后续增量更新发版手册 + +本文约定以当前完整离线包作为“基线包”。基线只安装一次;后续常规版本发布为 `deerflow-update-<基线>-to-<目标版本>.tar.gz`,不再重复制作并传输整套前后端基础包。 + +如果日常更新仅为前后端源码,优先使用 `11-只替换前后端源码.md` 的源码覆盖包。源码覆盖包不执行 `docker save`,因此不会重复携带基础镜像层;本文的通用镜像增量机制保留给依赖、Compose 或控制文件变化。 + +## 一、哪些内容需要重新打包 + +| 变更内容 | 增量包包含内容 | 不需要重复包含 | +|---|---|---| +| 仅文档、Compose、渲染脚本 | `payload/` 文件 | 任意 Docker 镜像 | +| 仅技能、命名智能体 | 经审查的增量文件和说明 | 前后端镜像;不得覆盖用户 custom 数据 | +| 仅前端源码 | 新前端镜像 | 后端镜像、数据库、数据卷 | +| 仅后端源码,依赖清单未变 | 新后端代码镜像 | 前端镜像、数据库、数据卷 | +| Python/系统依赖或基础镜像变化 | 新后端完整镜像 | 前端镜像及整套基线目录 | +| 前后端都变化 | 两个新镜像 | 配置、历史数据和旧镜像 | + +判断后端依赖是否变化时,至少比较: + +```text +offline-backend-20260512/backend/pyproject.toml +offline-backend-20260512/backend/uv.lock +offline-backend-20260512/backend/requirements-offline.txt +offline-backend-20260512/backend/packages/harness/pyproject.toml +deploy/linux-docker-offline/Dockerfile.backend +``` + +只要其中任一文件变化,就按“后端依赖变化”构建,禁止把新代码强行覆盖到旧依赖镜像。 + +## 二、增量包固定结构 + +```text +deerflow-update--to-/ +├── UPDATE_MANIFEST.env +├── SHA256SUMS +├── images/ +│ └── update-images-linux-amd64.tar.gz # 没有镜像变化时可省略 +└── payload/ + ├── VERSION.json + ├── images/deerflow-image-list.txt + ├── OFFLINE_INSTALL.sh # 有变化才放 + ├── UPDATE_INSTALL.sh # 有变化才放 + ├── compose/ # 有变化才放 + ├── scripts/ # 有变化才放 + ├── config/deerflow/ # 模板变化才放 + ├── config/frontend/ # 模板变化才放 + └── docs/ # 发版说明和迁移说明 +``` + +`UPDATE_MANIFEST.env` 示例: + +```dotenv +FORMAT_VERSION=1 +BASE_RELEASE=deerflow-linux-amd64-offline-20260826-xxxxxx +TARGET_RELEASE=deerflow-linux-amd64-offline-20260915-xxxxxx +TARGET_BACKEND_IMAGE=cmzs/deerflow-backend-offline:20260915-xxxxxx-abcdef12 +TARGET_FRONTEND_IMAGE=cmzs/deerflow-frontend-offline:20260826-xxxxxx-b2c60bb2 +``` + +即使某一端镜像没有变化,目标清单也必须写出它当前继续使用的完整标签。`payload/VERSION.json` 与 `payload/images/deerflow-image-list.txt` 必须使用同一组标签。 + +## 三、发版机制作步骤 + +1. 固定源码 commit,记录 `git rev-parse HEAD`,确认不误带未提交文件。 +2. 对照上表判断变更组件;依赖变化时重建完整后端镜像,源码变化且依赖完全一致时可以使用代码刷新 Dockerfile。 +3. 只导出发生变化的镜像。例如仅后端变化: + +```bash +docker image inspect <新后端镜像> +docker save <新后端镜像> | gzip -1 > images/update-images-linux-amd64.tar.gz +``` + +4. 在 `payload/` 放入新的 `VERSION.json`、两行镜像清单和必要控制文件。严禁放入下列路径: + +```text +config/deploy.env +data/ +agents-skills/deerflow-agents/ 中的用户智能体 +agents-skills/deerflow-skills/custom/ +agents-skills/deerflow-knowledge/ +``` + +5. 如果新增 `deploy.env` 配置项,只更新 `config/deploy.env.example` 或提供 `config/deploy.env.additions.example`,并在发版说明中写明;更新脚本永远不会自动覆盖真实 `deploy.env`。 +6. 生成包内校验: + +```bash +find . -type f ! -name SHA256SUMS -print0 | sort -z | xargs -0 sha256sum > SHA256SUMS +``` + +7. 打包并生成外层校验: + +```bash +tar -czf deerflow-update--to-.tar.gz deerflow-update--to-/ +sha256sum deerflow-update--to-.tar.gz \ + > deerflow-update--to-.tar.gz.sha256 +``` + +8. 在无外网环境至少验证:外层校验、包内校验、镜像 `docker load`、`linux/amd64` 架构、Office 自检、配置渲染以及旧镜像仍存在。 + +说明:Docker 标准 `save/load` 会携带目标镜像需要的层;依赖变化时后端增量包仍可能较大,但它不再重复包含未变化的前端镜像、配置目录和整套基线包。纯控制文件或技能更新可以非常小。 + +## 四、部署机应用步骤 + +```bash +sha256sum -c deerflow-update-*.tar.gz.sha256 +tar -xzf deerflow-update-*.tar.gz + +cd /当前DeerFlow基线部署目录 +sudo ./UPDATE_INSTALL.sh --apply /增量包解压目录 +``` + +脚本顺序为:校验基线版本和 SHA256 → 备份控制文件 → `docker-compose stop` → 加载变更镜像 → 应用白名单文件 → 使用原 `deploy.env` 重新渲染并启动。它不执行 `down`、`rmi` 或 `volume rm`。 + +如果希望先应用、人工检查后再启动: + +```bash +sudo ./UPDATE_INSTALL.sh --apply /增量包解压目录 --no-start +sudo ./OFFLINE_INSTALL.sh --office-check +sudo ./OFFLINE_INSTALL.sh --start +``` + +## 五、回退与版本链 + +每个增量包只允许从清单指定的 `BASE_RELEASE` 应用,不能跳版本。连续升级必须按顺序执行:`A → B → C`。如果要从 A 直接到 C,应另外制作 `A → C` 更新包或交付新的完整基线。 + +控制文件回退命令: + +```bash +sudo ./UPDATE_INSTALL.sh --rollback \ + /当前部署目录/data/updates/backups/ +``` + +`--stop` 和增量更新都保留旧镜像,因此正常回退无需再次 `docker load`。数据库回退不由脚本自动完成:每次涉及表结构或数据迁移的发版,都必须提供迁移说明、升级前备份和明确的回退兼容结论。 + +## 六、每次发版必须记录 + +- 基线版本、目标版本和源码 commit。 +- 新旧前后端镜像标签、镜像 ID、架构和 SHA256。 +- 是否修改依赖、`deploy.env` 字段、数据库结构、Compose 或数据目录。 +- 本次包含的技能/智能体文件及覆盖策略。 +- 升级命令、验证结果、回退命令和数据库回退要求。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/10-Office文档处理与自检.md b/deploy/linux-docker-offline-deerflow-only/docs/10-Office文档处理与自检.md new file mode 100644 index 00000000..c6c332d2 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/10-Office文档处理与自检.md @@ -0,0 +1,53 @@ +# 10 Office 文档处理与自检 + +本包的默认沙箱是 DeerFlow 后端容器内的 `LocalSandboxProvider`。Office 依赖必须安装在后端镜像的 `/opt/venv` 和操作系统层;只安装在宿主机上没有作用。 + +## 已内置能力 + +- Word:DOC、DOCX、ODT。 +- Excel:XLS、XLSX、ODS。 +- PowerPoint:PPT、PPTX、ODP。 +- PDF:文本抽取和 Markdown 转换。 +- Python:MarkItDown、python-docx、openpyxl、pandas、python-pptx、xlrd、lxml、mammoth。 +- 系统工具:LibreOffice Headless、Pandoc、Poppler、`file`、中文 Noto 字体。 + +现代 DOCX/XLSX/PPTX 直接由 Python 组件读取;旧 DOC/XLS/PPT 和 OpenDocument 文件先用独立 LibreOffice 临时配置转换为现代格式,再提取为 Markdown。独立临时配置可避免多个上传任务同时转换时争用 LibreOffice 用户目录锁。 + +`config/deerflow/config.base.yaml` 默认设置: + +```yaml +uploads: + auto_convert_documents: true + pdf_converter: auto +``` + +运行配置由脚本生成,正常部署无需再手工修改 YAML。 + +## 不依赖数据库的自检 + +```bash +sudo ./OFFLINE_INSTALL.sh --office-check +``` + +脚本会在后端镜像中生成最小 DOCX、XLSX、PPTX,再分别转换并检查标记文本,同时检查 LibreOffice、Pandoc、Poppler、`file` 和中文字体。通过时输出: + +```text +OFFICE_RUNTIME_SELF_CHECK=PASS +``` + +该命令只检查镜像和沙箱依赖,不读取 MySQL,也不启动前后端服务。 + +## 上传后的处理路径 + +1. 原文件保存到当前用户、当前会话的 `uploads/`。 +2. 支持的文档自动转换为同名 `.md`。 +3. Agent 同时收到原文件和 Markdown 虚拟路径。 +4. Agent 优先读取 `.md` 做总结、问答和内容抽取;需要生成/修改文件时,可在本地沙箱使用上述 Python 库和系统工具,产物应写入 `/mnt/user-data/outputs/`。 + +如果自检通过但某个文件失败,应查看: + +```bash +sudo ./OFFLINE_INSTALL.sh --logs deerflow-backend +``` + +损坏、加密、带复杂宏或受密码保护的 Office 文件可能仍无法自动解析。此时原文件会保留,但不会保证生成有效 Markdown;应先解除密码或另存为标准 DOCX/XLSX/PPTX 后重新上传。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/11-只替换前后端源码.md b/deploy/linux-docker-offline-deerflow-only/docs/11-只替换前后端源码.md new file mode 100644 index 00000000..9661dc76 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/11-只替换前后端源码.md @@ -0,0 +1,172 @@ +# 11 只替换前后端源码 + +本流程专门用于日常持续修改源码。完整离线包及其两个镜像作为固定“依赖基线”长期保存,后续更新包不再包含 Docker 镜像基础层,只包含: + +- 脱敏后的完整后端源码; +- 脱敏后的完整前端源码; +- 发版机编译好的前端 `dist` 静态文件; +- 两个很小的离线覆盖层 Dockerfile; +- 目标版本、镜像标签、SHA-256 和一键应用脚本。 + +部署机收到源码更新包后,以本机已经加载的旧镜像为 `FROM`,离线生成新的代码层镜像。Python、Office、LibreOffice、Pandoc、字体、浏览器、Nginx 等依赖都沿用固定基线,不重新安装,也不执行 `docker pull`。 + +## 一、固定基线 + +当前固定依赖基线为: + +```text +deerflow-linux-amd64-offline-20260827-010119.tar.gz +源码提交:8d3eeaac45a79daf31f6ee6e0bf18ba6bc7c4e0b +``` + +正式包、`.sha256` 和 `SOURCE_BASELINE.json` 必须长期保留。`latest` 链接可以以后指向其他版本,但 `base` 链接和带时间戳的文件不得覆盖或删除。 + +`SOURCE_BASELINE.json` 锁定以下后端运行依赖文件: + +```text +offline-backend-20260512/backend/pyproject.toml +offline-backend-20260512/backend/uv.lock +offline-backend-20260512/backend/requirements-offline.txt +offline-backend-20260512/backend/packages/harness/pyproject.toml +offline-backend-20260512/backend/Dockerfile.backend-offline +deploy/linux-docker-offline/Dockerfile.backend +``` + +任意锁定文件发生变化,源码增量构建会立即终止。此时必须制作新的完整依赖基线,不能强行覆盖,否则很容易出现源码已经引用新包、镜像里却没有该依赖的问题。 + +## 二、制作源码更新包 + +在能够完成前端构建的 Linux amd64 发版机上执行: + +```bash +cd /源码根目录 + +# 前后端都更新 +sudo ./deploy/linux-docker-offline-deerflow-only/build-source-update.sh \ + /源码根目录 \ + /当前正在运行的DeerFlow部署目录 \ + /opt/deerflow-source-build \ + all +``` + +只更新后端: + +```bash +sudo ./deploy/linux-docker-offline-deerflow-only/build-source-update.sh \ + /源码根目录 /当前部署目录 /opt/deerflow-source-build backend +``` + +只更新前端: + +```bash +sudo ./deploy/linux-docker-offline-deerflow-only/build-source-update.sh \ + /源码根目录 /当前部署目录 /opt/deerflow-source-build frontend +``` + +构建结果位于: + +```text +/opt/deerflow-source-build/dist/deerflow-source-update-<时间>-.tar.gz +``` + +制作脚本会完成以下检查: + +1. 当前部署版本和当前镜像标签可解析; +2. 当前镜像确实存在且为 `linux/amd64`; +3. `SOURCE_BASELINE.json` 中的所有依赖文件哈希完全一致; +4. 后端四个 Python 依赖清单与正在使用的后端镜像逐字节一致; +5. `config.yaml`、`extensions_config.json` 和 `runtime-config.js` 已脱敏; +6. 前端 `pnpm build` 成功并生成 `dist/index.html`; +7. 包内和包外 SHA-256、gzip 完整性正确。 + +前端浏览器不能直接运行 TSX/TypeScript,因此前端源码仍会保存在更新包中供审计,但部署时实际覆盖的是同一次构建产生的 `dist`。前端依赖安装只发生在发版机的构建缓存层,不会进入部署机,也不会在部署时访问网络。 + +## 三、部署机一键应用 + +把源码更新包放到部署服务器,先校验并解压: + +```bash +sha256sum -c deerflow-source-update-*.tar.gz.sha256 +tar -xzf deerflow-source-update-*.tar.gz +cd deerflow-source-update-* +``` + +只检查版本链、校验值和基线镜像,不切换服务: + +```bash +sudo ./APPLY_SOURCE_UPDATE.sh --check /当前正在运行的DeerFlow部署目录 +``` + +一键应用并启动: + +```bash +sudo ./APPLY_SOURCE_UPDATE.sh --apply /当前正在运行的DeerFlow部署目录 +``` + +希望应用后暂不启动: + +```bash +sudo ./APPLY_SOURCE_UPDATE.sh --apply /当前正在运行的DeerFlow部署目录 --no-start +cd /当前正在运行的DeerFlow部署目录 +sudo ./OFFLINE_INSTALL.sh --start +``` + +应用顺序为:校验全部文件 → 校验当前版本和镜像 ID → 离线构建代码层镜像 → 后端导入与 Office 自检 → 备份控制文件 → `docker-compose stop` → 切换镜像标签 → 使用原 `deploy.env` 渲染 → 启动和健康检查。 + +旧服务会一直运行到新覆盖层镜像构建和自检完成,停机时间仅发生在最后切换阶段。脚本不执行 `down`、`rmi`、`volume rm`,不会删除旧镜像和已加载镜像。 + +## 四、保留与不覆盖的内容 + +源码更新不会覆盖: + +- `config/deploy.env`; +- `data/state/runtime-secrets.json`; +- 外部 MySQL 数据; +- Docker 命名卷; +- 用户智能体目录; +- `skills/custom/`; +- 本地知识文件; +- 已加载的旧镜像。 + +后端 `config.yaml`、`extensions_config.json` 和前端 `runtime-config.js` 只更新脱敏模板,启动时仍由唯一配置源 `config/deploy.env` 重新生成实际运行配置。 + +技能、智能体或知识文件若需要单独发版,继续按 `04-技能与智能体位置.md` 指定目录投放,不要混入普通源码覆盖流程。 + +## 五、连续更新和回退 + +每次更新都以“服务器当前正在运行的部署目录”为基线制作,版本按顺序推进: + +```text +完整依赖基线 A → 源码版本 B → 源码版本 C → 源码版本 D +``` + +每个源码包都带完整后端源码和完整前端静态文件,不是二进制差分;但必须按照清单中的 `BASE_RELEASE` 应用,防止误把旧包覆盖到错误版本。 + +应用前的控制文件备份位于: + +```text +/当前部署目录/data/updates/backups/ +``` + +源码更新备份与原 `UPDATE_INSTALL.sh` 兼容,回退命令: + +```bash +cd /当前部署目录 +sudo ./UPDATE_INSTALL.sh --rollback \ + /当前部署目录/data/updates/backups/<对应备份目录> +``` + +旧镜像始终保留,因此回退不需要重新 `docker load`。如果某次源码修改同时包含数据库结构或不可逆数据迁移,必须先备份数据库,并另行提供数据库回退步骤;源码脚本不会自动回滚数据库。 + +## 六、什么时候不能用源码替换 + +出现以下任一情况必须重新制作完整依赖基线: + +- Python 依赖、`uv.lock` 或 harness 依赖变化; +- 新增或修改 apt/system package; +- 修改 Office、LibreOffice、Pandoc、字体或浏览器运行依赖; +- 更换 Python 基础镜像、CPU 架构; +- 修改 Docker 数据卷布局并需要迁移数据; +- 当前固定基线镜像或 `SOURCE_BASELINE.json` 丢失。 + +普通 `.py`、`.tsx`、`.ts`、`.css`、前端 npm 业务依赖、页面资源和后端业务逻辑更新可以继续使用源码覆盖包。前端 npm 依赖会在发版机重新编译进静态文件,不要求部署机安装 Node.js。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/12-直接覆盖源码并重启.md b/deploy/linux-docker-offline-deerflow-only/docs/12-直接覆盖源码并重启.md new file mode 100644 index 00000000..0c1428d7 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/12-直接覆盖源码并重启.md @@ -0,0 +1,66 @@ +# 直接覆盖源码并重启(最简单方式) + +当前部署已经把前后台代码作为宿主机目录挂载到容器。日常修改源码时不需要重新打镜像,也不需要重新加载离线镜像。 + +## 需要替换的目录 + +在 DeerFlow 部署目录中只替换以下内容: + +```text +source/ +├── backend/ +│ ├── app/ # 替换为 offline-backend-20260512/backend/app +│ ├── packages/ # 替换为 offline-backend-20260512/backend/packages +│ └── tests/ # 替换为 offline-backend-20260512/backend/tests +├── frontend/ +│ └── dist/ # 替换为 frontend-web 执行 pnpm build 后的 dist +└── wheelhouse/ # 离线 Python wheel 与 requirements.txt;日常源码替换不要删除 +``` + +注意:后端目录名是 `tests`,不是 `test`。 + +## 前端怎么生成 dist + +在开发或构建机器执行: + +```bash +cd frontend-web +pnpm install +pnpm build +``` + +只把生成的 `frontend-web/dist/` 内容上传到部署目录的 `source/frontend/dist/`。服务器不需要 Node.js,也不会在启动时安装 npm 依赖。 + +## 替换完成后重启 + +进入服务器当前 DeerFlow 部署目录: + +```bash +sudo ./OFFLINE_INSTALL.sh --restart +``` + +该脚本只执行以下操作: + +1. 检查四个代码目录和前端 `index.html`; +2. 从 `deploy.env` 重新生成数据库、模型和前端运行配置; +3. 检查 `source/wheelhouse`,仅在内容变化时离线安装依赖; +4. 使用一次性容器检查后端 Python 导入; +5. 依次重建并重启后端、前端容器; +6. 等待两个服务健康。 + +它不会执行数据库预检,不会 `docker load`,不会构建、拉取或删除镜像,也不会修改 `config/deploy.env`、数据库业务数据、技能、智能体或知识文件。容器会重建以应用新增的挂载和环境变量,但数据库卷、用户数据卷、浏览器上下文卷和 Python 依赖卷均保留。 + +## 哪些修改不能这样更新 + +如果修改了以下内容,不能只覆盖源码: + +- 后端 `pyproject.toml`、`uv.lock`、`requirements-offline.txt` 中涉及无法通过纯 Python wheel 提供的依赖; +- 系统软件、Python/Office/浏览器依赖; +- Nginx 镜像运行环境; +- Dockerfile 或容器启动命令。 + +这些修改涉及系统运行环境,需要重新制作基础镜像包。新增普通纯 Python/C-extension wheel 时,可按 `docs/13-wheelhouse离线新增依赖与多模型.md` 操作,无需重做镜像;单纯修改后端 `app/packages/tests` 和前端页面代码可以直接覆盖。 + +## 建议的安全替换方式 + +上传时先放到临时目录,确认完整后再改名覆盖,避免服务运行期间读到上传一半的文件。替换之前可复制一份旧目录作为备份;若新代码有问题,恢复旧目录后再次运行 `OFFLINE_INSTALL.sh --restart`。 diff --git a/deploy/linux-docker-offline-deerflow-only/docs/13-wheelhouse离线新增依赖与多模型.md b/deploy/linux-docker-offline-deerflow-only/docs/13-wheelhouse离线新增依赖与多模型.md new file mode 100644 index 00000000..311086eb --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/docs/13-wheelhouse离线新增依赖与多模型.md @@ -0,0 +1,98 @@ +# wheelhouse 离线新增依赖与多模型配置 + +## 一、数据库固定使用 aiomysql + +部署脚本生成的连接地址为: + +```text +mysql+aiomysql://用户名:编码后的密码@主机:端口/数据库?charset=utf8mb4 +``` + +镜像和 `source/wheelhouse` 都预置兼容组: + +```text +aiomysql==0.3.2 +PyMySQL==1.2.0 +SQLAlchemy==2.0.51 +``` + +这三个版本必须作为整体保留,避免出现 SQLAlchemy 的 `ping(reconnect)` 适配错误。人工只填写 `MYSQL_HOST`、`MYSQL_PORT`、`MYSQL_DATABASE`、`MYSQL_USER`、`MYSQL_PASSWORD`;不要直接编辑生成的数据库 URL。 + +## 二、以后怎样离线增加 Python 依赖 + +部署目录: + +```text +source/wheelhouse/ +├── requirements.txt +├── README.txt +├── *.whl +└── SHA256SUMS +``` + +后端镜像是 Python 3.12、Linux amd64。新增的 wheel 必须匹配该平台;常见可用标签是 `py3-none-any`、`cp312-manylinux2014_x86_64` 或兼容的 manylinux x86_64。Windows wheel、macOS wheel、ARM64 wheel、源码包 `.tar.gz` 均不能使用。 + +在联网的 Linux amd64/Python 3.12 机器下载示例: + +```bash +mkdir -p new-wheels +python3.12 -m pip download \ + --only-binary=:all: \ + --platform manylinux2014_x86_64 \ + --implementation cp \ + --python-version 312 \ + --abi cp312 \ + --dest new-wheels \ + 'your-package==1.2.3' +``` + +将 `new-wheels/` 中目标包及其全部依赖的 `.whl` 上传到 `source/wheelhouse/`,然后在 `source/wheelhouse/requirements.txt` 增加: + +```text +your-package==1.2.3 +``` + +执行: + +```bash +./OFFLINE_INSTALL.sh --restart +``` + +脚本只使用 `--no-index --find-links /app/wheelhouse`,不会访问互联网。它会计算 `requirements.txt` 和所有 wheel 的指纹;内容未变化时跳过安装,内容变化时安装到持久卷 `deerflow-python-vendor`。重建、停止或重启容器不会丢失已安装依赖,也不会修改基础镜像。 + +如果安装时报 `No matching distribution found`,通常是缺少传递依赖 wheel、wheel 平台不匹配,或 `requirements.txt` 中版本与文件名不一致。补齐后再次执行 `--restart` 即可;安装失败时后端不会被切换到半完成的新进程。 + +## 三、配置多个模型 + +`config/deploy.env` 是唯一配置源。示例文件已经给出三套模型块,使用: + +```dotenv +DEERFLOW_MODEL_COUNT=2 + +DEERFLOW_MODEL_1_NAME=general-model +DEERFLOW_MODEL_1_DISPLAY_NAME=通用模型 +DEERFLOW_MODEL_1_MODEL=provider-real-model-id +DEERFLOW_MODEL_1_BASE_URL=http://10.10.10.25:8000/v1 +DEERFLOW_MODEL_1_API_KEY=replace-with-real-key-1 +DEERFLOW_MODEL_1_SUPPORTS_VISION=false +DEERFLOW_MODEL_1_SUPPORTS_THINKING=false + +DEERFLOW_MODEL_2_NAME=vision-model +DEERFLOW_MODEL_2_DISPLAY_NAME=视觉模型 +DEERFLOW_MODEL_2_MODEL=provider-vision-model-id +DEERFLOW_MODEL_2_BASE_URL=http://10.10.10.26:8000/v1 +DEERFLOW_MODEL_2_API_KEY=replace-with-real-key-2 +DEERFLOW_MODEL_2_SUPPORTS_VISION=true +DEERFLOW_MODEL_2_SUPPORTS_THINKING=false +``` + +规则: + +1. 支持 1~20 个模型,编号必须从 1 连续排列; +2. 第 1 个模型是默认模型; +3. `NAME` 必须唯一;`MODEL` 是上游接口实际接收的 model id; +4. 每个模型可使用不同地址和 API Key; +5. `DISPLAY_NAME` 只影响页面显示; +6. 修改完成后执行 `./OFFLINE_INSTALL.sh --restart` 即可重新生成 `config.yaml` 并生效,无需手工改 YAML。 + +完整的超时、重试、最大 token、温度、视觉、thinking 和 Responses API 字段见 `docs/02-deploy.env配置说明.md`。 diff --git a/deploy/linux-docker-offline-deerflow-only/release-template/OFFLINE_INSTALL.sh b/deploy/linux-docker-offline-deerflow-only/release-template/OFFLINE_INSTALL.sh new file mode 100644 index 00000000..d287d265 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/release-template/OFFLINE_INSTALL.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +BUNDLE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="$BUNDLE_DIR/compose/docker-compose.yml" +GENERATED_ENV="$BUNDLE_DIR/data/generated/compose.env" +DEPLOY_ENV="$BUNDLE_DIR/config/deploy.env" +STATE_DIR="$BUNDLE_DIR/data/state" +IMAGE_LIST="$BUNDLE_DIR/images/deerflow-image-list.txt" +IMAGE_ARCHIVE="$BUNDLE_DIR/images/deerflow-images-linux-amd64.tar.gz" +PROJECT_NAME="cmzs-deerflow" +COMPOSE_CMD=() + +say() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; } +die() { printf '错误: %s\n' "$*" >&2; exit 1; } + +usage() { + cat <<'EOF' +用法: + ./OFFLINE_INSTALL.sh # 等同 --start + ./OFFLINE_INSTALL.sh --start # 按 deploy.env 渲染并启动 + ./OFFLINE_INSTALL.sh --stop # 只停止,不删除容器、镜像、网络或数据卷 + ./OFFLINE_INSTALL.sh --restart # 直接加载 source/ 下最新前后台代码并重启 + ./OFFLINE_INSTALL.sh --status # 查看服务状态与访问地址 + ./OFFLINE_INSTALL.sh --logs [服务名] + ./OFFLINE_INSTALL.sh --office-check # 不连接数据库,检查 Word/Excel/PPT 沙箱能力 +EOF +} + +detect_compose() { + # 目标离线服务器使用独立的 docker-compose 命令;同时保留 Compose v2 + # 插件回退,便于同一部署包在两类 Docker 主机上使用。 + if command -v docker-compose >/dev/null 2>&1; then + COMPOSE_CMD=(docker-compose) + elif docker compose version >/dev/null 2>&1; then + COMPOSE_CMD=(docker compose) + else + die "未安装 docker-compose,也没有 docker compose 插件" + fi + "${COMPOSE_CMD[@]}" version >/dev/null 2>&1 || die "Docker Compose 不可用" +} + +require_tools() { + command -v docker >/dev/null 2>&1 || die "未安装 Docker" + docker info >/dev/null 2>&1 || die "Docker 服务不可用,或当前用户无权访问 Docker" + detect_compose + command -v gzip >/dev/null 2>&1 || die "缺少 gzip" + command -v sha256sum >/dev/null 2>&1 || die "缺少 sha256sum" + command -v curl >/dev/null 2>&1 || die "缺少 curl" +} + +compose() { + [[ -f "$GENERATED_ENV" ]] || die "尚未生成运行配置,请先执行 --start" + "${COMPOSE_CMD[@]}" --project-name "$PROJECT_NAME" --env-file "$GENERATED_ENV" -f "$COMPOSE_FILE" "$@" +} + +images_missing() { + local image + while IFS= read -r image; do + [[ -z "$image" || "$image" == \#* ]] && continue + docker image inspect "$image" >/dev/null 2>&1 || return 0 + done < "$IMAGE_LIST" + return 1 +} + +load_images_if_missing() { + if ! images_missing; then + say "所需 DeerFlow 镜像已存在,跳过校验与 docker load" + return 0 + fi + say "校验 DeerFlow 离线镜像文件" + (cd "$BUNDLE_DIR" && sha256sum -c SHA256SUMS) + say "加载 DeerFlow 前端和后端镜像" + gzip -dc "$IMAGE_ARCHIVE" | docker load +} + +backend_image() { + sed -n '/^[^#[:space:]]/ {p;q;}' "$IMAGE_LIST" +} + +render_runtime() { + mkdir -p "$STATE_DIR" "$BUNDLE_DIR/data/generated" + chmod 0600 "$DEPLOY_ENV" + say "校验 deploy.env 并生成运行配置" + docker run --rm --network none \ + -v "$BUNDLE_DIR:/bundle" \ + "$(backend_image)" python /bundle/scripts/render_runtime.py /bundle +} + +install_wheelhouse() { + [[ -d "$BUNDLE_DIR/source/wheelhouse" ]] || die "缺少 source/wheelhouse" + [[ -f "$BUNDLE_DIR/source/wheelhouse/requirements.txt" ]] || \ + die "缺少 source/wheelhouse/requirements.txt" + say "检查 source/wheelhouse 并离线安装新增 Python 依赖" + compose run --rm --no-deps deerflow-backend \ + /opt/venv/bin/python /app/offline-deploy/install_wheelhouse.py \ + /app/wheelhouse /app/backend/.deer-flow/python-vendor +} + +preflight_mysql() { + say "检查 DeerFlow 外部 MySQL" + docker run --rm --network host \ + --add-host host.docker.internal:host-gateway \ + -v "$BUNDLE_DIR:/bundle:ro" \ + "$(backend_image)" python /bundle/scripts/preflight_mysql.py /bundle/config/deploy.env +} + +generated_value() { + local key="$1" + sed -n "s/^${key}=//p" "$GENERATED_ENV" | tail -n 1 +} + +wait_http() { + local name="$1" url="$2" attempts="${3:-90}" code i + for ((i=1; i<=attempts; i++)); do + code="$(curl -sS -o /dev/null -w '%{http_code}' "$url" 2>/dev/null || true)" + if [[ "$code" =~ ^(200|204|301|302)$ ]]; then + printf '%s 就绪(HTTP %s)\n' "$name" "$code" + return 0 + fi + sleep 3 + done + compose ps || true + compose logs --tail=160 || true + die "$name 启动超时: $url" +} + +start_all() { + [[ -f "$DEPLOY_ENV" ]] || die "缺少 $DEPLOY_ENV" + load_images_if_missing + render_runtime + preflight_mysql + install_wheelhouse + say "启动 DeerFlow(禁止在线拉取和现场构建)" + # 镜像已由 load_images_if_missing 校验并加载。这里不用 Compose v2 专属的 + # `--pull never`,以兼容服务器上的 docker-compose v1。 + compose up -d --no-build + wait_http "DeerFlow 后端" "http://127.0.0.1:$(generated_value BACKEND_PORT)/health" 120 + wait_http "DeerFlow 页面" "http://127.0.0.1:$(generated_value FRONTEND_PORT)/" 60 + say "部署完成" + printf 'DeerFlow: http://%s:%s\n' "$(generated_value PUBLIC_HOST)" "$(generated_value FRONTEND_PORT)" + compose ps +} + +stop_all() { + if [[ ! -f "$GENERATED_ENV" ]]; then + say "没有已生成的运行配置,无需停止" + return 0 + fi + say "只停止容器;保留容器、镜像、网络和全部数据卷" + compose stop +} + +restart_all() { + [[ -f "$DEPLOY_ENV" ]] || die "缺少 $DEPLOY_ENV" + [[ -s "$GENERATED_ENV" ]] || die "缺少运行配置,请先执行 --start" + [[ -d "$BUNDLE_DIR/source/backend/app" ]] || die "缺少 source/backend/app" + [[ -d "$BUNDLE_DIR/source/backend/packages" ]] || die "缺少 source/backend/packages" + [[ -d "$BUNDLE_DIR/source/backend/tests" ]] || die "缺少 source/backend/tests" + [[ -s "$BUNDLE_DIR/source/frontend/dist/index.html" ]] || \ + die "缺少 source/frontend/dist/index.html;请先完成前端 pnpm build" + + # --restart 不做 MySQL 预检,也不 load/build/pull 镜像;但会重新渲染 + # deploy.env,让新增模型和配置修改可随同源码在一次重启中生效。 + render_runtime + install_wheelhouse + + say "检查并加载 source/ 下最新前后台代码" + compose run --rm --no-deps deerflow-backend /opt/venv/bin/python -c \ + 'import app.gateway.app, deerflow; print("后端源码检查通过")' + + say "重建并重启后端容器(保留镜像、数据库和数据卷)" + compose up -d --no-build --no-deps --force-recreate deerflow-backend + wait_http "DeerFlow 后端" \ + "http://127.0.0.1:$(generated_value BACKEND_PORT)/health" 90 + + say "重启前端" + compose up -d --no-build --no-deps --force-recreate deerflow-frontend + wait_http "DeerFlow 页面" \ + "http://127.0.0.1:$(generated_value FRONTEND_PORT)/" 45 + say "最新源码已生效" + compose ps +} + +status_all() { + if [[ ! -f "$GENERATED_ENV" ]]; then + say "尚未执行过 --start" + return 0 + fi + compose ps + printf '\nDeerFlow: http://%s:%s\n' "$(generated_value PUBLIC_HOST)" "$(generated_value FRONTEND_PORT)" +} + +office_check() { + load_images_if_missing + say "检查本地沙箱的 Word / Excel / PowerPoint / PDF 处理能力" + docker run --rm --network none "$(backend_image)" \ + /opt/venv/bin/python /app/backend/scripts/office_runtime_selfcheck.py +} + +main() { + local mode="${1:---start}" + require_tools + case "$mode" in + --start) start_all ;; + --stop) stop_all ;; + --restart) restart_all ;; + --status) status_all ;; + --office-check) office_check ;; + --logs) + shift + if [[ $# -gt 0 ]]; then compose logs --tail=200 -f "$1"; else compose logs --tail=200 -f; fi + ;; + --help|-h) usage ;; + *) usage; die "未知参数: $mode" ;; + esac +} + +main "$@" diff --git a/deploy/linux-docker-offline-deerflow-only/release-template/UPDATE_INSTALL.sh b/deploy/linux-docker-offline-deerflow-only/release-template/UPDATE_INSTALL.sh new file mode 100644 index 00000000..8c0ea29e --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/release-template/UPDATE_INSTALL.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +BUNDLE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BACKUP_ROOT="$BUNDLE_DIR/data/updates/backups" +HISTORY_FILE="$BUNDLE_DIR/data/updates/history.log" + +say() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; } +die() { printf '错误: %s\n' "$*" >&2; exit 1; } + +usage() { + cat <<'EOF' +用法: + ./UPDATE_INSTALL.sh --apply /path/to/deerflow-update-目录 + ./UPDATE_INSTALL.sh --apply /path/to/deerflow-update-目录 --no-start + ./UPDATE_INSTALL.sh --rollback /本部署目录/data/updates/backups/备份目录名 + +更新包必须先解压。脚本不会覆盖 config/deploy.env、data/state、业务数据卷、 +自定义智能体、custom 技能或本地知识文件。 +EOF +} + +current_release() { + sed -n 's/^[[:space:]]*"release":[[:space:]]*"\([^"]*\)".*/\1/p' \ + "$BUNDLE_DIR/VERSION.json" | head -n 1 +} + +read_manifest() { + local manifest="$1" key value + FORMAT_VERSION="" + BASE_RELEASE="" + TARGET_RELEASE="" + TARGET_BACKEND_IMAGE="" + TARGET_FRONTEND_IMAGE="" + while IFS='=' read -r key value; do + [[ -z "$key" || "$key" == \#* ]] && continue + case "$key" in + FORMAT_VERSION|BASE_RELEASE|TARGET_RELEASE|TARGET_BACKEND_IMAGE|TARGET_FRONTEND_IMAGE) ;; + *) die "UPDATE_MANIFEST.env 含未知字段: $key" ;; + esac + [[ "$value" =~ ^[A-Za-z0-9._:/,-]+$ ]] || die "更新清单字段 $key 含非法字符" + printf -v "$key" '%s' "$value" + done < "$manifest" + [[ "$FORMAT_VERSION" == "1" ]] || die "不支持的增量包格式: $FORMAT_VERSION" + [[ -n "$BASE_RELEASE" && -n "$TARGET_RELEASE" ]] || die "更新清单缺少版本字段" + [[ -n "$TARGET_BACKEND_IMAGE" && -n "$TARGET_FRONTEND_IMAGE" ]] || die "更新清单缺少镜像字段" +} + +validate_payload() { + local payload="$1" path relative + [[ -d "$payload" ]] || die "更新包缺少 payload 目录" + if find "$payload" -type l -print -quit | grep -q .; then + die "payload 不允许包含符号链接" + fi + while IFS= read -r -d '' path; do + relative="${path#"$payload"/}" + case "$relative" in + OFFLINE_INSTALL.sh|UPDATE_INSTALL.sh|README_OFFLINE.md|VERSION.json|\ + compose/*|scripts/*|docs/*|config/deerflow/*|config/frontend/*|\ + config/deploy.env.example|config/deploy.env.additions.example|\ + images/deerflow-image-list.txt) ;; + *) die "更新包试图写入受保护路径: $relative" ;; + esac + done < <(find "$payload" -type f -print0) +} + +backup_controls() { + local current="$1" target="$2" backup + backup="$BACKUP_ROOT/${current}_before_${target}_$(date +%Y%m%d-%H%M%S)" + mkdir -p "$backup" + tar -C "$BUNDLE_DIR" -czf "$backup/control-files.tar.gz" \ + VERSION.json README_OFFLINE.md OFFLINE_INSTALL.sh UPDATE_INSTALL.sh \ + compose scripts docs config/deploy.env.example config/deerflow config/frontend \ + images/deerflow-image-list.txt + cp -a "$BUNDLE_DIR/config/deploy.env" "$backup/deploy.env.snapshot" + printf '%s\n' "$backup" +} + +apply_update() { + local update_dir="$1" start_after="$2" current backup image_archive + update_dir="$(realpath "$update_dir")" + [[ -d "$update_dir" ]] || die "更新目录不存在: $update_dir" + [[ -f "$update_dir/UPDATE_MANIFEST.env" ]] || die "缺少 UPDATE_MANIFEST.env" + [[ -f "$update_dir/SHA256SUMS" ]] || die "缺少 SHA256SUMS" + read_manifest "$update_dir/UPDATE_MANIFEST.env" + current="$(current_release)" + [[ "$current" == "$BASE_RELEASE" ]] || die "当前版本 $current 与更新包基线 $BASE_RELEASE 不一致" + + say "校验增量更新包" + (cd "$update_dir" && sha256sum -c SHA256SUMS) + validate_payload "$update_dir/payload" + backup="$(backup_controls "$current" "$TARGET_RELEASE")" + say "控制文件备份:$backup" + + "$BUNDLE_DIR/OFFLINE_INSTALL.sh" --stop + image_archive="$update_dir/images/update-images-linux-amd64.tar.gz" + if [[ -f "$image_archive" ]]; then + say "加载本次变更镜像;保留原镜像用于回退" + gzip -dc "$image_archive" | docker load + fi + docker image inspect "$TARGET_BACKEND_IMAGE" "$TARGET_FRONTEND_IMAGE" >/dev/null \ + || die "目标镜像不完整" + + say "应用白名单内的控制文件;保留 deploy.env 和业务数据" + cp -a "$update_dir/payload/." "$BUNDLE_DIR/" + chmod 0755 "$BUNDLE_DIR/OFFLINE_INSTALL.sh" "$BUNDLE_DIR/UPDATE_INSTALL.sh" + find "$BUNDLE_DIR/scripts" -maxdepth 1 -type f -name '*.py' -exec chmod 0755 {} + + grep -Fq "$TARGET_BACKEND_IMAGE" "$BUNDLE_DIR/VERSION.json" || die "VERSION.json 后端镜像不匹配" + grep -Fq "$TARGET_FRONTEND_IMAGE" "$BUNDLE_DIR/VERSION.json" || die "VERSION.json 前端镜像不匹配" + printf '%s APPLY %s -> %s backup=%s\n' "$(date --iso-8601=seconds)" "$current" "$TARGET_RELEASE" "$backup" \ + >> "$HISTORY_FILE" + + if [[ "$start_after" == "true" ]]; then + "$BUNDLE_DIR/OFFLINE_INSTALL.sh" --start + else + say "增量更新已应用但未启动;确认后执行 ./OFFLINE_INSTALL.sh --start" + fi +} + +rollback_update() { + local backup="$1" resolved current previous + resolved="$(realpath "$backup")" + case "$resolved" in + "$BACKUP_ROOT"/*) ;; + *) die "只能回退本部署目录 data/updates/backups 下的备份" ;; + esac + [[ -f "$resolved/control-files.tar.gz" ]] || die "备份不完整: $resolved" + current="$(current_release)" + "$BUNDLE_DIR/OFFLINE_INSTALL.sh" --stop + tar -xzf "$resolved/control-files.tar.gz" -C "$BUNDLE_DIR" + previous="$(current_release)" + printf '%s ROLLBACK %s -> %s source=%s\n' "$(date --iso-8601=seconds)" "$current" "$previous" "$resolved" \ + >> "$HISTORY_FILE" + say "控制文件和旧镜像引用已恢复;正在启动旧版本" + "$BUNDLE_DIR/OFFLINE_INSTALL.sh" --start +} + +main() { + mkdir -p "$BACKUP_ROOT" + case "${1:-}" in + --apply) + [[ $# -ge 2 && $# -le 3 ]] || { usage; exit 2; } + [[ "${3:-}" == "" || "${3:-}" == "--no-start" ]] || die "未知参数: ${3:-}" + apply_update "$2" "$([[ "${3:-}" == "--no-start" ]] && echo false || echo true)" + ;; + --rollback) + [[ $# -eq 2 ]] || { usage; exit 2; } + rollback_update "$2" + ;; + --help|-h|"") usage ;; + *) usage; die "未知参数: $1" ;; + esac +} + +main "$@" diff --git a/deploy/linux-docker-offline-deerflow-only/release-template/compose/docker-compose.yml b/deploy/linux-docker-offline-deerflow-only/release-template/compose/docker-compose.yml new file mode 100644 index 00000000..57f2222e --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/release-template/compose/docker-compose.yml @@ -0,0 +1,70 @@ +version: "2.4" + +services: + deerflow-backend: + image: ${DEERFLOW_BACKEND_IMAGE} + restart: unless-stopped + env_file: + - ../data/generated/compose.env + environment: + BACKEND_PORT: "8001" + DEER_FLOW_PROJECT_ROOT: /app/backend + DEER_FLOW_HOME: /app/backend/.deer-flow + DEER_FLOW_CONFIG_PATH: /app/backend/config.yaml + DEER_FLOW_EXTENSIONS_CONFIG_PATH: /app/backend/extensions_config.json + DEER_FLOW_SKILLS_PATH: /app/backend/skills + # source/wheelhouse 安装到持久卷;放在最前面可覆盖镜像内的旧依赖。 + PYTHONPATH: /app/backend/.deer-flow/python-vendor:/app/backend:/app/backend/packages/harness + TZ: Asia/Shanghai + ports: + - "${BACKEND_PORT}:8001" + volumes: + # 直接替换宿主机 source/backend 下的源码后,重启容器即可生效。 + - ../source/backend/app:/app/backend/app:ro + - ../source/backend/packages:/app/backend/packages:ro + - ../source/backend/tests:/app/backend/tests:ro + - ../source/wheelhouse:/app/wheelhouse:ro + - ../scripts/install_wheelhouse.py:/app/offline-deploy/install_wheelhouse.py:ro + - ../data/generated/deerflow/config.yaml:/app/backend/config.yaml:ro + - ../config/deerflow/extensions_config.json:/app/backend/extensions_config.json:ro + - ../agents-skills/deerflow-agents:/app/backend/.deer-flow/agents + - ../agents-skills/deerflow-skills:/app/backend/skills + - ../agents-skills/deerflow-knowledge:/app/backend/knowledge + - deerflow-data:/app/backend/.deer-flow/data + - deerflow-browser-context:/app/backend/.deer-flow/browser-context + - deerflow-users:/app/backend/.deer-flow/users + - deerflow-python-vendor:/app/backend/.deer-flow/python-vendor + - /var/run/docker.sock:/var/run/docker.sock + extra_hosts: + - host.docker.internal:host-gateway + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8001/health"] + interval: 20s + timeout: 8s + retries: 10 + start_period: 90s + + deerflow-frontend: + image: ${DEERFLOW_FRONTEND_IMAGE} + restart: unless-stopped + ports: + - "${FRONTEND_PORT}:80" + volumes: + # 前端只接收已经执行 pnpm build 生成的 dist 目录。 + - ../source/frontend/dist:/usr/share/nginx/html:ro + - ../data/generated/frontend/runtime-config.js:/usr/share/nginx/html/runtime-config.js:ro + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1/health"] + interval: 20s + timeout: 5s + retries: 5 + start_period: 10s + depends_on: + deerflow-backend: + condition: service_healthy + +volumes: + deerflow-data: + deerflow-browser-context: + deerflow-users: + deerflow-python-vendor: diff --git a/deploy/linux-docker-offline-deerflow-only/release-template/config/deploy.env b/deploy/linux-docker-offline-deerflow-only/release-template/config/deploy.env new file mode 100644 index 00000000..0614d39e --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/release-template/config/deploy.env @@ -0,0 +1,113 @@ +# ================================================================ +# DeerFlow 纯项目完全离线部署:唯一人工配置文件 +# ================================================================ +# 规则: +# 1. 只修改本文件;data/generated/ 下全部由启动脚本生成。 +# 2. 示例使用 RFC 5737 保留地址,必须替换后才能启动。 +# 3. PUBLIC_HOST 只填 IP/域名;接口地址必须包含 http:// 或 https://。 +# 4. 值中不要包含空格、换行或 $ 字符。 + +# ---------- 全局访问地址与端口 ---------- +PUBLIC_HOST=192.0.2.10 +FRONTEND_PORT=5174 +BACKEND_PORT=8001 + +# ---------- DeerFlow 外部 MySQL ---------- +MYSQL_HOST=192.0.2.20 +MYSQL_PORT=3306 +MYSQL_DATABASE=deerflow +MYSQL_USER=deerflow +MYSQL_PASSWORD=ExampleMysql@123 + +# ---------- DeerFlow 模型列表(1~20 个;第 1 个是默认模型) ---------- +# 只需要一个模型时把 COUNT 改成 1,后面的示例块会被忽略。 +DEERFLOW_MODEL_COUNT=3 + +DEERFLOW_MODEL_1_NAME=qwen3-32b +DEERFLOW_MODEL_1_DISPLAY_NAME=通用模型 +DEERFLOW_MODEL_1_MODEL=qwen3-32b +DEERFLOW_MODEL_1_BASE_URL=http://192.0.2.25:8000/v1 +DEERFLOW_MODEL_1_API_KEY=sk-example-model-1 +DEERFLOW_MODEL_1_SUPPORTS_VISION=false +DEERFLOW_MODEL_1_SUPPORTS_THINKING=false +DEERFLOW_MODEL_1_SUPPORTS_REASONING_EFFORT=false +DEERFLOW_MODEL_1_REQUEST_TIMEOUT=600 +DEERFLOW_MODEL_1_MAX_RETRIES=2 +DEERFLOW_MODEL_1_STREAM_USAGE=true +DEERFLOW_MODEL_1_MAX_TOKENS= +DEERFLOW_MODEL_1_TEMPERATURE= +DEERFLOW_MODEL_1_USE_RESPONSES_API=false + +DEERFLOW_MODEL_2_NAME=vision-model +DEERFLOW_MODEL_2_DISPLAY_NAME=视觉模型 +DEERFLOW_MODEL_2_MODEL=vision-model +DEERFLOW_MODEL_2_BASE_URL=http://192.0.2.25:8001/v1 +DEERFLOW_MODEL_2_API_KEY=sk-example-model-2 +DEERFLOW_MODEL_2_SUPPORTS_VISION=true +DEERFLOW_MODEL_2_SUPPORTS_THINKING=false +DEERFLOW_MODEL_2_SUPPORTS_REASONING_EFFORT=false +DEERFLOW_MODEL_2_REQUEST_TIMEOUT=600 +DEERFLOW_MODEL_2_MAX_RETRIES=2 +DEERFLOW_MODEL_2_STREAM_USAGE=true +DEERFLOW_MODEL_2_MAX_TOKENS=8192 +DEERFLOW_MODEL_2_TEMPERATURE=0.7 +DEERFLOW_MODEL_2_USE_RESPONSES_API=false + +DEERFLOW_MODEL_3_NAME=thinking-model +DEERFLOW_MODEL_3_DISPLAY_NAME=推理模型 +DEERFLOW_MODEL_3_MODEL=thinking-model +DEERFLOW_MODEL_3_BASE_URL=http://192.0.2.25:8002/v1 +DEERFLOW_MODEL_3_API_KEY=sk-example-model-3 +DEERFLOW_MODEL_3_SUPPORTS_VISION=false +DEERFLOW_MODEL_3_SUPPORTS_THINKING=true +DEERFLOW_MODEL_3_SUPPORTS_REASONING_EFFORT=false +DEERFLOW_MODEL_3_REQUEST_TIMEOUT=1200 +DEERFLOW_MODEL_3_MAX_RETRIES=2 +DEERFLOW_MODEL_3_STREAM_USAGE=true +DEERFLOW_MODEL_3_MAX_TOKENS=16384 +DEERFLOW_MODEL_3_TEMPERATURE=0.6 +DEERFLOW_MODEL_3_USE_RESPONSES_API=false + +# ---------- 可选搜索服务(不用就留空) ---------- +TAVILY_API_KEY= +SERPER_API_KEY= + +# ---------- 任务深链后端(默认使用内置模拟数据) ---------- +TASK_DEEPLINK_MOCK_MODE=true +# 示例:http://192.0.2.50:9000/api/task/detail +TASK_COP_DETAIL_URL= +TASK_PURPOSE_DETAIL_URL= +TASK_DELETE_URL= +TASK_TREEMAP_EDIT_URL= +TASK_TREEMAP_NEXT_URL= +TASK_ACTION_DETAIL_URL= +TASK_ACTION_HISTORY_URL= +TASK_DETAIL_3Q_URL= + +# ---------- 前端可选业务系统接口(不用就留空) ---------- +# 示例:http://192.0.2.60:9000/api +VITE_NOTE_API_BASE_URL= +VITE_API_BASE_URL= +VITE_8BF_TASK_API_BASE_URL= +VITE_8BF_ROUNDTABLE_BACK_URL= +VITE_HOST_NAVIGATE_ORIGIN= +VITE_AIPI_CHAT_URL= +VITE_AIPI_GENERATE_CHAT_URL= +VITE_AIPI_TASK_CHAT_URL= +VITE_AIPI_KNOWLEDGE_URL= +VITE_AIPI_B1COH_URL= +VITE_AIPI_B2ACTOR_URL= +VITE_AIPI_KNOWLEDGE_IFRAME_API_URL= +VITE_AIPI_MOPECHAT_URL= +VITE_AIPI_BACKENP_URL= +VITE_NOTEBOOK_API_BASE_URL= +VITE_AIPI_FRONTEND_URL= +VITE_AIPI_CONSUMER_URL= +VITE_AIPI_PRESENT_URL= +VITE_AIPI_KNOWES_URL= +VITE_AUTH_BASE_URL= +VITE_AIPI_PGLINKTO_URL= +VITE_AIPI_XKLINKTO_URL= +VITE_SENTIMENT_AGENT_API_URL= +VITE_SENTIMENT_AGENT_AUTH_USERNAME= +VITE_SENTIMENT_AGENT_AUTH_PASSWORD= diff --git a/deploy/linux-docker-offline-deerflow-only/release-template/config/deploy.env.example b/deploy/linux-docker-offline-deerflow-only/release-template/config/deploy.env.example new file mode 100644 index 00000000..2912c07b --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/release-template/config/deploy.env.example @@ -0,0 +1,98 @@ +# 与 deploy.env 内容相同;用于误修改后的示例恢复。 +# 唯一人工配置文件仍是 config/deploy.env。 + +PUBLIC_HOST=192.0.2.10 +FRONTEND_PORT=5174 +BACKEND_PORT=8001 + +MYSQL_HOST=192.0.2.20 +MYSQL_PORT=3306 +MYSQL_DATABASE=deerflow +MYSQL_USER=deerflow +MYSQL_PASSWORD=ExampleMysql@123 + +DEERFLOW_MODEL_COUNT=3 + +DEERFLOW_MODEL_1_NAME=qwen3-32b +DEERFLOW_MODEL_1_DISPLAY_NAME=通用模型 +DEERFLOW_MODEL_1_MODEL=qwen3-32b +DEERFLOW_MODEL_1_BASE_URL=http://192.0.2.25:8000/v1 +DEERFLOW_MODEL_1_API_KEY=sk-example-model-1 +DEERFLOW_MODEL_1_SUPPORTS_VISION=false +DEERFLOW_MODEL_1_SUPPORTS_THINKING=false +DEERFLOW_MODEL_1_SUPPORTS_REASONING_EFFORT=false +DEERFLOW_MODEL_1_REQUEST_TIMEOUT=600 +DEERFLOW_MODEL_1_MAX_RETRIES=2 +DEERFLOW_MODEL_1_STREAM_USAGE=true +DEERFLOW_MODEL_1_MAX_TOKENS= +DEERFLOW_MODEL_1_TEMPERATURE= +DEERFLOW_MODEL_1_USE_RESPONSES_API=false + +DEERFLOW_MODEL_2_NAME=vision-model +DEERFLOW_MODEL_2_DISPLAY_NAME=视觉模型 +DEERFLOW_MODEL_2_MODEL=vision-model +DEERFLOW_MODEL_2_BASE_URL=http://192.0.2.25:8001/v1 +DEERFLOW_MODEL_2_API_KEY=sk-example-model-2 +DEERFLOW_MODEL_2_SUPPORTS_VISION=true +DEERFLOW_MODEL_2_SUPPORTS_THINKING=false +DEERFLOW_MODEL_2_SUPPORTS_REASONING_EFFORT=false +DEERFLOW_MODEL_2_REQUEST_TIMEOUT=600 +DEERFLOW_MODEL_2_MAX_RETRIES=2 +DEERFLOW_MODEL_2_STREAM_USAGE=true +DEERFLOW_MODEL_2_MAX_TOKENS=8192 +DEERFLOW_MODEL_2_TEMPERATURE=0.7 +DEERFLOW_MODEL_2_USE_RESPONSES_API=false + +DEERFLOW_MODEL_3_NAME=thinking-model +DEERFLOW_MODEL_3_DISPLAY_NAME=推理模型 +DEERFLOW_MODEL_3_MODEL=thinking-model +DEERFLOW_MODEL_3_BASE_URL=http://192.0.2.25:8002/v1 +DEERFLOW_MODEL_3_API_KEY=sk-example-model-3 +DEERFLOW_MODEL_3_SUPPORTS_VISION=false +DEERFLOW_MODEL_3_SUPPORTS_THINKING=true +DEERFLOW_MODEL_3_SUPPORTS_REASONING_EFFORT=false +DEERFLOW_MODEL_3_REQUEST_TIMEOUT=1200 +DEERFLOW_MODEL_3_MAX_RETRIES=2 +DEERFLOW_MODEL_3_STREAM_USAGE=true +DEERFLOW_MODEL_3_MAX_TOKENS=16384 +DEERFLOW_MODEL_3_TEMPERATURE=0.6 +DEERFLOW_MODEL_3_USE_RESPONSES_API=false + +TAVILY_API_KEY= +SERPER_API_KEY= + +TASK_DEEPLINK_MOCK_MODE=true +TASK_COP_DETAIL_URL= +TASK_PURPOSE_DETAIL_URL= +TASK_DELETE_URL= +TASK_TREEMAP_EDIT_URL= +TASK_TREEMAP_NEXT_URL= +TASK_ACTION_DETAIL_URL= +TASK_ACTION_HISTORY_URL= +TASK_DETAIL_3Q_URL= + +VITE_NOTE_API_BASE_URL= +VITE_API_BASE_URL= +VITE_8BF_TASK_API_BASE_URL= +VITE_8BF_ROUNDTABLE_BACK_URL= +VITE_HOST_NAVIGATE_ORIGIN= +VITE_AIPI_CHAT_URL= +VITE_AIPI_GENERATE_CHAT_URL= +VITE_AIPI_TASK_CHAT_URL= +VITE_AIPI_KNOWLEDGE_URL= +VITE_AIPI_B1COH_URL= +VITE_AIPI_B2ACTOR_URL= +VITE_AIPI_KNOWLEDGE_IFRAME_API_URL= +VITE_AIPI_MOPECHAT_URL= +VITE_AIPI_BACKENP_URL= +VITE_NOTEBOOK_API_BASE_URL= +VITE_AIPI_FRONTEND_URL= +VITE_AIPI_CONSUMER_URL= +VITE_AIPI_PRESENT_URL= +VITE_AIPI_KNOWES_URL= +VITE_AUTH_BASE_URL= +VITE_AIPI_PGLINKTO_URL= +VITE_AIPI_XKLINKTO_URL= +VITE_SENTIMENT_AGENT_API_URL= +VITE_SENTIMENT_AGENT_AUTH_USERNAME= +VITE_SENTIMENT_AGENT_AUTH_PASSWORD= diff --git a/deploy/linux-docker-offline-deerflow-only/release-template/scripts/install_wheelhouse.py b/deploy/linux-docker-offline-deerflow-only/release-template/scripts/install_wheelhouse.py new file mode 100644 index 00000000..35b1177b --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/release-template/scripts/install_wheelhouse.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Install operator-selected offline wheels into the persistent vendor path.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import subprocess +import sys +from pathlib import Path + + +def digest_inputs(wheelhouse: Path, requirements: Path) -> str: + digest = hashlib.sha256() + inputs = [requirements, *sorted(wheelhouse.glob("*.whl"), key=lambda path: path.name.lower())] + for path in inputs: + digest.update(path.name.encode("utf-8")) + digest.update(b"\0") + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def has_requirements(path: Path) -> bool: + return any(line.strip() and not line.lstrip().startswith("#") for line in path.read_text(encoding="utf-8-sig").splitlines()) + + +def verify_aiomysql_bundle(vendor: Path, requirements: Path) -> None: + if "aiomysql" not in requirements.read_text(encoding="utf-8-sig").lower(): + return + sys.path.insert(0, str(vendor)) + import aiomysql # noqa: F401 + import pymysql # noqa: F401 + from sqlalchemy.dialects.mysql.aiomysql import AsyncAdapt_aiomysql_connection + + reconnect = inspect.signature(AsyncAdapt_aiomysql_connection.ping).parameters.get("reconnect") + if reconnect is None or reconnect.default is not False: + raise RuntimeError("SQLAlchemy aiomysql 适配器不兼容:ping(reconnect=False) 校验失败") + + +def main() -> int: + wheelhouse = Path(sys.argv[1] if len(sys.argv) > 1 else "/app/wheelhouse").resolve() + vendor = Path( + sys.argv[2] if len(sys.argv) > 2 else "/app/backend/.deer-flow/python-vendor" + ).resolve() + requirements = wheelhouse / "requirements.txt" + if not wheelhouse.is_dir(): + raise SystemExit(f"wheelhouse 目录不存在: {wheelhouse}") + if not requirements.is_file(): + raise SystemExit(f"wheelhouse 缺少 requirements.txt: {requirements}") + if not has_requirements(requirements): + print("wheelhouse/requirements.txt 没有启用的依赖,跳过安装。") + return 0 + + fingerprint = digest_inputs(wheelhouse, requirements) + marker = vendor / ".wheelhouse-state.json" + vendor.mkdir(parents=True, exist_ok=True) + try: + state = json.loads(marker.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError): + state = {} + if state.get("fingerprint") == fingerprint and any(vendor.iterdir()): + verify_aiomysql_bundle(vendor, requirements) + print("wheelhouse 内容未变化,持久依赖已就绪。") + return 0 + + command = [ + sys.executable, + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--no-index", + "--find-links", + str(wheelhouse), + "--target", + str(vendor), + "--upgrade", + "-r", + str(requirements), + ] + print("检测到 wheelhouse 变化,正在从本地 wheel 离线安装……") + subprocess.run(command, check=True) + verify_aiomysql_bundle(vendor, requirements) + marker.write_text( + json.dumps({"format_version": 1, "fingerprint": fingerprint}, ensure_ascii=False, indent=2) + + "\n", + encoding="utf-8", + ) + print("wheelhouse 依赖安装完成。") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/linux-docker-offline-deerflow-only/release-template/scripts/preflight_mysql.py b/deploy/linux-docker-offline-deerflow-only/release-template/scripts/preflight_mysql.py new file mode 100644 index 00000000..f1364689 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/release-template/scripts/preflight_mysql.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Check/create the configured DeerFlow MySQL database before Compose starts.""" + +from __future__ import annotations + +import asyncio +import re +import sys +from pathlib import Path + +import aiomysql + +from render_runtime import load_env + + +async def check(path: Path) -> None: + env = load_env(path) + host = env["MYSQL_HOST"].strip() + port = int(env["MYSQL_PORT"]) + user = env["MYSQL_USER"].strip() + password = env["MYSQL_PASSWORD"] + database = env["MYSQL_DATABASE"].strip() + if not re.fullmatch(r"[A-Za-z0-9_]+", database): + raise RuntimeError("MYSQL_DATABASE 只能包含字母、数字和下划线") + connection = await aiomysql.connect(host=host, port=port, user=user, password=password, autocommit=True) + try: + async with connection.cursor() as cursor: + await cursor.execute( + f"CREATE DATABASE IF NOT EXISTS `{database}` " + "CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci" + ) + await connection.select_db(database) + async with connection.cursor() as cursor: + await cursor.execute("SELECT 1") + assert (await cursor.fetchone())[0] == 1 + finally: + connection.close() + + +def main() -> int: + try: + asyncio.run(check(Path(sys.argv[1]))) + print("DeerFlow 外部 MySQL 连接检查通过。") + return 0 + except Exception: + print("MySQL 预检失败:无法连接或自动创建 DeerFlow 数据库;请检查地址、端口、账号权限和防火墙。", file=sys.stderr) + return 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/linux-docker-offline-deerflow-only/release-template/scripts/render_runtime.py b/deploy/linux-docker-offline-deerflow-only/release-template/scripts/render_runtime.py new file mode 100644 index 00000000..1991c296 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/release-template/scripts/render_runtime.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +"""Validate deploy.env and render the DeerFlow-only runtime configuration.""" + +from __future__ import annotations + +import json +import os +import re +import secrets +import stat +import sys +from pathlib import Path +from urllib.parse import quote, urlsplit + +import yaml + + +DOC_NETS = ("192.0.2.", "198.51.100.", "203.0.113.") +TRUE_VALUES = {"1", "true", "yes", "on"} +FALSE_VALUES = {"0", "false", "no", "off"} +FRONTEND_OPTIONAL_URL_KEYS = ( + "VITE_NOTE_API_BASE_URL", + "VITE_API_BASE_URL", + "VITE_8BF_TASK_API_BASE_URL", + "VITE_8BF_ROUNDTABLE_BACK_URL", + "VITE_HOST_NAVIGATE_ORIGIN", + "VITE_AIPI_CHAT_URL", + "VITE_AIPI_GENERATE_CHAT_URL", + "VITE_AIPI_TASK_CHAT_URL", + "VITE_AIPI_KNOWLEDGE_URL", + "VITE_AIPI_B1COH_URL", + "VITE_AIPI_B2ACTOR_URL", + "VITE_AIPI_KNOWLEDGE_IFRAME_API_URL", + "VITE_AIPI_MOPECHAT_URL", + "VITE_AIPI_BACKENP_URL", + "VITE_NOTEBOOK_API_BASE_URL", + "VITE_AIPI_FRONTEND_URL", + "VITE_AIPI_CONSUMER_URL", + "VITE_AIPI_PRESENT_URL", + "VITE_AIPI_KNOWES_URL", + "VITE_AUTH_BASE_URL", + "VITE_AIPI_PGLINKTO_URL", + "VITE_AIPI_XKLINKTO_URL", + "VITE_SENTIMENT_AGENT_API_URL", +) +TASK_URL_KEYS = { + "cop_task_detail_url": "TASK_COP_DETAIL_URL", + "purpose_detail_url": "TASK_PURPOSE_DETAIL_URL", + "delete_url": "TASK_DELETE_URL", + "treemap_edit_url": "TASK_TREEMAP_EDIT_URL", + "treemap_next_url": "TASK_TREEMAP_NEXT_URL", + "action_detail_url": "TASK_ACTION_DETAIL_URL", + "action_history_url": "TASK_ACTION_HISTORY_URL", + "detail_3q_url": "TASK_DETAIL_3Q_URL", +} + + +def fail(message: str) -> None: + raise ValueError(message) + + +def load_env(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + for lineno, original in enumerate(path.read_text(encoding="utf-8-sig").splitlines(), 1): + line = original.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + fail(f"{path.name}:{lineno} 不是 KEY=VALUE 格式") + key, raw = line.split("=", 1) + key = key.strip() + if not re.fullmatch(r"[A-Z][A-Z0-9_]*", key): + fail(f"{path.name}:{lineno} 配置名不合法: {key!r}") + raw = raw.strip() + if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"'", '"'}: + raw = raw[1:-1] + values[key] = raw + return values + + +def required(env: dict[str, str], key: str) -> str: + value = env.get(key, "").strip() + if not value: + fail(f"{key} 必须填写") + if "\n" in value or "\r" in value or "$" in value or any(ch.isspace() for ch in value): + fail(f"{key} 不能包含空格、换行或 $ 字符") + if "example" in value.lower() or value.startswith(DOC_NETS): + fail(f"{key} 仍是示例值,请按实际环境修改") + return value + + +def optional(env: dict[str, str], key: str) -> str: + value = env.get(key, "").strip() + if "\n" in value or "\r" in value or "$" in value or any(ch.isspace() for ch in value): + fail(f"{key} 不能包含空格、换行或 $ 字符") + if value and ("example" in value.lower() or value.startswith(DOC_NETS)): + fail(f"{key} 仍是示例值,请按实际环境修改或留空") + return value + + +def integer(env: dict[str, str], key: str, minimum: int, maximum: int) -> int: + try: + number = int(required(env, key)) + except ValueError: + fail(f"{key} 必须是整数") + if not minimum <= number <= maximum: + fail(f"{key} 必须在 {minimum}~{maximum} 之间") + return number + + +def boolean(env: dict[str, str], key: str) -> bool: + value = required(env, key).lower() + if value in TRUE_VALUES: + return True + if value in FALSE_VALUES: + return False + fail(f"{key} 必须填写 true 或 false") + + +def optional_boolean(env: dict[str, str], key: str, default: bool = False) -> bool: + value = env.get(key, "").strip() + if not value: + return default + if value.lower() in TRUE_VALUES: + return True + if value.lower() in FALSE_VALUES: + return False + fail(f"{key} 必须填写 true 或 false") + + +def optional_integer( + env: dict[str, str], key: str, minimum: int, maximum: int, default: int | None = None +) -> int | None: + value = env.get(key, "").strip() + if not value: + return default + try: + number = int(value) + except ValueError: + fail(f"{key} 必须是整数") + if not minimum <= number <= maximum: + fail(f"{key} 必须在 {minimum}~{maximum} 之间") + return number + + +def optional_number( + env: dict[str, str], key: str, minimum: float, maximum: float, default: float | None = None +) -> float | None: + value = env.get(key, "").strip() + if not value: + return default + try: + number = float(value) + except ValueError: + fail(f"{key} 必须是数字") + if not minimum <= number <= maximum: + fail(f"{key} 必须在 {minimum}~{maximum} 之间") + return number + + +def service_url(env: dict[str, str], key: str, *, allow_blank: bool = False) -> str: + value = optional(env, key) if allow_blank else required(env, key) + if not value and allow_blank: + return "" + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + fail(f"{key} 必须是完整的 http(s) 地址") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + fail(f"{key} 不能包含账号、密码、查询参数或锚点") + if parsed.hostname.startswith(DOC_NETS): + fail(f"{key} 仍是文档示例地址") + return value.rstrip("/") + + +def render_models(env: dict[str, str]) -> tuple[list[dict[str, object]], dict[str, str]]: + """Render one or more OpenAI-compatible models from deploy.env. + + New bundles use DEERFLOW_MODEL_COUNT plus numbered fields. The legacy + DEERFLOW_LLM_* block remains accepted so an existing deploy.env can be + upgraded without an all-at-once configuration rewrite. + """ + + count_raw = env.get("DEERFLOW_MODEL_COUNT", "").strip() + if not count_raw: + name = required(env, "DEERFLOW_LLM_MODEL") + api_key = required(env, "DEERFLOW_LLM_API_KEY") + return ( + [ + { + "name": name, + "display_name": name, + "use": "langchain_openai:ChatOpenAI", + "model": name, + "api_key": "$DEERFLOW_LLM_API_KEY", + "base_url": service_url(env, "DEERFLOW_LLM_BASE_URL"), + "request_timeout": 600.0, + "max_retries": 2, + "stream_usage": True, + "supports_vision": boolean(env, "DEERFLOW_LLM_SUPPORTS_VISION"), + "supports_thinking": boolean(env, "DEERFLOW_LLM_SUPPORTS_THINKING"), + "supports_reasoning_effort": False, + } + ], + {"DEERFLOW_LLM_API_KEY": api_key}, + ) + + try: + count = int(count_raw) + except ValueError: + fail("DEERFLOW_MODEL_COUNT 必须是整数") + if not 1 <= count <= 20: + fail("DEERFLOW_MODEL_COUNT 必须在 1~20 之间") + + models: list[dict[str, object]] = [] + secrets: dict[str, str] = {} + names: set[str] = set() + for index in range(1, count + 1): + prefix = f"DEERFLOW_MODEL_{index}" + name = required(env, f"{prefix}_NAME") + if name in names: + fail(f"{prefix}_NAME 与前面的模型重名: {name}") + names.add(name) + display_name = env.get(f"{prefix}_DISPLAY_NAME", "").strip() or name + provider_model = env.get(f"{prefix}_MODEL", "").strip() or name + provider = env.get(f"{prefix}_USE", "").strip() or "langchain_openai:ChatOpenAI" + if any(ch.isspace() for ch in provider) or "$" in provider: + fail(f"{prefix}_USE 不能包含空格或 $ 字符") + api_key_name = f"{prefix}_API_KEY" + api_key = required(env, api_key_name) + secrets[api_key_name] = api_key + model: dict[str, object] = { + "name": name, + "display_name": display_name, + "use": provider, + "model": provider_model, + "api_key": f"${api_key_name}", + "base_url": service_url(env, f"{prefix}_BASE_URL"), + "request_timeout": optional_number(env, f"{prefix}_REQUEST_TIMEOUT", 1, 3600, 600.0), + "max_retries": optional_integer(env, f"{prefix}_MAX_RETRIES", 0, 10, 2), + "stream_usage": optional_boolean(env, f"{prefix}_STREAM_USAGE", True), + "supports_vision": optional_boolean(env, f"{prefix}_SUPPORTS_VISION"), + "supports_thinking": optional_boolean(env, f"{prefix}_SUPPORTS_THINKING"), + "supports_reasoning_effort": optional_boolean(env, f"{prefix}_SUPPORTS_REASONING_EFFORT"), + } + max_tokens = optional_integer(env, f"{prefix}_MAX_TOKENS", 1, 1000000) + temperature = optional_number(env, f"{prefix}_TEMPERATURE", 0, 2) + if max_tokens is not None: + model["max_tokens"] = max_tokens + if temperature is not None: + model["temperature"] = temperature + if optional_boolean(env, f"{prefix}_USE_RESPONSES_API"): + model["use_responses_api"] = True + model["output_version"] = "responses/v1" + models.append(model) + return models, secrets + + +def dotenv_text(values: dict[str, object]) -> str: + lines = ["# 自动生成;唯一人工配置源是 config/deploy.env。"] + for key, raw_value in values.items(): + value = "true" if raw_value is True else "false" if raw_value is False else str(raw_value) + if "\n" in value or "\r" in value or "$" in value or any(ch.isspace() for ch in value): + fail(f"自动生成的 {key} 含 Docker env 不支持的字符") + lines.append(f"{key}={value}") + return "\n".join(lines) + "\n" + + +def set_js_property(text: str, key: str, value: str) -> str: + rendered = f" {key}: {json.dumps(value, ensure_ascii=False)}," + pattern = re.compile(rf"^\s*(?://\s*)?{re.escape(key)}\s*:.*$", re.MULTILINE) + if pattern.search(text): + return pattern.sub(rendered, text, count=1) + end = text.rfind("};") + if end < 0: + fail("frontend runtime-config.js 模板格式不正确") + return text[:end] + rendered + "\n" + text[end:] + + +def stable_secrets(path: Path) -> dict[str, str]: + if path.exists(): + data = json.loads(path.read_text(encoding="utf-8")) + else: + data = {"deerflow_auth_jwt_secret": secrets.token_hex(32)} + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) + if set(data) != {"deerflow_auth_jwt_secret"} or len(data["deerflow_auth_jwt_secret"]) != 64: + fail("data/state/runtime-secrets.json 损坏;请从备份恢复") + return data + + +def render(bundle: Path) -> None: + env = load_env(bundle / "config" / "deploy.env") + public_host = required(env, "PUBLIC_HOST") + if "://" in public_host or "/" in public_host: + fail("PUBLIC_HOST 只填 IP 或域名,不要带协议、路径") + frontend_port = integer(env, "FRONTEND_PORT", 1, 65535) + backend_port = integer(env, "BACKEND_PORT", 1, 65535) + if frontend_port == backend_port: + fail("FRONTEND_PORT 与 BACKEND_PORT 不能重复") + + mysql_host = required(env, "MYSQL_HOST") + if mysql_host in {"127.0.0.1", "localhost", "::1"}: + fail("MYSQL_HOST 不能填容器内回环地址;宿主机 MySQL 请填 host.docker.internal 或宿主机内网 IP") + mysql_port = integer(env, "MYSQL_PORT", 1, 65535) + mysql_database = required(env, "MYSQL_DATABASE") + if not re.fullmatch(r"[A-Za-z0-9_]+", mysql_database): + fail("MYSQL_DATABASE 只能包含字母、数字和下划线") + mysql_user = required(env, "MYSQL_USER") + mysql_password = required(env, "MYSQL_PASSWORD") + + models, model_secrets = render_models(env) + tavily_api_key = optional(env, "TAVILY_API_KEY") + serper_api_key = optional(env, "SERPER_API_KEY") + task_mock_mode = boolean(env, "TASK_DEEPLINK_MOCK_MODE") + task_urls = {key: service_url(env, env_key, allow_blank=True) for key, env_key in TASK_URL_KEYS.items()} + frontend_urls = {key: service_url(env, key, allow_blank=True) for key in FRONTEND_OPTIONAL_URL_KEYS} + sentiment_username = optional(env, "VITE_SENTIMENT_AGENT_AUTH_USERNAME") + sentiment_password = optional(env, "VITE_SENTIMENT_AGENT_AUTH_PASSWORD") + + version = json.loads((bundle / "VERSION.json").read_text(encoding="utf-8")) + images = version["images"] + generated = bundle / "data" / "generated" + secrets_data = stable_secrets(bundle / "data" / "state" / "runtime-secrets.json") + mysql_url = ( + "mysql+aiomysql://" + f"{quote(mysql_user, safe='')}:{quote(mysql_password, safe='')}@" + f"{mysql_host}:{mysql_port}/{mysql_database}?charset=utf8mb4" + ) + frontend_url = f"http://{public_host}:{frontend_port}" + backend_url = f"http://{public_host}:{backend_port}" + compose_values: dict[str, object] = { + "COMPOSE_PROJECT_NAME": "cmzs-deerflow", + "DEERFLOW_BACKEND_IMAGE": images["deerflow_backend"], + "DEERFLOW_FRONTEND_IMAGE": images["deerflow_frontend"], + "PUBLIC_HOST": public_host, + "FRONTEND_PORT": frontend_port, + "BACKEND_PORT": backend_port, + "MYSQL_DATABASE_URL": mysql_url, + "GATEWAY_CORS_ORIGINS": frontend_url, + "GATEWAY_ENABLE_DOCS": "false", + "GATEWAY_WORKERS": "1", + "AUTH_JWT_SECRET": secrets_data["deerflow_auth_jwt_secret"], + "DEER_FLOW_OFFLINE_MODE": "true", + "SCHEDULER_INSTALL_DEPS": "false", + "SCHEDULED_TASKS_ENABLED": "true", + "ROUNDTABLE_JOB_GATEWAY": "http://127.0.0.1:8001", + "TAVILY_API_KEY": tavily_api_key, + "SERPER_API_KEY": serper_api_key, + } + compose_values.update(model_secrets) + generated.mkdir(parents=True, exist_ok=True) + compose_env = generated / "compose.env" + compose_env.write_text(dotenv_text(compose_values), encoding="utf-8") + os.chmod(compose_env, stat.S_IRUSR | stat.S_IWUSR) + + config = yaml.safe_load((bundle / "config" / "deerflow" / "config.base.yaml").read_text(encoding="utf-8")) + config.setdefault("gateway", {})["port"] = 8001 + config.setdefault("database", {}).update( + {"backend": "mysql", "mysql_url": "$MYSQL_DATABASE_URL", "sqlite_dir": ".deer-flow/data"} + ) + config["checkpointer"] = {"type": "sqlite", "connection_string": ".deer-flow/data/checkpoints.db"} + config["models"] = models + config["vision"] = {} + config["deep_research"] = {"image_generation": {"enabled": False}} + # 本发行包不包含 WeKnora;显式清空连接并关闭本地 Wiki 同步/向量索引。 + llmwiki = config.setdefault("llmwiki", {}) + llmwiki["weknora"] = {"api_base_url": "", "web_base_url": "", "admin_email": "", "admin_password": ""} + llmwiki["local_wiki_index"] = { + "enabled": False, + "frontend_base_url": frontend_url, + "auto_sync": False, + "embedding": {"enabled": False, "provider": "openai", "model": "", "base_url": "", "api_key": "", "dimensions": 1024}, + } + task_deeplink = config.setdefault("task_deeplink", {}) + for flag in ("test", "delete_test", "action_test", "cop_task_detail_test"): + task_deeplink[flag] = task_mock_mode + task_deeplink.update(task_urls) + deerflow_dir = generated / "deerflow" + deerflow_dir.mkdir(parents=True, exist_ok=True) + (deerflow_dir / "config.yaml").write_text( + yaml.safe_dump(config, allow_unicode=True, sort_keys=False, width=120), encoding="utf-8" + ) + + runtime_js = (bundle / "config" / "frontend" / "runtime-config.base.js").read_text(encoding="utf-8") + runtime_js = set_js_property(runtime_js, "VITE_BACKEND_BASE_URL", backend_url) + runtime_js = set_js_property(runtime_js, "VITE_LANGGRAPH_BASE_URL", f"{backend_url}/api") + runtime_js = set_js_property(runtime_js, "VITE_CANVAS_LANGGRAPH_BASE_URL", f"{backend_url}/api") + runtime_js = set_js_property(runtime_js, "VITE_WEKNORA_BASE_URL", "") + runtime_js = set_js_property(runtime_js, "VITE_USE_REAL_DATA", "false" if task_mock_mode else "true") + runtime_js = set_js_property(runtime_js, "VITE_MOCK_COP_TASK_DETAIL", "true" if task_mock_mode else "false") + runtime_js = set_js_property( + runtime_js, + "VITE_SENTIMENT_AGENT_VISIBLE", + "true" if frontend_urls["VITE_SENTIMENT_AGENT_API_URL"] else "false", + ) + for key, value in frontend_urls.items(): + runtime_js = set_js_property(runtime_js, key, value) + runtime_js = set_js_property(runtime_js, "VITE_SENTIMENT_AGENT_AUTH_USERNAME", sentiment_username) + runtime_js = set_js_property(runtime_js, "VITE_SENTIMENT_AGENT_AUTH_PASSWORD", sentiment_password) + frontend_dir = generated / "frontend" + frontend_dir.mkdir(parents=True, exist_ok=True) + (frontend_dir / "runtime-config.js").write_text(runtime_js, encoding="utf-8") + + print("配置校验通过,纯 DeerFlow 运行时文件已从 config/deploy.env 重新生成。") + print(f"DeerFlow: {frontend_url}") + + +def main() -> int: + try: + render(Path(sys.argv[1] if len(sys.argv) > 1 else "/bundle").resolve()) + return 0 + except Exception as exc: + print(f"配置错误: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/linux-docker-offline-deerflow-only/source-update-template/APPLY_SOURCE_UPDATE.sh b/deploy/linux-docker-offline-deerflow-only/source-update-template/APPLY_SOURCE_UPDATE.sh new file mode 100644 index 00000000..5235b005 --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/source-update-template/APPLY_SOURCE_UPDATE.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +UPDATE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +say() { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; } +die() { printf '错误: %s\n' "$*" >&2; exit 1; } + +usage() { + cat <<'EOF' +用法: + ./APPLY_SOURCE_UPDATE.sh --check /当前DeerFlow部署目录 + ./APPLY_SOURCE_UPDATE.sh --apply /当前DeerFlow部署目录 + ./APPLY_SOURCE_UPDATE.sh --apply /当前DeerFlow部署目录 --no-start + +本脚本只基于服务器已经存在的基线镜像构建源码覆盖层,不联网、不执行 +docker pull,不删除旧镜像、数据卷、deploy.env、自定义智能体、custom 技能或知识文件。 +EOF +} + +current_release() { + sed -n 's/^[[:space:]]*"release":[[:space:]]*"\([^"]*\)".*/\1/p' "$1/VERSION.json" | head -n 1 +} + +read_manifest() { + local manifest="$1" key value + FORMAT_VERSION="" + BASE_RELEASE="" + TARGET_RELEASE="" + BASE_BACKEND_IMAGE="" + BASE_FRONTEND_IMAGE="" + BASE_BACKEND_IMAGE_ID="" + BASE_FRONTEND_IMAGE_ID="" + TARGET_BACKEND_IMAGE="" + TARGET_FRONTEND_IMAGE="" + BACKEND_CHANGED="" + FRONTEND_CHANGED="" + SOURCE_COMMIT="" + SOURCE_DIRTY="" + while IFS='=' read -r key value; do + [[ -z "$key" || "$key" == \#* ]] && continue + case "$key" in + FORMAT_VERSION|BASE_RELEASE|TARGET_RELEASE|BASE_BACKEND_IMAGE|BASE_FRONTEND_IMAGE|\ + BASE_BACKEND_IMAGE_ID|BASE_FRONTEND_IMAGE_ID|TARGET_BACKEND_IMAGE|TARGET_FRONTEND_IMAGE|\ + BACKEND_CHANGED|FRONTEND_CHANGED|SOURCE_COMMIT|SOURCE_DIRTY) ;; + *) die "SOURCE_UPDATE_MANIFEST.env 含未知字段: $key" ;; + esac + [[ "$value" =~ ^[A-Za-z0-9._:/,+-]+$ ]] || die "更新清单字段 $key 含非法字符" + printf -v "$key" '%s' "$value" + done < "$manifest" + [[ "$FORMAT_VERSION" == "1" ]] || die "不支持的源码更新格式: $FORMAT_VERSION" + [[ -n "$BASE_RELEASE" && -n "$TARGET_RELEASE" ]] || die "更新清单缺少版本" + [[ -n "$BASE_BACKEND_IMAGE" && -n "$BASE_FRONTEND_IMAGE" ]] || die "更新清单缺少基线镜像" + [[ -n "$TARGET_BACKEND_IMAGE" && -n "$TARGET_FRONTEND_IMAGE" ]] || die "更新清单缺少目标镜像" + [[ "$BACKEND_CHANGED" =~ ^(true|false)$ && "$FRONTEND_CHANGED" =~ ^(true|false)$ ]] \ + || die "组件变化标记必须是 true 或 false" + [[ "$SOURCE_DIRTY" =~ ^(true|false)$ ]] || die "SOURCE_DIRTY 必须是 true 或 false" +} + +validate_bundle() { + [[ -f "$UPDATE_DIR/SOURCE_UPDATE_MANIFEST.env" ]] || die "缺少 SOURCE_UPDATE_MANIFEST.env" + [[ -f "$UPDATE_DIR/SHA256SUMS" ]] || die "缺少 SHA256SUMS" + if find "$UPDATE_DIR" -type l -print -quit | grep -q .; then + die "源码更新包不允许包含符号链接" + fi + (cd "$UPDATE_DIR" && sha256sum -c SHA256SUMS) + read_manifest "$UPDATE_DIR/SOURCE_UPDATE_MANIFEST.env" + [[ -f "$UPDATE_DIR/payload/VERSION.json" ]] || die "缺少 payload/VERSION.json" + [[ -f "$UPDATE_DIR/payload/images/deerflow-image-list.txt" ]] || die "缺少目标镜像清单" + if [[ "$BACKEND_CHANGED" == "true" ]]; then + [[ -f "$UPDATE_DIR/source/backend/pyproject.toml" ]] || die "缺少后端源码" + [[ -f "$UPDATE_DIR/docker/Dockerfile.backend-source-overlay" ]] || die "缺少后端覆盖 Dockerfile" + fi + if [[ "$FRONTEND_CHANGED" == "true" ]]; then + [[ -s "$UPDATE_DIR/runtime/frontend-dist/index.html" ]] || die "缺少前端编译产物" + [[ -f "$UPDATE_DIR/docker/Dockerfile.frontend-dist-overlay" ]] || die "缺少前端覆盖 Dockerfile" + [[ -f "$UPDATE_DIR/runtime/nginx/deerflow.conf" ]] || die "缺少 Nginx 配置" + fi +} + +image_id() { + docker image inspect -f '{{.Id}}' "$1" 2>/dev/null || true +} + +verify_base() { + local deploy_dir="$1" current actual_backend_id actual_frontend_id + [[ -f "$deploy_dir/VERSION.json" && -x "$deploy_dir/OFFLINE_INSTALL.sh" ]] \ + || die "不是有效的 DeerFlow 部署目录: $deploy_dir" + current="$(current_release "$deploy_dir")" + [[ "$current" == "$BASE_RELEASE" ]] \ + || die "当前版本 $current 与源码更新基线 $BASE_RELEASE 不一致" + actual_backend_id="$(image_id "$BASE_BACKEND_IMAGE")" + actual_frontend_id="$(image_id "$BASE_FRONTEND_IMAGE")" + [[ "$actual_backend_id" == "$BASE_BACKEND_IMAGE_ID" ]] \ + || die "后端基线镜像缺失或镜像 ID 不匹配: $BASE_BACKEND_IMAGE" + [[ "$actual_frontend_id" == "$BASE_FRONTEND_IMAGE_ID" ]] \ + || die "前端基线镜像缺失或镜像 ID 不匹配: $BASE_FRONTEND_IMAGE" +} + +compare_backend_dependencies() { + local cid temp manifest failed=false + temp="$(mktemp -d)" + cid="$(docker create "$BASE_BACKEND_IMAGE")" + for manifest in pyproject.toml uv.lock requirements-offline.txt packages/harness/pyproject.toml; do + mkdir -p "$temp/$(dirname "$manifest")" + if ! docker cp "$cid:/app/backend/$manifest" "$temp/$manifest" >/dev/null \ + || ! cmp -s "$UPDATE_DIR/source/backend/$manifest" "$temp/$manifest"; then + printf '后端依赖清单已变化,不能执行源码覆盖: %s\n' "$manifest" >&2 + failed=true + fi + done + docker rm -f "$cid" >/dev/null 2>&1 || true + rm -rf -- "$temp" + [[ "$failed" == "false" ]] || die "请制作新的完整依赖基线" +} + +build_images() { + if [[ "$BACKEND_CHANGED" == "true" ]]; then + say "基于已保存的后端镜像构建源码覆盖层(全程离线)" + compare_backend_dependencies + docker build --pull=false --platform linux/amd64 --network none --provenance=false \ + --build-arg "BASE_IMAGE=$BASE_BACKEND_IMAGE" \ + -f "$UPDATE_DIR/docker/Dockerfile.backend-source-overlay" \ + -t "$TARGET_BACKEND_IMAGE" "$UPDATE_DIR/source/backend" + [[ "$(docker image inspect -f '{{.Architecture}}' "$TARGET_BACKEND_IMAGE")" == "amd64" ]] \ + || die "新后端镜像不是 linux/amd64" + docker run --rm --network none "$TARGET_BACKEND_IMAGE" python -c \ + 'import app.gateway.app, deerflow; print("backend source overlay smoke passed")' + docker run --rm --network none "$TARGET_BACKEND_IMAGE" \ + /opt/venv/bin/python /app/backend/scripts/office_runtime_selfcheck.py + fi + + if [[ "$FRONTEND_CHANGED" == "true" ]]; then + say "基于已保存的前端镜像替换静态文件(全程离线)" + docker build --pull=false --platform linux/amd64 --network none --provenance=false \ + --build-arg "BASE_IMAGE=$BASE_FRONTEND_IMAGE" \ + -f "$UPDATE_DIR/docker/Dockerfile.frontend-dist-overlay" \ + -t "$TARGET_FRONTEND_IMAGE" "$UPDATE_DIR/runtime" + [[ "$(docker image inspect -f '{{.Architecture}}' "$TARGET_FRONTEND_IMAGE")" == "amd64" ]] \ + || die "新前端镜像不是 linux/amd64" + docker run --rm --network none --entrypoint sh "$TARGET_FRONTEND_IMAGE" \ + -c 'test -s /usr/share/nginx/html/index.html' + fi + + docker image inspect "$TARGET_BACKEND_IMAGE" "$TARGET_FRONTEND_IMAGE" >/dev/null \ + || die "目标源码覆盖镜像不完整" +} + +backup_controls() { + local deploy_dir="$1" backup_root backup + backup_root="$deploy_dir/data/updates/backups" + backup="$backup_root/${BASE_RELEASE}_before_${TARGET_RELEASE}_$(date +%Y%m%d-%H%M%S)" + mkdir -p "$backup" + local items=(VERSION.json config/deerflow config/frontend images/deerflow-image-list.txt) + [[ -f "$deploy_dir/SOURCE_BASELINE.json" ]] && items+=(SOURCE_BASELINE.json) + tar -C "$deploy_dir" -czf "$backup/control-files.tar.gz" "${items[@]}" + cp -a "$deploy_dir/config/deploy.env" "$backup/deploy.env.snapshot" + printf '%s\n' "$backup" +} + +restore_controls() { + local deploy_dir="$1" backup="$2" + tar -xzf "$backup/control-files.tar.gz" -C "$deploy_dir" +} + +apply_update() { + local deploy_dir="$1" start_after="$2" backup history_file + deploy_dir="$(realpath "$deploy_dir")" + verify_base "$deploy_dir" + build_images + backup="$(backup_controls "$deploy_dir")" + history_file="$deploy_dir/data/updates/history.log" + say "控制文件备份:$backup" + "$deploy_dir/OFFLINE_INSTALL.sh" --stop + cp -a "$UPDATE_DIR/payload/." "$deploy_dir/" + printf '%s SOURCE_APPLY %s -> %s commit=%s backup=%s\n' \ + "$(date --iso-8601=seconds)" "$BASE_RELEASE" "$TARGET_RELEASE" "$SOURCE_COMMIT" "$backup" \ + >> "$history_file" + + if [[ "$start_after" == "false" ]]; then + say "源码更新已应用但未启动;确认后执行 ./OFFLINE_INSTALL.sh --start" + return + fi + if ! "$deploy_dir/OFFLINE_INSTALL.sh" --start; then + say "新版本启动失败,自动恢复旧控制文件并尝试启动旧版本" + restore_controls "$deploy_dir" "$backup" + "$deploy_dir/OFFLINE_INSTALL.sh" --start || true + die "源码更新启动失败;旧镜像仍保留,备份位于 $backup" + fi + say "源码增量更新完成:$BASE_RELEASE -> $TARGET_RELEASE" +} + +main() { + local action="${1:-}" deploy_dir="${2:-}" start_after=true + [[ "$action" == "--check" || "$action" == "--apply" ]] || { usage; exit 2; } + [[ -n "$deploy_dir" ]] || { usage; exit 2; } + [[ $# -le 3 ]] || { usage; exit 2; } + [[ "${3:-}" == "" || "${3:-}" == "--no-start" ]] || die "未知参数: ${3:-}" + [[ "${3:-}" != "--no-start" ]] || start_after=false + validate_bundle + deploy_dir="$(realpath "$deploy_dir")" + verify_base "$deploy_dir" + if [[ "$action" == "--check" ]]; then + say "源码更新包、版本链和基线镜像校验通过" + exit 0 + fi + apply_update "$deploy_dir" "$start_after" +} + +main "$@" diff --git a/deploy/linux-docker-offline-deerflow-only/test_release_renderer.py b/deploy/linux-docker-offline-deerflow-only/test_release_renderer.py new file mode 100644 index 00000000..284adb8f --- /dev/null +++ b/deploy/linux-docker-offline-deerflow-only/test_release_renderer.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Static QA for the pure DeerFlow release sanitizer and renderer.""" + +from __future__ import annotations + +import json +import shutil +import sys +import tempfile +from pathlib import Path + +import yaml + + +def main() -> int: + workspace = Path(sys.argv[1] if len(sys.argv) > 1 else ".").resolve() + common = workspace / "deploy" / "linux-docker-offline" + pure = workspace / "deploy" / "linux-docker-offline-deerflow-only" + sys.path.insert(0, str(common)) + from sanitize_release_sources import sanitize_backend, sanitize_frontend + + with tempfile.TemporaryDirectory(prefix="deerflow-only-release-qa-") as temporary: + root = Path(temporary) + bundle = root / "bundle" + shutil.copytree(pure / "release-template", bundle) + + backend = root / "backend" + backend.mkdir() + source_backend = workspace / "offline-backend-20260512" / "backend" + shutil.copy2(source_backend / "config.yaml", backend / "config.yaml") + shutil.copy2(source_backend / "extensions_config.json", backend / "extensions_config.json") + sanitize_backend(backend) + + frontend = root / "frontend" + (frontend / "public").mkdir(parents=True) + source_frontend = workspace / "frontend-web" / "public" / "runtime-config.js" + shutil.copy2(source_frontend, frontend / "public" / "runtime-config.js") + sanitize_frontend(frontend) + sanitized_runtime = (frontend / "public" / "runtime-config.js").read_text(encoding="utf-8") + assert "ch1.b.uat.4.cn" not in sanitized_runtime + assert "47.88.25.99" not in sanitized_runtime + assert "gw.y.sit.4.cn" not in sanitized_runtime + + (bundle / "config" / "deerflow").mkdir(parents=True) + (bundle / "config" / "frontend").mkdir(parents=True) + shutil.copy2(backend / "config.yaml", bundle / "config" / "deerflow" / "config.base.yaml") + shutil.copy2( + backend / "extensions_config.json", + bundle / "config" / "deerflow" / "extensions_config.json", + ) + shutil.copy2( + frontend / "public" / "runtime-config.js", + bundle / "config" / "frontend" / "runtime-config.base.js", + ) + (bundle / "VERSION.json").write_text( + json.dumps( + { + "weknora_included": False, + "images": { + "deerflow_backend": "qa/deerflow-backend:1", + "deerflow_frontend": "qa/deerflow-frontend:1", + }, + } + ), + encoding="utf-8", + ) + + env_path = bundle / "config" / "deploy.env" + env_text = env_path.read_text(encoding="utf-8") + env_text = env_text.replace("192.0.2.10", "10.20.30.10") + env_text = env_text.replace("192.0.2.20", "10.20.30.20") + env_text = env_text.replace("192.0.2.25", "10.20.30.25") + env_text = env_text.replace("sk-example-model-", "qa-model-key-") + env_text = env_text.replace("ExampleMysql@123", "QaMysql@123") + env_path.write_text(env_text, encoding="utf-8") + + sys.path.insert(0, str(bundle / "scripts")) + from render_runtime import render + + render(bundle) + generated = bundle / "data" / "generated" + config = yaml.safe_load((generated / "deerflow" / "config.yaml").read_text(encoding="utf-8")) + assert len(config["models"]) == 3 + assert config["uploads"]["auto_convert_documents"] is True + assert config["models"][0]["api_key"] == "$DEERFLOW_MODEL_1_API_KEY" + assert config["models"][1]["supports_vision"] is True + assert config["models"][2]["supports_thinking"] is True + assert config["database"]["backend"] == "mysql" + assert config["llmwiki"]["weknora"]["api_base_url"] == "" + assert config["llmwiki"]["local_wiki_index"]["enabled"] is False + runtime = (generated / "frontend" / "runtime-config.js").read_text(encoding="utf-8") + assert 'VITE_BACKEND_BASE_URL: "http://10.20.30.10:8001"' in runtime + assert 'VITE_LANGGRAPH_BASE_URL: "http://10.20.30.10:8001/api"' in runtime + assert 'VITE_CANVAS_LANGGRAPH_BASE_URL: "http://10.20.30.10:8001/api"' in runtime + assert 'VITE_USE_REAL_DATA: "false"' in runtime + assert 'VITE_SENTIMENT_AGENT_AUTH_PASSWORD: ""' in runtime + assert 'VITE_SENTIMENT_AGENT_VISIBLE: "false"' in runtime + compose_env = (generated / "compose.env").read_text(encoding="utf-8") + assert "MYSQL_DATABASE_URL=mysql+aiomysql://" in compose_env + assert "DEERFLOW_MODEL_1_API_KEY=qa-model-key-1" in compose_env + assert "DEERFLOW_MODEL_2_API_KEY=qa-model-key-2" in compose_env + assert "DEERFLOW_MODEL_3_API_KEY=qa-model-key-3" in compose_env + assert "WEKNORA" not in compose_env.upper() + + install_script = (bundle / "OFFLINE_INSTALL.sh").read_text(encoding="utf-8") + assert "command -v docker-compose" in install_script + assert "--office-check" in install_script + compose = yaml.safe_load((bundle / "compose" / "docker-compose.yml").read_text(encoding="utf-8")) + assert str(compose["version"]) == "2.4" + backend_volumes = compose["services"]["deerflow-backend"]["volumes"] + frontend_volumes = compose["services"]["deerflow-frontend"]["volumes"] + assert "../source/backend/app:/app/backend/app:ro" in backend_volumes + assert "../source/backend/packages:/app/backend/packages:ro" in backend_volumes + assert "../source/backend/tests:/app/backend/tests:ro" in backend_volumes + assert "../source/wheelhouse:/app/wheelhouse:ro" in backend_volumes + assert "deerflow-python-vendor:/app/backend/.deer-flow/python-vendor" in backend_volumes + assert "../source/frontend/dist:/usr/share/nginx/html:ro" in frontend_volumes + assert "deerflow-python-vendor" in compose["volumes"] + assert (bundle / "scripts" / "install_wheelhouse.py").is_file() + assert (bundle / "source" / "wheelhouse" / "requirements.txt").is_file() + assert (bundle / "UPDATE_INSTALL.sh").is_file() + + print("pure DeerFlow release sanitizer and renderer QA passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/deploy/linux-docker-offline/Dockerfile.backend b/deploy/linux-docker-offline/Dockerfile.backend new file mode 100644 index 00000000..6b3ca0fa --- /dev/null +++ b/deploy/linux-docker-offline/Dockerfile.backend @@ -0,0 +1,132 @@ +ARG PYTHON_IMAGE=python:3.12-slim-bookworm + +FROM ${PYTHON_IMAGE} AS builder + +ARG APT_MIRROR=mirrors.aliyun.com +ARG UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple/ +ARG UV_VERSION=0.7.20 +ARG UV_NO_CACHE=1 +ARG UV_COMPILE_BYTECODE=0 + +ENV UV_PROJECT_ENVIRONMENT=/opt/venv \ + UV_INDEX_URL=${UV_INDEX_URL} \ + UV_LINK_MODE=copy \ + UV_NO_CACHE=${UV_NO_CACHE} \ + UV_COMPILE_BYTECODE=${UV_COMPILE_BYTECODE} \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /build/backend + +RUN if [ -n "${APT_MIRROR}" ]; then \ + sed -i "s|deb.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list.d/debian.sources 2>/dev/null || true; \ + sed -i "s|security.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list.d/debian.sources 2>/dev/null || true; \ + sed -i "s|deb.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list 2>/dev/null || true; \ + sed -i "s|security.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list 2>/dev/null || true; \ + fi \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* \ + && python -m pip install --no-cache-dir -i "${UV_INDEX_URL}" "uv==${UV_VERSION}" + +ARG UV_HTTP_TIMEOUT=300 +ARG UV_CONCURRENT_DOWNLOADS=4 +ARG UV_CONCURRENT_INSTALLS=4 +ARG UV_CONCURRENT_BUILDS=2 + +ENV UV_HTTP_TIMEOUT=${UV_HTTP_TIMEOUT} \ + UV_CONCURRENT_DOWNLOADS=${UV_CONCURRENT_DOWNLOADS} \ + UV_CONCURRENT_INSTALLS=${UV_CONCURRENT_INSTALLS} \ + UV_CONCURRENT_BUILDS=${UV_CONCURRENT_BUILDS} + +COPY pyproject.toml uv.lock README.md requirements-offline.txt ./ +COPY packages ./packages + +RUN python -m venv /opt/venv \ + && /opt/venv/bin/python -m pip install --no-cache-dir --upgrade -i "${UV_INDEX_URL}" pip setuptools wheel \ + && sed -E '/--hash=/d; s/[[:space:]]+\\$//; s|^-e[[:space:]]+./packages/harness$|./packages/harness|' requirements-offline.txt > /tmp/requirements-docker.txt \ + && /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" -r /tmp/requirements-docker.txt \ + && /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" \ + "aiomysql==0.3.2" \ + "PyMySQL==1.2.0" \ + "SQLAlchemy==2.0.51" \ + "python-docx==1.2.0" \ + && /opt/venv/bin/python -c 'import aiomysql, docx, lxml, mammoth, markitdown, openpyxl, pandas, pptx, pymysql, sqlalchemy, xlrd; print("aiomysql and Office Python dependencies OK")' + +COPY . . + +# Only runtime material is copied into the final image. Development env files, +# tests and repository documentation are deliberately excluded from that copy. +RUN find /build/backend -maxdepth 2 -type f -name '.env*' -delete \ + && rm -rf \ + /build/backend/.pytest_cache \ + /build/backend/tests \ + /build/backend/packages/harness/tests \ + /build/backend/docs \ + /build/backend/CLAUDE.md \ + /build/backend/AGENTS.md \ + /build/backend/README.md + +FROM ${PYTHON_IMAGE} AS runtime + +ARG APT_MIRROR=mirrors.aliyun.com + +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONPATH=/app/backend:/app/backend/packages/harness \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + DEER_FLOW_PROJECT_ROOT=/app/backend \ + DEER_FLOW_HOME=/app/backend/.deer-flow \ + DEER_FLOW_CONFIG_PATH=/app/backend/config.yaml \ + DEER_FLOW_EXTENSIONS_CONFIG_PATH=/app/backend/extensions_config.json \ + DEER_FLOW_SKILLS_PATH=/app/backend/skills \ + BACKEND_PORT=8001 + +WORKDIR /app/backend + +RUN if [ -n "${APT_MIRROR}" ]; then \ + sed -i "s|deb.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list.d/debian.sources 2>/dev/null || true; \ + sed -i "s|security.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list.d/debian.sources 2>/dev/null || true; \ + sed -i "s|deb.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list 2>/dev/null || true; \ + sed -i "s|security.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list 2>/dev/null || true; \ + fi \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + docker.io \ + file \ + fontconfig \ + fonts-liberation \ + fonts-noto-cjk \ + libmagic1 \ + libreoffice-calc \ + libreoffice-impress \ + libreoffice-writer \ + nodejs \ + npm \ + pandoc \ + poppler-utils \ + tini \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /opt/venv /opt/venv +COPY --from=builder /build/backend /app/backend + +# Fail the image build if the local sandbox cannot process the default Office +# formats. The production deployment uses LocalSandboxProvider, so these are +# the exact Python modules and native tools available to agent bash commands. +RUN /opt/venv/bin/python -c 'import aiomysql, docx, lxml, mammoth, markitdown, openpyxl, pandas, pptx, pymysql, sqlalchemy, xlrd' \ + && command -v soffice >/dev/null \ + && command -v pandoc >/dev/null \ + && fc-list :lang=zh | grep -q . + +EXPOSE 8001 + +ENTRYPOINT ["tini", "--"] +CMD ["sh", "-c", "cd /app/backend && exec uvicorn app.gateway.app:app --host 0.0.0.0 --port ${BACKEND_PORT:-8001}"] diff --git a/deploy/linux-docker-offline/Dockerfile.frontend b/deploy/linux-docker-offline/Dockerfile.frontend new file mode 100644 index 00000000..cfb3477b --- /dev/null +++ b/deploy/linux-docker-offline/Dockerfile.frontend @@ -0,0 +1,24 @@ +FROM node:22-bookworm-slim AS builder + +WORKDIR /workspace/frontend-web + +RUN corepack enable && corepack prepare pnpm@10.26.2 --activate + +COPY frontend-web/package.json frontend-web/pnpm-lock.yaml ./ +RUN --mount=type=cache,target=/root/.local/share/pnpm/store \ + pnpm install --frozen-lockfile + +COPY frontend-web/ ./ + +ENV VITE_ASSET_BASE=/ +RUN pnpm build + +FROM nginx:1.28-alpine + +COPY deploy/linux-docker-offline/nginx/deerflow.conf /etc/nginx/conf.d/default.conf +COPY --from=builder /workspace/frontend-web/dist /usr/share/nginx/html + +EXPOSE 80 + +HEALTHCHECK --interval=20s --timeout=5s --start-period=10s --retries=5 \ + CMD wget -q -O /dev/null http://127.0.0.1/ || exit 1 diff --git a/deploy/linux-docker-offline/sanitize_release_sources.py b/deploy/linux-docker-offline/sanitize_release_sources.py new file mode 100644 index 00000000..3804ab31 --- /dev/null +++ b/deploy/linux-docker-offline/sanitize_release_sources.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Create secret-free build inputs for the offline release. + +The customized working tree contains environment-specific development values. +They must never become defaults in an exported image or release template. +""" + +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +import yaml + + +SECRET_KEY = re.compile( + r"(?:api[_-]?key|password|passwd|secret|access[_-]?token|auth[_-]?(?:token|password))$", + re.IGNORECASE, +) +URL_KEY = re.compile(r"(?:^|_)(?:url|uri|endpoint)$|base_url$", re.IGNORECASE) +SUSPICIOUS_VALUE = re.compile( + r"(?:\bsk-[A-Za-z0-9_.-]{12,}|\bBearer\s+[A-Za-z0-9_.-]{12,})" +) + + +def scrub_mapping(value: object) -> object: + if isinstance(value, dict): + result: dict[object, object] = {} + for key, child in value.items(): + key_text = str(key) + # Names such as ``require_password`` are boolean feature flags, + # not credentials. Only redact scalar string/null secret values. + if SECRET_KEY.search(key_text) and (isinstance(child, str) or child is None): + result[key] = "" + elif URL_KEY.search(key_text) and (isinstance(child, str) or child is None): + result[key] = "" + else: + result[key] = scrub_mapping(child) + return result + if isinstance(value, list): + return [scrub_mapping(item) for item in value] + if isinstance(value, str) and SUSPICIOUS_VALUE.search(value): + return "" + return value + + +def sanitize_backend(root: Path) -> None: + config_path = root / "config.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8-sig")) + config = scrub_mapping(config) + if not isinstance(config, dict): + raise RuntimeError("DeerFlow config.yaml must contain a mapping") + + # The renderer recreates these sections exclusively from deploy.env. + config["models"] = [] + config["vision"] = {} + config["deep_research"] = {"image_generation": {"enabled": False}} + config["database"] = { + "backend": "sqlite", + "sqlite_dir": ".deer-flow/data", + "mysql_url": "", + } + config["checkpointer"] = { + "type": "sqlite", + "connection_string": ".deer-flow/data/checkpoints.db", + } + config["llmwiki"] = { + "weknora": { + "api_base_url": "", + "web_base_url": "", + "admin_email": "", + "admin_password": "", + }, + "local_wiki_index": {"enabled": False, "auto_sync": False}, + } + + task = config.setdefault("task_deeplink", {}) + if isinstance(task, dict): + for flag in ("test", "delete_test", "action_test", "cop_task_detail_test"): + task[flag] = True + for key in tuple(task): + if str(key).endswith("_url"): + task[key] = "" + + memory = config.get("memory") + if isinstance(memory, dict) and isinstance(memory.get("hindsight"), dict): + memory["hindsight"]["api_url"] = "" + memory["hindsight"]["api_key"] = "" + + rendered = yaml.safe_dump(config, allow_unicode=True, sort_keys=False, width=120) + if SUSPICIOUS_VALUE.search(rendered): + raise RuntimeError("sanitized DeerFlow config still contains a credential-like value") + config_path.write_text(rendered, encoding="utf-8") + + extensions_path = root / "extensions_config.json" + extensions = json.loads(extensions_path.read_text(encoding="utf-8-sig")) + extensions_path.write_text( + json.dumps(scrub_mapping(extensions), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + for env_path in root.glob(".env*"): + if env_path.is_file(): + env_path.unlink() + + +def sanitize_frontend(root: Path) -> None: + for env_path in root.glob(".env*"): + if env_path.is_file(): + env_path.unlink() + + runtime_path = root / "public" / "runtime-config.js" + text = runtime_path.read_text(encoding="utf-8-sig") + property_pattern = re.compile( + r'^(?P\s*)(?P//\s*)?(?PVITE_[A-Z0-9_]+)\s*:\s*(?P["\'])(?P.*?)(?P=quote)(?P\s*,?\s*)$', + re.MULTILINE, + ) + + def replace(match: re.Match[str]) -> str: + key = match.group("key") + value = match.group("value") + sensitive_key = re.search( + r"(?:URL|URI|HOST|ENDPOINT|PASSWORD|USERNAME|TOKEN|SECRET|AUTH)", key + ) + absolute_value = re.match(r"^(?:https?|wss?)://", value, re.IGNORECASE) + credential_value = SUSPICIOUS_VALUE.search(value) + safe_brand_asset = key.startswith("VITE_BRAND_") and bool(value) and not absolute_value + if (sensitive_key and not safe_brand_asset) or absolute_value or credential_value: + value = "" + quote = match.group("quote") + comment = match.group("comment") or "" + return f'{match.group("prefix")}{comment}{key}: {quote}{value}{quote}{match.group("comma")}' + + text = property_pattern.sub(replace, text) + for match in property_pattern.finditer(text): + value = match.group("value") + if re.match(r"^(?:https?|wss?)://", value, re.IGNORECASE) or SUSPICIOUS_VALUE.search(value): + raise RuntimeError(f"sanitized runtime-config still contains a configured value: {match.group('key')}") + runtime_path.write_text(text, encoding="utf-8") + + +def main() -> int: + if len(sys.argv) != 3 or sys.argv[1] not in {"backend", "frontend"}: + print("usage: sanitize_release_sources.py backend|frontend PATH", file=sys.stderr) + return 2 + root = Path(sys.argv[2]).resolve() + if sys.argv[1] == "backend": + sanitize_backend(root) + else: + sanitize_frontend(root) + print(f"sanitized {sys.argv[1]} release source: {root}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/DeerFlow纯离线标准发版与部署.md b/docs/DeerFlow纯离线标准发版与部署.md new file mode 100644 index 00000000..542f255d --- /dev/null +++ b/docs/DeerFlow纯离线标准发版与部署.md @@ -0,0 +1,218 @@ +# DeerFlow 纯离线标准发版与部署 + +本文定义本仓库后续唯一的 Linux AMD64 离线发版方式。每次发版都必须从最新 +`master` 构建,使用 +`deploy/linux-docker-offline-deerflow-only/build-release.sh`,最终包只包含 +DeerFlow 后端和前端两张镜像,不包含 WeKnora 服务或镜像。 + +## 1. 标准能力基线 + +标准离线包必须同时具备以下能力: + +- `config/deploy.env` 是唯一需要人工修改的配置源;生成文件不得手工修改。 +- MySQL 使用 SQLAlchemy 异步 URL `mysql+aiomysql://`。 +- 基础兼容版本为 `aiomysql 0.3.2`、`PyMySQL 1.2.0`、 + `SQLAlchemy 2.0.51`。 +- `deploy.env` 支持 1~20 个模型;编号为 1 的模型是默认模型。 +- `source/wheelhouse` 支持完全离线增加 Python 依赖,安装结果保存在 Docker + named volume 中。 +- 后端镜像内置 DOC/DOCX、XLS/XLSX、PPT/PPTX、PDF 处理能力,以及 + LibreOffice、Pandoc 和中文字体。 +- `OFFLINE_INSTALL.sh --stop` 只停止容器,不删除已加载镜像、数据目录或 + named volume。 +- `OFFLINE_INSTALL.sh --restart` 不重新 load 镜像;它重新渲染配置、检测 + wheelhouse、校验源码并重建容器,使覆盖后的前后端源码生效。 +- 启动脚本兼容 `docker-compose` 和 `docker compose`。 + +## 2. 仓库中的标准目录 + +| 路径 | 用途 | +|---|---| +| `deploy/linux-docker-offline-deerflow-only/` | 标准构建脚本、运行模板、测试和交付文档 | +| `deploy/linux-docker-offline/Dockerfile.backend` | 后端基础镜像构建文件 | +| `deploy/linux-docker-offline/Dockerfile.frontend` | 前端基础镜像构建文件 | +| `deploy/linux-docker-offline/sanitize_release_sources.py` | 发布源码清理器 | +| `offline-backend-20260512/wheelhouse/` | 随包携带的 Linux AMD64 / Python 3.12 基础 wheel | +| `offline-backend-20260512/backend/` | 后端源码 | +| `frontend-web/` | 前端源码 | + +不要使用历史的 DeerFlow + WeKnora 全量部署模板发本项目的标准包。 + +## 3. 每次发版前更新源码 + +发版工作区必须没有未提交的业务源码修改。未跟踪的本地构建产物不得混入包中。 + +```bash +git switch master +git fetch origin master +git merge --ff-only origin/master +git status --short --branch +git log -1 --format='%H %ad %s' --date=iso-strict +``` + +必须记录最终提交的完整 SHA。构建服务器如果使用不带 `.git` 的源码快照,应通过 +`DEERFLOW_COMMIT_OVERRIDE` 写入同一个 SHA。 + +## 4. wheelhouse 维护 + +基础 wheel 位于 `offline-backend-20260512/wheelhouse/`。最终包构建时会把它们 +复制到 `source/wheelhouse/`,并保留运行模板中的 `requirements.txt` 和说明文件。 + +增加依赖时: + +1. 下载与 Linux AMD64、Python 3.12 匹配的 wheel,以及它的全部传递依赖。 +2. 把 wheel 放入 `offline-backend-20260512/wheelhouse/`。 +3. 在 + `deploy/linux-docker-offline-deerflow-only/release-template/source/wheelhouse/requirements.txt` + 中加入精确版本,例如 `some-package==1.2.3`。 +4. 重新生成并核对 `offline-backend-20260512/wheelhouse/SHA256SUMS`。 +5. 运行发布渲染测试,再构建完整包。 + +不能放源码包、Windows wheel、macOS wheel,不能只放顶层依赖而遗漏传递依赖。 + +部署完成后临时增加依赖时,也可以直接把 wheel 放进已解压交付目录的 +`source/wheelhouse/`,更新其中的 `requirements.txt`,然后执行: + +```bash +./OFFLINE_INSTALL.sh --restart +``` + +安装器按 `requirements.txt + wheel 文件` 的哈希判断是否需要重新安装,全程使用 +`--no-index`,不会访问互联网。 + +## 5. 构建标准完整包 + +推荐在联网的 Linux AMD64 Docker 构建服务器执行。以下示例把构建工作区和输出 +放在 `/opt/codex-build`,不会修改现网部署目录: + +```bash +RELEASE_STAMP="$(date +%Y%m%d-%H%M%S)" +COMMIT_SHA="$(git rev-parse HEAD)" +SHORT_SHA="$(git rev-parse --short=8 HEAD)" +BUILD_ROOT="/opt/codex-build/deerflow-full-${RELEASE_STAMP}-${SHORT_SHA}" + +RELEASE_STAMP="$RELEASE_STAMP" \ +DEERFLOW_COMMIT_OVERRIDE="$COMMIT_SHA" \ +DEERFLOW_DIRTY_OVERRIDE=false \ +bash deploy/linux-docker-offline-deerflow-only/build-release.sh \ + "$(pwd)" "$BUILD_ROOT" +``` + +不要设置 `DEERFLOW_FRONTEND_RUNTIME_BASE` 来复用旧前端镜像,除非本次只修改 +`runtime-config.js`。从新 `master` 正式发版时必须从当前前端源码完整构建。 + +输出文件为: + +```text +${BUILD_ROOT}/dist/deerflow-linux-amd64-offline-${RELEASE_STAMP}.tar.gz +${BUILD_ROOT}/dist/deerflow-linux-amd64-offline-${RELEASE_STAMP}.tar.gz.sha256 +``` + +## 6. 发版前强制校验 + +```bash +python3 -m py_compile \ + deploy/linux-docker-offline-deerflow-only/release-template/scripts/render_runtime.py \ + deploy/linux-docker-offline-deerflow-only/release-template/scripts/preflight_mysql.py \ + deploy/linux-docker-offline-deerflow-only/release-template/scripts/install_wheelhouse.py + +python3 deploy/linux-docker-offline-deerflow-only/test_release_renderer.py "$(pwd)" +bash -n deploy/linux-docker-offline-deerflow-only/build-release.sh + +cd "${BUILD_ROOT}/dist" +sha256sum -c "deerflow-linux-amd64-offline-${RELEASE_STAMP}.tar.gz.sha256" +gzip -t "deerflow-linux-amd64-offline-${RELEASE_STAMP}.tar.gz" +``` + +还必须确认: + +- `images/deerflow-image-list.txt` 恰好只有 DeerFlow 后端、前端两行。 +- `VERSION.json` 的提交 SHA 与本次 `master` 一致。 +- 包含 `source/wheelhouse`、`scripts/install_wheelhouse.py` 和第 13 章文档。 +- 后端镜像能导入 `aiomysql`、`PyMySQL`、`SQLAlchemy`。 +- 后端镜像 Office 自检输出 `OFFICE_RUNTIME_SELF_CHECK=PASS`。 +- 配置渲染后数据库驱动为 `mysql+aiomysql`。 + +## 7. 新机器完全离线部署 + +把 `.tar.gz` 和 `.sha256` 一起复制到目标服务器后执行: + +```bash +sha256sum -c deerflow-linux-amd64-offline-*.tar.gz.sha256 +tar -xzf deerflow-linux-amd64-offline-*.tar.gz +cd deerflow-linux-amd64-offline-* +cp config/deploy.env.example config/deploy.env +vi config/deploy.env +./OFFLINE_INSTALL.sh --start +``` + +至少修改: + +- `PUBLIC_HOST`、前后端端口; +- MySQL 地址、端口、数据库名、用户名、密码; +- `DEERFLOW_MODEL_COUNT`; +- 每个 `DEERFLOW_MODEL_N_*` 模型块; +- 实际需要启用的搜索、认证和外部业务接口配置。 + +禁止直接修改 `data/generated/compose.env`、生成后的 `config.yaml` 或前端 +`runtime-config.js`;它们会在下次启动或重启时被 `deploy.env` 重新生成。 + +常用命令: + +```bash +./OFFLINE_INSTALL.sh --start +./OFFLINE_INSTALL.sh --restart +./OFFLINE_INSTALL.sh --stop +./OFFLINE_INSTALL.sh --status +``` + +## 8. 只替换源码的更新方式 + +后端可以覆盖交付目录中的: + +```text +source/backend/app/ +source/backend/packages/ +source/backend/tests/ +``` + +前端先在构建机执行 `pnpm build`,再覆盖: + +```text +source/frontend/dist/ +``` + +覆盖完成后只执行: + +```bash +./OFFLINE_INSTALL.sh --restart +``` + +如果 Python 依赖或系统依赖发生变化,不能只替换源码:Python wheel 应同步更新 +`source/wheelhouse`;LibreOffice、Pandoc、字体或系统库变化必须重新构建后端镜像并 +发布新的完整包。 + +## 9. 数据、停止与回退 + +- `--stop` 不删除镜像、数据或依赖 volume。 +- 业务数据和生成配置保存在交付目录的 `data/`,升级前必须整体备份。 +- 新版本应解压到新的版本目录,不要直接覆盖旧版本目录。 +- 回退时停止新目录,在旧目录执行 `./OFFLINE_INSTALL.sh --start`。 +- 不执行 `docker compose down -v`,除非明确要销毁数据并已经完成备份。 + +服务器交付目录建议统一使用 `/opt/deerflow-offline-delivery/`;至少保留当前版本和 +上一个已验证版本,并保留各自的 SHA256 文件。 + +## 10. Git 提交流程 + +部署逻辑变化必须同时更新本文件及交付包内对应文档。提交前执行: + +```bash +git status --short +git diff --check +python3 deploy/linux-docker-offline-deerflow-only/test_release_renderer.py "$(pwd)" +``` + +禁止提交真实 `deploy.env`、数据库密码、模型 API Key、运行数据、生成配置、 +Docker 镜像 tar 或最终离线 `.tar.gz`。仓库中的 `deploy.env` 只能包含明确的假地址 +和示例凭据。