feat: standardize DeerFlow offline deployment

2026-08-27 22:06:43 +08:00
1 parent 1764d0d5f2
commit d9a94245fc
@@ -0,0 +1,132 @@
+ARG PYTHON_IMAGE=python:3.12-slim-bookworm
+
+FROM ${PYTHON_IMAGE} AS builder
+
+ARG APT_MIRROR=mirrors.aliyun.com
+ARG UV_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple/
+ARG UV_VERSION=0.7.20
+ARG UV_NO_CACHE=1
+ARG UV_COMPILE_BYTECODE=0
+
+ENV UV_PROJECT_ENVIRONMENT=/opt/venv \
+    UV_INDEX_URL=${UV_INDEX_URL} \
+    UV_LINK_MODE=copy \
+    UV_NO_CACHE=${UV_NO_CACHE} \
+    UV_COMPILE_BYTECODE=${UV_COMPILE_BYTECODE} \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /build/backend
+
+RUN if [ -n "${APT_MIRROR}" ]; then \
+      sed -i "s|deb.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list.d/debian.sources 2>/dev/null || true; \
+      sed -i "s|security.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list.d/debian.sources 2>/dev/null || true; \
+      sed -i "s|deb.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list 2>/dev/null || true; \
+      sed -i "s|security.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list 2>/dev/null || true; \
+    fi \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        curl \
+        git \
+    && rm -rf /var/lib/apt/lists/* \
+    && python -m pip install --no-cache-dir -i "${UV_INDEX_URL}" "uv==${UV_VERSION}"
+
+ARG UV_HTTP_TIMEOUT=300
+ARG UV_CONCURRENT_DOWNLOADS=4
+ARG UV_CONCURRENT_INSTALLS=4
+ARG UV_CONCURRENT_BUILDS=2
+
+ENV UV_HTTP_TIMEOUT=${UV_HTTP_TIMEOUT} \
+    UV_CONCURRENT_DOWNLOADS=${UV_CONCURRENT_DOWNLOADS} \
+    UV_CONCURRENT_INSTALLS=${UV_CONCURRENT_INSTALLS} \
+    UV_CONCURRENT_BUILDS=${UV_CONCURRENT_BUILDS}
+
+COPY pyproject.toml uv.lock README.md requirements-offline.txt ./
+COPY packages ./packages
+
+RUN python -m venv /opt/venv \
+    && /opt/venv/bin/python -m pip install --no-cache-dir --upgrade -i "${UV_INDEX_URL}" pip setuptools wheel \
+    && sed -E '/--hash=/d; s/[[:space:]]+\\$//; s|^-e[[:space:]]+./packages/harness$|./packages/harness|' requirements-offline.txt > /tmp/requirements-docker.txt \
+    && /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" -r /tmp/requirements-docker.txt \
+    && /opt/venv/bin/pip install --no-cache-dir -i "${UV_INDEX_URL}" \
+        "aiomysql==0.3.2" \
+        "PyMySQL==1.2.0" \
+        "SQLAlchemy==2.0.51" \
+        "python-docx==1.2.0" \
+    && /opt/venv/bin/python -c 'import aiomysql, docx, lxml, mammoth, markitdown, openpyxl, pandas, pptx, pymysql, sqlalchemy, xlrd; print("aiomysql and Office Python dependencies OK")'
+
+COPY . .
+
+# Only runtime material is copied into the final image. Development env files,
+# tests and repository documentation are deliberately excluded from that copy.
+RUN find /build/backend -maxdepth 2 -type f -name '.env*' -delete \
+    && rm -rf \
+      /build/backend/.pytest_cache \
+      /build/backend/tests \
+      /build/backend/packages/harness/tests \
+      /build/backend/docs \
+      /build/backend/CLAUDE.md \
+      /build/backend/AGENTS.md \
+      /build/backend/README.md
+
+FROM ${PYTHON_IMAGE} AS runtime
+
+ARG APT_MIRROR=mirrors.aliyun.com
+
+ENV PATH=/opt/venv/bin:$PATH \
+    PYTHONPATH=/app/backend:/app/backend/packages/harness \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    DEER_FLOW_PROJECT_ROOT=/app/backend \
+    DEER_FLOW_HOME=/app/backend/.deer-flow \
+    DEER_FLOW_CONFIG_PATH=/app/backend/config.yaml \
+    DEER_FLOW_EXTENSIONS_CONFIG_PATH=/app/backend/extensions_config.json \
+    DEER_FLOW_SKILLS_PATH=/app/backend/skills \
+    BACKEND_PORT=8001
+
+WORKDIR /app/backend
+
+RUN if [ -n "${APT_MIRROR}" ]; then \
+      sed -i "s|deb.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list.d/debian.sources 2>/dev/null || true; \
+      sed -i "s|security.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list.d/debian.sources 2>/dev/null || true; \
+      sed -i "s|deb.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list 2>/dev/null || true; \
+      sed -i "s|security.debian.org|${APT_MIRROR}|g" /etc/apt/sources.list 2>/dev/null || true; \
+    fi \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ca-certificates \
+        curl \
+        docker.io \
+        file \
+        fontconfig \
+        fonts-liberation \
+        fonts-noto-cjk \
+        libmagic1 \
+        libreoffice-calc \
+        libreoffice-impress \
+        libreoffice-writer \
+        nodejs \
+        npm \
+        pandoc \
+        poppler-utils \
+        tini \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /opt/venv /opt/venv
+COPY --from=builder /build/backend /app/backend
+
+# Fail the image build if the local sandbox cannot process the default Office
+# formats.  The production deployment uses LocalSandboxProvider, so these are
+# the exact Python modules and native tools available to agent bash commands.
+RUN /opt/venv/bin/python -c 'import aiomysql, docx, lxml, mammoth, markitdown, openpyxl, pandas, pptx, pymysql, sqlalchemy, xlrd' \
+    && command -v soffice >/dev/null \
+    && command -v pandoc >/dev/null \
+    && fc-list :lang=zh | grep -q .
+
+EXPOSE 8001
+
+ENTRYPOINT ["tini", "--"]
+CMD ["sh", "-c", "cd /app/backend && exec uvicorn app.gateway.app:app --host 0.0.0.0 --port ${BACKEND_PORT:-8001}"]
@@ -0,0 +1,24 @@
+FROM node:22-bookworm-slim AS builder
+
+WORKDIR /workspace/frontend-web
+
+RUN corepack enable && corepack prepare pnpm@10.26.2 --activate
+
+COPY frontend-web/package.json frontend-web/pnpm-lock.yaml ./
+RUN --mount=type=cache,target=/root/.local/share/pnpm/store \
+    pnpm install --frozen-lockfile
+
+COPY frontend-web/ ./
+
+ENV VITE_ASSET_BASE=/
+RUN pnpm build
+
+FROM nginx:1.28-alpine
+
+COPY deploy/linux-docker-offline/nginx/deerflow.conf /etc/nginx/conf.d/default.conf
+COPY --from=builder /workspace/frontend-web/dist /usr/share/nginx/html
+
+EXPOSE 80
+
+HEALTHCHECK --interval=20s --timeout=5s --start-period=10s --retries=5 \
+  CMD wget -q -O /dev/null http://127.0.0.1/ || exit 1
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""Create secret-free build inputs for the offline release.
+
+The customized working tree contains environment-specific development values.
+They must never become defaults in an exported image or release template.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+import yaml
+
+
+SECRET_KEY = re.compile(
+    r"(?:api[_-]?key|password|passwd|secret|access[_-]?token|auth[_-]?(?:token|password))$",
+    re.IGNORECASE,
+)
+URL_KEY = re.compile(r"(?:^|_)(?:url|uri|endpoint)$|base_url$", re.IGNORECASE)
+SUSPICIOUS_VALUE = re.compile(
+    r"(?:\bsk-[A-Za-z0-9_.-]{12,}|\bBearer\s+[A-Za-z0-9_.-]{12,})"
+)
+
+
+def scrub_mapping(value: object) -> object:
+    if isinstance(value, dict):
+        result: dict[object, object] = {}
+        for key, child in value.items():
+            key_text = str(key)
+            # Names such as ``require_password`` are boolean feature flags,
+            # not credentials. Only redact scalar string/null secret values.
+            if SECRET_KEY.search(key_text) and (isinstance(child, str) or child is None):
+                result[key] = ""
+            elif URL_KEY.search(key_text) and (isinstance(child, str) or child is None):
+                result[key] = ""
+            else:
+                result[key] = scrub_mapping(child)
+        return result
+    if isinstance(value, list):
+        return [scrub_mapping(item) for item in value]
+    if isinstance(value, str) and SUSPICIOUS_VALUE.search(value):
+        return ""
+    return value
+
+
+def sanitize_backend(root: Path) -> None:
+    config_path = root / "config.yaml"
+    config = yaml.safe_load(config_path.read_text(encoding="utf-8-sig"))
+    config = scrub_mapping(config)
+    if not isinstance(config, dict):
+        raise RuntimeError("DeerFlow config.yaml must contain a mapping")
+
+    # The renderer recreates these sections exclusively from deploy.env.
+    config["models"] = []
+    config["vision"] = {}
+    config["deep_research"] = {"image_generation": {"enabled": False}}
+    config["database"] = {
+        "backend": "sqlite",
+        "sqlite_dir": ".deer-flow/data",
+        "mysql_url": "",
+    }
+    config["checkpointer"] = {
+        "type": "sqlite",
+        "connection_string": ".deer-flow/data/checkpoints.db",
+    }
+    config["llmwiki"] = {
+        "weknora": {
+            "api_base_url": "",
+            "web_base_url": "",
+            "admin_email": "",
+            "admin_password": "",
+        },
+        "local_wiki_index": {"enabled": False, "auto_sync": False},
+    }
+
+    task = config.setdefault("task_deeplink", {})
+    if isinstance(task, dict):
+        for flag in ("test", "delete_test", "action_test", "cop_task_detail_test"):
+            task[flag] = True
+        for key in tuple(task):
+            if str(key).endswith("_url"):
+                task[key] = ""
+
+    memory = config.get("memory")
+    if isinstance(memory, dict) and isinstance(memory.get("hindsight"), dict):
+        memory["hindsight"]["api_url"] = ""
+        memory["hindsight"]["api_key"] = ""
+
+    rendered = yaml.safe_dump(config, allow_unicode=True, sort_keys=False, width=120)
+    if SUSPICIOUS_VALUE.search(rendered):
+        raise RuntimeError("sanitized DeerFlow config still contains a credential-like value")
+    config_path.write_text(rendered, encoding="utf-8")
+
+    extensions_path = root / "extensions_config.json"
+    extensions = json.loads(extensions_path.read_text(encoding="utf-8-sig"))
+    extensions_path.write_text(
+        json.dumps(scrub_mapping(extensions), ensure_ascii=False, indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+    for env_path in root.glob(".env*"):
+        if env_path.is_file():
+            env_path.unlink()
+
+
+def sanitize_frontend(root: Path) -> None:
+    for env_path in root.glob(".env*"):
+        if env_path.is_file():
+            env_path.unlink()
+
+    runtime_path = root / "public" / "runtime-config.js"
+    text = runtime_path.read_text(encoding="utf-8-sig")
+    property_pattern = re.compile(
+        r'^(?P<prefix>\s*)(?P<comment>//\s*)?(?P<key>VITE_[A-Z0-9_]+)\s*:\s*(?P<quote>["\'])(?P<value>.*?)(?P=quote)(?P<comma>\s*,?\s*)$',
+        re.MULTILINE,
+    )
+
+    def replace(match: re.Match[str]) -> str:
+        key = match.group("key")
+        value = match.group("value")
+        sensitive_key = re.search(
+            r"(?:URL|URI|HOST|ENDPOINT|PASSWORD|USERNAME|TOKEN|SECRET|AUTH)", key
+        )
+        absolute_value = re.match(r"^(?:https?|wss?)://", value, re.IGNORECASE)
+        credential_value = SUSPICIOUS_VALUE.search(value)
+        safe_brand_asset = key.startswith("VITE_BRAND_") and bool(value) and not absolute_value
+        if (sensitive_key and not safe_brand_asset) or absolute_value or credential_value:
+            value = ""
+        quote = match.group("quote")
+        comment = match.group("comment") or ""
+        return f'{match.group("prefix")}{comment}{key}: {quote}{value}{quote}{match.group("comma")}'
+
+    text = property_pattern.sub(replace, text)
+    for match in property_pattern.finditer(text):
+        value = match.group("value")
+        if re.match(r"^(?:https?|wss?)://", value, re.IGNORECASE) or SUSPICIOUS_VALUE.search(value):
+            raise RuntimeError(f"sanitized runtime-config still contains a configured value: {match.group('key')}")
+    runtime_path.write_text(text, encoding="utf-8")
+
+
+def main() -> int:
+    if len(sys.argv) != 3 or sys.argv[1] not in {"backend", "frontend"}:
+        print("usage: sanitize_release_sources.py backend|frontend PATH", file=sys.stderr)
+        return 2
+    root = Path(sys.argv[2]).resolve()
+    if sys.argv[1] == "backend":
+        sanitize_backend(root)
+    else:
+        sanitize_frontend(root)
+    print(f"sanitized {sys.argv[1]} release source: {root}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())