deerflow-code/deploy/linux-docker-offline/sanitize_release_sources.py

#!/usr/bin/env python3
"""Create secret-free build inputs for the offline release.

The customized working tree contains environment-specific development values.
They must never become defaults in an exported image or release template.
"""

from __future__ import annotations

import json
import re
import sys
from pathlib import Path

import yaml


SECRET_KEY = re.compile(
    r"(?:api[_-]?key|password|passwd|secret|access[_-]?token|auth[_-]?(?:token|password))$",
    re.IGNORECASE,
)
URL_KEY = re.compile(r"(?:^|_)(?:url|uri|endpoint)$|base_url$", re.IGNORECASE)
SUSPICIOUS_VALUE = re.compile(
    r"(?:\bsk-[A-Za-z0-9_.-]{12,}|\bBearer\s+[A-Za-z0-9_.-]{12,})"
)


def scrub_mapping(value: object) -> object:
    if isinstance(value, dict):
        result: dict[object, object] = {}
        for key, child in value.items():
            key_text = str(key)
            # Names such as ``require_password`` are boolean feature flags,
            # not credentials. Only redact scalar string/null secret values.
            if SECRET_KEY.search(key_text) and (isinstance(child, str) or child is None):
                result[key] = ""
            elif URL_KEY.search(key_text) and (isinstance(child, str) or child is None):
                result[key] = ""
            else:
                result[key] = scrub_mapping(child)
        return result
    if isinstance(value, list):
        return [scrub_mapping(item) for item in value]
    if isinstance(value, str) and SUSPICIOUS_VALUE.search(value):
        return ""
    return value


def sanitize_backend(root: Path) -> None:
    config_path = root / "config.yaml"
    config = yaml.safe_load(config_path.read_text(encoding="utf-8-sig"))
    config = scrub_mapping(config)
    if not isinstance(config, dict):
        raise RuntimeError("DeerFlow config.yaml must contain a mapping")

    # The renderer recreates these sections exclusively from deploy.env.
    config["models"] = []
    config["vision"] = {}
    config["deep_research"] = {"image_generation": {"enabled": False}}
    config["database"] = {
        "backend": "sqlite",
        "sqlite_dir": ".deer-flow/data",
        "mysql_url": "",
    }
    config["checkpointer"] = {
        "type": "sqlite",
        "connection_string": ".deer-flow/data/checkpoints.db",
    }
    config["llmwiki"] = {
        "weknora": {
            "api_base_url": "",
            "web_base_url": "",
            "admin_email": "",
            "admin_password": "",
        },
        "local_wiki_index": {"enabled": False, "auto_sync": False},
    }

    task = config.setdefault("task_deeplink", {})
    if isinstance(task, dict):
        for flag in ("test", "delete_test", "action_test", "cop_task_detail_test"):
            task[flag] = True
        for key in tuple(task):
            if str(key).endswith("_url"):
                task[key] = ""

    memory = config.get("memory")
    if isinstance(memory, dict) and isinstance(memory.get("hindsight"), dict):
        memory["hindsight"]["api_url"] = ""
        memory["hindsight"]["api_key"] = ""

    rendered = yaml.safe_dump(config, allow_unicode=True, sort_keys=False, width=120)
    if SUSPICIOUS_VALUE.search(rendered):
        raise RuntimeError("sanitized DeerFlow config still contains a credential-like value")
    config_path.write_text(rendered, encoding="utf-8")

    extensions_path = root / "extensions_config.json"
    extensions = json.loads(extensions_path.read_text(encoding="utf-8-sig"))
    extensions_path.write_text(
        json.dumps(scrub_mapping(extensions), ensure_ascii=False, indent=2) + "\n",
        encoding="utf-8",
    )

    for env_path in root.glob(".env*"):
        if env_path.is_file():
            env_path.unlink()


def sanitize_frontend(root: Path) -> None:
    for env_path in root.glob(".env*"):
        if env_path.is_file():
            env_path.unlink()

    runtime_path = root / "public" / "runtime-config.js"
    text = runtime_path.read_text(encoding="utf-8-sig")
    property_pattern = re.compile(
        r'^(?P<prefix>\s*)(?P<comment>//\s*)?(?P<key>VITE_[A-Z0-9_]+)\s*:\s*(?P<quote>["\'])(?P<value>.*?)(?P=quote)(?P<comma>\s*,?\s*)$',
        re.MULTILINE,
    )

    def replace(match: re.Match[str]) -> str:
        key = match.group("key")
        value = match.group("value")
        sensitive_key = re.search(
            r"(?:URL|URI|HOST|ENDPOINT|PASSWORD|USERNAME|TOKEN|SECRET|AUTH)", key
        )
        absolute_value = re.match(r"^(?:https?|wss?)://", value, re.IGNORECASE)
        credential_value = SUSPICIOUS_VALUE.search(value)
        safe_brand_asset = key.startswith("VITE_BRAND_") and bool(value) and not absolute_value
        if (sensitive_key and not safe_brand_asset) or absolute_value or credential_value:
            value = ""
        quote = match.group("quote")
        comment = match.group("comment") or ""
        return f'{match.group("prefix")}{comment}{key}: {quote}{value}{quote}{match.group("comma")}'

    text = property_pattern.sub(replace, text)
    for match in property_pattern.finditer(text):
        value = match.group("value")
        if re.match(r"^(?:https?|wss?)://", value, re.IGNORECASE) or SUSPICIOUS_VALUE.search(value):
            raise RuntimeError(f"sanitized runtime-config still contains a configured value: {match.group('key')}")
    runtime_path.write_text(text, encoding="utf-8")


def main() -> int:
    if len(sys.argv) != 3 or sys.argv[1] not in {"backend", "frontend"}:
        print("usage: sanitize_release_sources.py backend|frontend PATH", file=sys.stderr)
        return 2
    root = Path(sys.argv[2]).resolve()
    if sys.argv[1] == "backend":
        sanitize_backend(root)
    else:
        sanitize_frontend(root)
    print(f"sanitized {sys.argv[1]} release source: {root}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())