diff --git a/scripts/start-backend.sh b/scripts/start-backend.sh index a3900164..472f3c6a 100644 --- a/scripts/start-backend.sh +++ b/scripts/start-backend.sh @@ -5,6 +5,10 @@ source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/env.sh" PID_FILE="${PID_DIR}/backend.pid" LOG_FILE="${LOG_DIR}/backend.log" +BACKEND_VENDOR_DIR="${BACKEND_VENDOR_DIR:-${BACKEND_DIR}/.deer-flow/python-vendor}" +BACKEND_WHEELHOUSE="${BACKEND_WHEELHOUSE:-${BACKEND_DIR}/wheelhouse}" +BACKEND_INSTALL_DEPS="${BACKEND_INSTALL_DEPS:-1}" +BACKEND_PIP_PACKAGES="${BACKEND_PIP_PACKAGES:-croniter==6.2.2 langgraph-checkpoint-postgres==3.0.5 psycopg==3.3.3 psycopg-binary==3.3.3 psycopg-pool==3.3.0 orjson==3.11.9 tzdata==2026.2}" if is_running "${PID_FILE}"; then echo "Backend is already running (pid $(cat "${PID_FILE}"))." @@ -17,16 +21,41 @@ rm -f "${PID_FILE}" echo "Starting backend..." echo "Log: ${LOG_FILE}" +mkdir -p "${BACKEND_VENDOR_DIR}" + +if [[ "${BACKEND_INSTALL_DEPS}" != "0" ]]; then + if ! ( + cd "${BACKEND_DIR}" && \ + PYTHONPATH=".:${BACKEND_VENDOR_DIR}${PYTHONPATH:+:${PYTHONPATH}}" \ + uv run --no-sync python -c "import croniter; import psycopg; import psycopg_pool; import langgraph.checkpoint.postgres; import langgraph.store.postgres" + ) >/dev/null 2>&1; then + if [[ ! -d "${BACKEND_WHEELHOUSE}" ]]; then + echo "ERROR: wheelhouse not found: ${BACKEND_WHEELHOUSE}" >&2 + exit 1 + fi + echo "Installing backend runtime deps into ${BACKEND_VENDOR_DIR}" + ( + cd "${BACKEND_DIR}" && \ + uv pip install \ + --target "${BACKEND_VENDOR_DIR}" \ + --no-deps \ + --no-index \ + --find-links "${BACKEND_WHEELHOUSE}" \ + ${BACKEND_PIP_PACKAGES} + ) + fi +fi + setsid bash -c ' cd "$0" || exit 1 exec env \ GATEWAY_CORS_ORIGINS="$1" \ - PYTHONPATH=. \ + PYTHONPATH=".:$4${PYTHONPATH:+:${PYTHONPATH}}" \ uv run uvicorn app.gateway.app:app \ --host "$2" \ --port "$3" \ --reload -' "${BACKEND_DIR}" "${GATEWAY_CORS_ORIGINS}" "${BACKEND_HOST}" "${BACKEND_PORT}" \ +' "${BACKEND_DIR}" "${GATEWAY_CORS_ORIGINS}" "${BACKEND_HOST}" "${BACKEND_PORT}" "${BACKEND_VENDOR_DIR}" \ >"${LOG_FILE}" 2>&1 & echo "$!" >"${PID_FILE}" diff --git a/scripts/start_backend-online.sh b/scripts/start_backend-online.sh new file mode 100644 index 00000000..75755953 --- /dev/null +++ b/scripts/start_backend-online.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +IMAGE_TAG="${IMAGE_TAG:-deerflow-backend-offline:20260505}" +CONTAINER_NAME="${CONTAINER_NAME:-deerflow-backend}" +HOST_PORT="${HOST_PORT:-2026}" +IMAGE_TAR="${IMAGE_TAR:-${SCRIPT_DIR}/deerflow-backend-offline-image-20260505.tar}" +# Default timezone for the container. The offline image is built from a base +# layer that ships with UTC, which made scheduled tasks fire at the wrong wall +# clock and made log timestamps confusing for ops. ``Asia/Shanghai`` keeps the +# scheduler aligned with the operator's expectation; override with TZ=... when +# deploying to a different region. +CONTAINER_TZ="${CONTAINER_TZ:-${TZ:-Asia/Shanghai}}" + +die() { + echo "ERROR: $*" >&2 + exit 1 +} + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "command not found: $1" +} + +require_file() { + [ -f "$1" ] || die "required file not found: $1" +} + +require_dir() { + [ -d "$1" ] || die "required directory not found: $1" +} + +require_env_key() { + local key="$1" + grep -Eq "^[[:space:]]*${key}=" "${SCRIPT_DIR}/.env" || die "required .env key not found: ${key}" +} + +require_cmd docker + +if [ ! -f "${SCRIPT_DIR}/.env" ]; then + if [ -f "${SCRIPT_DIR}/.env.example" ]; then + cp "${SCRIPT_DIR}/.env.example" "${SCRIPT_DIR}/.env" + echo "Generated .env from .env.example. Please edit .env and run this script again." + exit 1 + fi + die "missing .env" +fi + +if [ ! -f "${SCRIPT_DIR}/config.yaml" ]; then + if [ -f "${SCRIPT_DIR}/config.yaml.example" ]; then + cp "${SCRIPT_DIR}/config.yaml.example" "${SCRIPT_DIR}/config.yaml" + echo "Generated config.yaml from config.yaml.example. Please edit model/API settings and run this script again." + exit 1 + fi + die "missing config.yaml" +fi + +require_env_key "AUTH_JWT_SECRET" +require_env_key "SCHEDULED_TASKS_ENABLED" +require_env_key "ELEMENT_ENABLED" +require_file "${SCRIPT_DIR}/config.yaml" +require_file "${SCRIPT_DIR}/extensions_config.json" +require_file "${SCRIPT_DIR}/backend/pyproject.toml" +require_file "${SCRIPT_DIR}/backend/app/gateway/app.py" +require_file "${SCRIPT_DIR}/backend/packages/harness/deerflow/runtime/scheduler/element.py" +require_dir "${SCRIPT_DIR}/skills" +require_dir "${SCRIPT_DIR}/wheelhouse" +# Knowledge base directory, mounted into the container at /mnt/knowledge_base. +require_dir "${SCRIPT_DIR}/backend/knowledge/knowledge_base" +mkdir -p "${SCRIPT_DIR}/data" + +if ! docker image inspect "${IMAGE_TAG}" >/dev/null 2>&1; then + require_file "${IMAGE_TAR}" + echo "Loading Docker image from ${IMAGE_TAR}" + docker load -i "${IMAGE_TAR}" +fi + +if docker ps -a --format '{{.Names}}' | awk -v name="${CONTAINER_NAME}" '$0 == name { found=1 } END { exit(found ? 0 : 1) }'; then + echo "Removing existing container ${CONTAINER_NAME}" + docker rm -f "${CONTAINER_NAME}" >/dev/null +fi + +echo "Starting backend container ${CONTAINER_NAME}" +docker run -d \ + --name "${CONTAINER_NAME}" \ + --restart unless-stopped \ + -p "${HOST_PORT}:8001" \ + --env-file "${SCRIPT_DIR}/.env" \ + -e TZ="${CONTAINER_TZ}" \ + -e DEER_FLOW_PROJECT_ROOT=/app \ + -e DEER_FLOW_HOME=/app/backend/.deer-flow \ + -e DEER_FLOW_CONFIG_PATH=/app/config.yaml \ + -e DEER_FLOW_EXTENSIONS_CONFIG_PATH=/app/extensions_config.json \ + -e DEER_FLOW_SKILLS_PATH=/app/skills \ + -e UV_LINK_MODE=copy \ + -v "${SCRIPT_DIR}/backend/app:/app/backend/app" \ + -v "${SCRIPT_DIR}/backend/packages:/app/backend/packages" \ + -v "${SCRIPT_DIR}/backend/scripts:/app/backend/scripts" \ + -v "${SCRIPT_DIR}/backend/pyproject.toml:/app/backend/pyproject.toml:ro" \ + -v "${SCRIPT_DIR}/backend/uv.lock:/app/backend/uv.lock:ro" \ + -v "${SCRIPT_DIR}/backend/langgraph.json:/app/backend/langgraph.json:ro" \ + -v "${SCRIPT_DIR}/config.yaml:/app/config.yaml:ro" \ + -v "${SCRIPT_DIR}/skill_evolution.json:/app/skill_evolution.json:rw" \ + -v "${SCRIPT_DIR}/extensions_config.json:/app/extensions_config.json:rw" \ + -v "${SCRIPT_DIR}/skills:/app/skills:rw" \ + -v "${SCRIPT_DIR}/wheelhouse:/app/wheelhouse:ro" \ + -v "${SCRIPT_DIR}/backend/knowledge/knowledge_base:/mnt/knowledge_base:ro" \ + -v "${SCRIPT_DIR}/data:/app/backend/.deer-flow" \ + "${IMAGE_TAG}" \ + sh -c 'cd backend +# Apply timezone inside the container. ``TZ`` alone is enough for Python / +# uvicorn log timestamps, but linking /etc/localtime keeps C-level tools +# (e.g. anything that reads /etc/localtime directly) on the same clock. +if [ -n "${TZ:-}" ] && [ -f "/usr/share/zoneinfo/${TZ}" ]; then + ln -sf "/usr/share/zoneinfo/${TZ}" /etc/localtime 2>/dev/null || true + echo "${TZ}" > /etc/timezone 2>/dev/null || true +fi +if [ "${SCHEDULER_INSTALL_DEPS:-1}" != "0" ]; then + uv run --no-sync python -c "import croniter" >/dev/null 2>&1 || { + PYTHON_BIN="$(uv run --no-sync python -c "import sys; print(sys.executable)")" + uv pip install --python "${PYTHON_BIN}" --no-index --find-links /app/wheelhouse ${SCHEDULER_PIP_PACKAGES:-croniter==6.2.2} + } +fi +if [ "${HINDSIGHT_INSTALL_DEPS:-1}" != "0" ]; then + uv run --no-sync python -c "import hindsight_client" >/dev/null 2>&1 || { + PYTHON_BIN="$(uv run --no-sync python -c "import sys; print(sys.executable)")" + uv pip install --python "${PYTHON_BIN}" --no-index --find-links /app/wheelhouse ${HINDSIGHT_INSTALL_DEPS:-hindsight_client==0.6.2} + } +fi +if [ "${PG_CHECKPOINTER_INSTALL_DEPS:-1}" != "0" ]; then + uv run --no-sync python -c "import psycopg; import psycopg_pool; import langgraph.checkpoint.postgres; import langgraph.store.postgres" >/dev/null 2>&1 || { + PYTHON_BIN="$(uv run --no-sync python -c "import sys; print(sys.executable)")" + uv pip install --python "${PYTHON_BIN}" --no-deps --no-index --find-links /app/wheelhouse ${PG_CHECKPOINTER_PIP_PACKAGES:-langgraph-checkpoint-postgres==3.0.5 psycopg==3.3.3 psycopg-binary==3.3.3 psycopg-pool==3.3.0 orjson==3.11.9 tzdata==2026.2} + } +fi +PYTHONPATH=. uv run --no-sync python -m uvicorn app.gateway.app:app --host 0.0.0.0 --port 8001 --workers ${GATEWAY_WORKERS:-8}' + +echo +echo "Backend started." +echo " URL: http://:${HOST_PORT}" +echo " Logs: docker logs -f ${CONTAINER_NAME}" +echo " Stop: ./stop_backend.sh"