deerflow-code/offline-backend-20260512/backend/app/gateway/routers/workflow_data_sources.py

"""Workflow data sources and HTTP credentials.

Write-only secrets: the DSN (or the credential header blob) is accepted on
create/update, encrypted immediately, and never returned. Reads expose only a
masked target so an operator can tell two sources apart.

Shared sources (``ownerId`` null) are admin-only; a regular user may manage
their own.
"""

from __future__ import annotations

import json
from typing import Any
from urllib.parse import urlsplit

from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field

from app.gateway.deps import get_current_user, get_optional_user_from_request
from app.gateway.workflow_audit import audit_workflow
from deerflow.config.app_config import get_app_config
from deerflow.workflows.security.secret_box import SecretKeyMissingError, secrets_available
from deerflow.workflows.security.sql_policy import assert_read_only, assert_tables_allowed

router = APIRouter(prefix="/api/workflows/data-sources", tags=["workflow-data-sources"])

_HTTP_METHODS = frozenset({"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"})


class DataSourceCreate(BaseModel):
    name: str = Field(min_length=1, max_length=191)
    description: str = ""
    kind: str = Field(default="sql", pattern="^(sql|http)$")
    dsn: str | None = None
    headers: dict[str, str] | None = None
    base_url: str = Field(default="", max_length=2048, alias="baseUrl")
    allowed_methods: list[str] = Field(default_factory=list, alias="allowedMethods")
    max_rows: int = Field(default=1000, ge=1, le=100_000, alias="maxRows")
    allowed_tables: list[str] = Field(default_factory=list, alias="allowedTables")
    enabled: bool = True
    shared: bool = False

    model_config = {"populate_by_name": True}


class QueryBody(BaseModel):
    statement: str = Field(min_length=1)


class DataSourceUpdate(BaseModel):
    name: str | None = Field(default=None, min_length=1, max_length=191)
    description: str | None = None
    dsn: str | None = None
    headers: dict[str, str] | None = None
    base_url: str | None = Field(default=None, max_length=2048, alias="baseUrl")
    allowed_methods: list[str] | None = Field(default=None, alias="allowedMethods")
    max_rows: int | None = Field(default=None, ge=1, le=100_000, alias="maxRows")
    allowed_tables: list[str] | None = Field(default=None, alias="allowedTables")
    enabled: bool | None = None

    model_config = {"populate_by_name": True}


def _require_enabled() -> None:
    if not get_app_config().workflows.enabled:
        raise HTTPException(status_code=503, detail="Workflow Studio is disabled")


def _store(request: Request):
    store = getattr(request.app.state, "workflow_data_source_store", None)
    if store is None:
        raise HTTPException(status_code=503, detail="Workflow data source store not available")
    return store


async def _is_admin(request: Request) -> bool:
    user = await get_optional_user_from_request(request)
    return user is not None and getattr(user, "system_role", None) == "admin"


def _normalise_http_metadata(
    base_url: str,
    allowed_methods: list[str],
) -> tuple[str, list[str]]:
    target = base_url.strip()
    try:
        parsed = urlsplit(target)
    except ValueError as exc:
        raise HTTPException(status_code=400, detail="HTTP 接口地址格式无效") from exc
    if (
        parsed.scheme not in {"http", "https"}
        or not parsed.netloc
        or parsed.username is not None
        or parsed.password is not None
        or parsed.query
        or parsed.fragment
    ):
        raise HTTPException(
            status_code=400,
            detail="HTTP 接口地址必须是无凭据、无查询参数的 http(s) 地址",
        )

    normalised_methods: list[str] = []
    for raw_method in allowed_methods:
        method = str(raw_method).strip().upper()
        if method not in _HTTP_METHODS:
            raise HTTPException(status_code=400, detail=f"不支持的 HTTP 方法：{method}")
        if method not in normalised_methods:
            normalised_methods.append(method)
    if not normalised_methods:
        raise HTTPException(status_code=400, detail="至少选择一种 HTTP 方法")
    return target, normalised_methods


def _normalise_headers(headers: dict[str, str] | None) -> dict[str, str]:
    normalised: dict[str, str] = {}
    for raw_name, raw_value in (headers or {}).items():
        name = str(raw_name).strip()
        value = str(raw_value)
        if not name or any(char in name + value for char in "\r\n"):
            raise HTTPException(status_code=400, detail="HTTP 请求头格式无效")
        normalised[name] = value
    return normalised


def _secret_from(kind: str, dsn: str | None, headers: dict[str, str] | None) -> str:
    if kind == "http":
        return json.dumps({"headers": _normalise_headers(headers)}, ensure_ascii=False)
    if not dsn:
        raise HTTPException(status_code=400, detail="SQL 数据源需要提供 dsn")
    if "://" not in dsn:
        raise HTTPException(status_code=400, detail="dsn 格式无效")
    return dsn


@router.get("")
async def list_data_sources(request: Request) -> dict[str, Any]:
    _require_enabled()
    user_id = await get_current_user(request)
    rows = await _store(request).list_sources(owner_id=user_id)
    return {"dataSources": rows, "secretsEncryptionAvailable": secrets_available()}


@router.post("")
async def create_data_source(request: Request, body: DataSourceCreate) -> dict[str, Any]:
    _require_enabled()
    user_id = await get_current_user(request)
    is_admin = await _is_admin(request)
    if body.shared and not is_admin:
        raise HTTPException(status_code=403, detail="仅管理员可创建共享数据源")
    secret = _secret_from(body.kind, body.dsn, body.headers)
    base_url = ""
    allowed_methods: list[str] = []
    if body.kind == "http":
        base_url, allowed_methods = _normalise_http_metadata(
            body.base_url,
            body.allowed_methods,
        )
    try:
        created = await _store(request).create_source(
            {
                "name": body.name,
                "description": body.description,
                "kind": body.kind,
                "dsn": secret,
                "base_url": base_url,
                "allowed_methods": allowed_methods,
                "max_rows": body.max_rows,
                "allowed_tables": body.allowed_tables,
                "enabled": body.enabled,
                "owner_id": None if body.shared else user_id,
                "created_by": user_id,
            }
        )
    except SecretKeyMissingError as exc:
        raise HTTPException(
            status_code=503,
            detail={
                "code": "WORKFLOW_RESOURCE_MISSING",
                "message": "服务端未配置 WORKFLOW_SECRET_KEY，无法安全保存连接信息",
            },
        ) from exc
    audit_workflow("data_source.create", user_id=user_id, dataSourceId=created["id"], kind=body.kind, shared=body.shared)
    return created


async def _owned(request: Request, source_id: str) -> dict[str, Any]:
    row = await _store(request).get_source(source_id)
    if row is None:
        raise HTTPException(status_code=404, detail="数据源不存在")
    user_id = await get_current_user(request)
    if row.get("owner_id") == user_id:
        return row
    if await _is_admin(request):
        return row
    raise HTTPException(status_code=403, detail="无权操作该数据源")


async def _visible(request: Request, source_id: str) -> dict[str, Any]:
    row = await _store(request).get_source(source_id)
    if row is None:
        raise HTTPException(status_code=404, detail="数据源不存在")
    user_id = await get_current_user(request)
    if row.get("owner_id") in (user_id, None):
        return row
    if await _is_admin(request):
        return row
    raise HTTPException(status_code=403, detail="无权查看该数据源")


@router.get("/{source_id}")
async def get_data_source(request: Request, source_id: str) -> dict[str, Any]:
    _require_enabled()
    return await _visible(request, source_id)


@router.post("/{source_id}/introspect")
async def introspect_data_source(request: Request, source_id: str) -> dict[str, Any]:
    """Return the configured table allowlist only — never sample rows."""
    _require_enabled()
    row = await _visible(request, source_id)
    tables = list(row.get("allowed_tables") or [])
    return {
        "dataSourceId": row["id"],
        "kind": row.get("kind"),
        "tables": [{"name": name} for name in tables],
        "schemas": [],
    }


@router.post("/{source_id}/validate-query")
async def validate_data_source_query(request: Request, source_id: str, body: QueryBody) -> dict[str, Any]:
    """Validate SQL against the read-only policy without executing it."""
    _require_enabled()
    row = await _visible(request, source_id)
    try:
        statement = assert_read_only(body.statement)
        assert_tables_allowed(statement, list(row.get("allowed_tables") or []))
    except Exception as exc:  # noqa: BLE001
        from deerflow.workflows.errors import WorkflowError

        if isinstance(exc, WorkflowError):
            raise HTTPException(status_code=400, detail=exc.to_body().model_dump(by_alias=True)) from exc
        raise
    audit_workflow("data_source.validate_query", user_id=await get_current_user(request), dataSourceId=source_id)
    return {"ok": True, "statement": statement}


@router.put("/{source_id}")
async def update_data_source(request: Request, source_id: str, body: DataSourceUpdate) -> dict[str, Any]:
    _require_enabled()
    row = await _owned(request, source_id)
    payload = body.model_dump(exclude_unset=True, by_alias=False)
    kind = str(row.get("kind") or "sql")
    if body.dsn is not None or body.headers is not None:
        payload["dsn"] = _secret_from(kind, body.dsn, body.headers)
    payload.pop("headers", None)
    if kind == "http":
        base_url, allowed_methods = _normalise_http_metadata(
            body.base_url if body.base_url is not None else str(row.get("base_url") or ""),
            body.allowed_methods
            if body.allowed_methods is not None
            else list(row.get("allowed_methods") or []),
        )
        payload["base_url"] = base_url
        payload["allowed_methods"] = allowed_methods
    else:
        payload.pop("base_url", None)
        payload.pop("allowed_methods", None)
    try:
        updated = await _store(request).update_source(source_id, payload)
    except SecretKeyMissingError as exc:
        raise HTTPException(status_code=503, detail="服务端未配置 WORKFLOW_SECRET_KEY") from exc
    if updated is None:
        raise HTTPException(status_code=404, detail="数据源不存在")
    audit_workflow("data_source.update", user_id=await get_current_user(request), dataSourceId=source_id)
    return updated


@router.delete("/{source_id}")
async def delete_data_source(request: Request, source_id: str) -> dict[str, Any]:
    _require_enabled()
    await _owned(request, source_id)
    deleted = await _store(request).delete_source(source_id)
    audit_workflow("data_source.delete", user_id=await get_current_user(request), dataSourceId=source_id)
    return {"deleted": deleted}