代码提交
This commit is contained in:
1 parent
4ac7bc21cd
commit
17cf849361
110 files changed
+17007
-518
No files matched your search
@@ -27,10 +27,10 @@ from app.gateway.csrf_middleware import CSRFMiddleware
|
||||
from app.gateway.deps import langgraph_runtime
|
||||
from app.gateway.public_knowledge_cors import PublicKnowledgeCORSMiddleware
|
||||
from app.gateway.routers import (
|
||||
ai_writing,
|
||||
admin_active_runs,
|
||||
admin_users,
|
||||
agents,
|
||||
ai_writing,
|
||||
artifacts,
|
||||
assistants_compat,
|
||||
auth,
|
||||
@@ -39,11 +39,12 @@ from app.gateway.routers import (
|
||||
channels,
|
||||
checkpoint_migration,
|
||||
compact_menu_overrides,
|
||||
dashboard_sessions,
|
||||
deep_research,
|
||||
external_llmwiki,
|
||||
feedback,
|
||||
fixed_questions,
|
||||
dashboard_sessions,
|
||||
deep_research,
|
||||
flow_extract,
|
||||
html_page_favorites,
|
||||
intent,
|
||||
knowledge,
|
||||
@@ -63,19 +64,19 @@ from app.gateway.routers import (
|
||||
open_chat,
|
||||
parallel_agent_tasks,
|
||||
parallel_agents,
|
||||
position_roles,
|
||||
position_roundtable,
|
||||
positions,
|
||||
position_roles,
|
||||
public_config,
|
||||
public_embed,
|
||||
public_knowledge_vector_search,
|
||||
public_leaderboard,
|
||||
public_scheduled_tasks,
|
||||
recommend,
|
||||
flow_extract,
|
||||
recommended_questions,
|
||||
roundtable_chains,
|
||||
report_structures,
|
||||
roundtable_artifact_submissions,
|
||||
roundtable_chains,
|
||||
roundtable_diagnostics,
|
||||
roundtable_draft_shares,
|
||||
roundtable_drafts,
|
||||
@@ -91,16 +92,22 @@ from app.gateway.routers import (
|
||||
system_settings,
|
||||
tags,
|
||||
task_buttons,
|
||||
taskcop_tasks,
|
||||
report_structures,
|
||||
task_reports,
|
||||
taskcop_tasks,
|
||||
thread_runs,
|
||||
tool_metrics,
|
||||
thread_shares,
|
||||
threads,
|
||||
tool_metrics,
|
||||
uploads,
|
||||
user_preferences,
|
||||
user_prompts,
|
||||
workflow_data_sources,
|
||||
workflow_embed,
|
||||
workflow_resources,
|
||||
workflow_runs,
|
||||
workflow_studio,
|
||||
workflows,
|
||||
workflows_coze_compat,
|
||||
writing,
|
||||
)
|
||||
from deerflow.config import app_config as deerflow_app_config
|
||||
@@ -342,10 +349,10 @@ async def _run_db_migrations() -> None:
|
||||
deadline = asyncio.get_running_loop().time() + timeout_s
|
||||
while result_queue.empty():
|
||||
if asyncio.get_running_loop().time() >= deadline:
|
||||
raise asyncio.TimeoutError
|
||||
raise TimeoutError
|
||||
await asyncio.sleep(0.2)
|
||||
result = result_queue.get_nowait()
|
||||
except asyncio.TimeoutError:
|
||||
except TimeoutError:
|
||||
logger.error(
|
||||
"Database migration did not finish within %.0fs; continuing startup on create_all schema. "
|
||||
"Likely cause: MySQL metadata-lock wait or a slow DDL. Set DEER_FLOW_SKIP_ALEMBIC=1 "
|
||||
@@ -1591,6 +1598,18 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Light-app registry (?????) at /api/light-apps
|
||||
app.include_router(light_apps.router)
|
||||
|
||||
# Workflow Studio: definitions / runs / SSE / data sources / embed / Coze compat.
|
||||
# Static prefixes (node-types, resources, runs, data-sources, embed) must be
|
||||
# registered before the /{workflow_id} catch-alls in workflows.router.
|
||||
app.include_router(workflow_resources.router)
|
||||
app.include_router(workflow_data_sources.router)
|
||||
app.include_router(workflow_embed.router)
|
||||
app.include_router(workflow_runs.router)
|
||||
app.include_router(workflow_studio.router)
|
||||
app.include_router(workflows.router)
|
||||
app.include_router(workflows_coze_compat.router)
|
||||
app.include_router(workflows_coze_compat.chrome_router)
|
||||
|
||||
# Sidebar-menu overrides (菜单管理) at /api/menu-overrides
|
||||
app.include_router(menu_overrides.router)
|
||||
|
||||
|
||||
@@ -4,25 +4,25 @@ Per RFC-001:
|
||||
State-changing operations require CSRF protection.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
from collections.abc import Callable
|
||||
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from app.gateway.auth.disabled_mode import is_auth_disabled
|
||||
from app.gateway.proxy_path import app_relative_path
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_COOKIE,
|
||||
has_valid_weknora_embed_cookie,
|
||||
is_weknora_embed_proxy_path,
|
||||
)
|
||||
|
||||
CSRF_COOKIE_NAME = "csrf_token"
|
||||
CSRF_HEADER_NAME = "X-CSRF-Token"
|
||||
CSRF_TOKEN_LENGTH = 64 # bytes
|
||||
import secrets
|
||||
from collections.abc import Callable
|
||||
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp
|
||||
|
||||
from app.gateway.auth.disabled_mode import is_auth_disabled
|
||||
from app.gateway.proxy_path import app_relative_path
|
||||
from app.gateway.weknora_embed import (
|
||||
WEKNORA_EMBED_COOKIE,
|
||||
has_valid_weknora_embed_cookie,
|
||||
is_weknora_embed_proxy_path,
|
||||
)
|
||||
|
||||
CSRF_COOKIE_NAME = "csrf_token"
|
||||
CSRF_HEADER_NAME = "X-CSRF-Token"
|
||||
CSRF_TOKEN_LENGTH = 64 # bytes
|
||||
|
||||
|
||||
def is_secure_request(request: Request) -> bool:
|
||||
@@ -35,7 +35,7 @@ def generate_csrf_token() -> str:
|
||||
return secrets.token_urlsafe(CSRF_TOKEN_LENGTH)
|
||||
|
||||
|
||||
def should_check_csrf(request: Request) -> bool:
|
||||
def should_check_csrf(request: Request) -> bool:
|
||||
"""Determine if a request needs CSRF validation.
|
||||
|
||||
CSRF is checked for state-changing methods (POST, PUT, DELETE, PATCH).
|
||||
@@ -49,12 +49,12 @@ def should_check_csrf(request: Request) -> bool:
|
||||
path = app_relative_path(request)
|
||||
stripped = path.rstrip("/")
|
||||
# Exempt /api/v1/auth/me endpoint
|
||||
if stripped == "/api/v1/auth/me":
|
||||
return False
|
||||
if stripped == "/api/knowledge/vector-search":
|
||||
return False
|
||||
if stripped == "/api/writing/export/docx":
|
||||
return False
|
||||
if stripped == "/api/v1/auth/me":
|
||||
return False
|
||||
if stripped == "/api/knowledge/vector-search":
|
||||
return False
|
||||
if stripped == "/api/writing/export/docx":
|
||||
return False
|
||||
# Anonymous public endpoints (mirror auth_middleware._PUBLIC_PATH_PREFIXES).
|
||||
# These intentionally have no cookie session, so the Double-Submit Cookie
|
||||
# pattern is both impossible (no token to compare) and unnecessary (no
|
||||
@@ -64,18 +64,37 @@ def should_check_csrf(request: Request) -> bool:
|
||||
return False
|
||||
# 开放问答接口同样是匿名、无会话的(见 auth_middleware._PUBLIC_PATH_PREFIXES),
|
||||
# 双提交 Cookie 既无 token 可比、也无会话可被伪造,豁免之。
|
||||
if path.startswith("/api/open/"):
|
||||
return False
|
||||
if is_weknora_embed_proxy_request(request):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def is_weknora_embed_proxy_request(request: Request) -> bool:
|
||||
path = app_relative_path(request)
|
||||
if not is_weknora_embed_proxy_path(path):
|
||||
return False
|
||||
return has_valid_weknora_embed_cookie(request.cookies.get(WEKNORA_EMBED_COOKIE))
|
||||
if path.startswith("/api/open/"):
|
||||
return False
|
||||
if is_coze_studio_compat_path(path):
|
||||
return False
|
||||
if is_weknora_embed_proxy_request(request):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# Coze playground 自己的 axios 不会带 X-CSRF-Token;独立编排页经
|
||||
# localhost:8080 同源代理访问这些接口。Cookie 会话仍由 AuthMiddleware 校验。
|
||||
_COZE_STUDIO_PREFIXES: tuple[str, ...] = (
|
||||
"/api/passport/",
|
||||
"/api/playground_api/",
|
||||
"/api/workflow_api/",
|
||||
"/api/space/",
|
||||
)
|
||||
|
||||
|
||||
def is_coze_studio_compat_path(path: str) -> bool:
|
||||
stripped = path.rstrip("/")
|
||||
if stripped == "/api/bot/get_type_list":
|
||||
return True
|
||||
return any(path.startswith(prefix) for prefix in _COZE_STUDIO_PREFIXES)
|
||||
|
||||
|
||||
def is_weknora_embed_proxy_request(request: Request) -> bool:
|
||||
path = app_relative_path(request)
|
||||
if not is_weknora_embed_proxy_path(path):
|
||||
return False
|
||||
return has_valid_weknora_embed_cookie(request.cookies.get(WEKNORA_EMBED_COOKIE))
|
||||
|
||||
|
||||
_AUTH_EXEMPT_PATHS: frozenset[str] = frozenset(
|
||||
@@ -90,7 +109,7 @@ _AUTH_EXEMPT_PATHS: frozenset[str] = frozenset(
|
||||
"/api/v1/auth/register",
|
||||
"/api/v1/auth/initialize",
|
||||
# 并行多智能体面板专用取 token 端点(公共 + CSRF 豁免)。
|
||||
"/api/parallel-agents/auth/token",
|
||||
"/api/parallel-agents/auth/token",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -94,15 +94,15 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
app.state.run_store = RunRepository(sf)
|
||||
app.state.feedback_repo = FeedbackRepository(sf)
|
||||
else:
|
||||
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
||||
|
||||
app.state.run_store = MemoryRunStore()
|
||||
app.state.feedback_repo = None
|
||||
|
||||
from deerflow.persistence.fixed_questions import make_fixed_question_store
|
||||
|
||||
app.state.fixed_question_store = make_fixed_question_store(sf)
|
||||
else:
|
||||
from deerflow.runtime.runs.store.memory import MemoryRunStore
|
||||
|
||||
app.state.run_store = MemoryRunStore()
|
||||
app.state.feedback_repo = None
|
||||
|
||||
from deerflow.persistence.fixed_questions import make_fixed_question_store
|
||||
|
||||
app.state.fixed_question_store = make_fixed_question_store(sf)
|
||||
|
||||
from deerflow.persistence.thread_meta import make_thread_store
|
||||
|
||||
@@ -285,6 +285,18 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
app.state.light_app_store = make_light_app_store(sf)
|
||||
|
||||
from deerflow.persistence.workflows import make_workflow_store
|
||||
|
||||
app.state.workflow_store = make_workflow_store(sf)
|
||||
|
||||
from deerflow.persistence.workflow_data_sources import make_workflow_data_source_store
|
||||
from deerflow.persistence.workflow_events import make_workflow_event_store
|
||||
from deerflow.persistence.workflow_runs import make_workflow_run_store
|
||||
|
||||
app.state.workflow_run_store = make_workflow_run_store(sf)
|
||||
app.state.workflow_event_store = make_workflow_event_store(sf, run_store=app.state.workflow_run_store)
|
||||
app.state.workflow_data_source_store = make_workflow_data_source_store(sf)
|
||||
|
||||
from deerflow.persistence.menu_overrides import make_menu_override_store
|
||||
|
||||
app.state.menu_override_store = make_menu_override_store(sf)
|
||||
@@ -312,72 +324,72 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
from deerflow.integrations.weknora.runtime import register_llmwiki_store
|
||||
from deerflow.persistence.llmwiki import make_llmwiki_store
|
||||
|
||||
app.state.llmwiki_store = make_llmwiki_store(sf)
|
||||
register_llmwiki_store(app.state.llmwiki_store)
|
||||
|
||||
# DeerFlow-local Wiki mirror/vector index. The store is always wired so
|
||||
# migrations and status inspection work while feature flags are off;
|
||||
# network/search services only exist when explicitly enabled.
|
||||
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
|
||||
from deerflow.persistence.llmwiki_index import make_llmwiki_index_store
|
||||
|
||||
local_index_config = config.llmwiki.local_wiki_index
|
||||
app.state.llmwiki_index_store = make_llmwiki_index_store(
|
||||
sf,
|
||||
write_batch_size=local_index_config.database_batch_size,
|
||||
)
|
||||
app.state.llmwiki_embedding = None
|
||||
app.state.llmwiki_vector_cache = None
|
||||
app.state.llmwiki_vector_search = None
|
||||
app.state.llmwiki_sync_service = None
|
||||
app.state.llmwiki_retrieval_service = None
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
|
||||
resolved_llmwiki = get_resolved_llmwiki_runtime(config)
|
||||
weknora_client = build_weknora_client(resolved_llmwiki) if resolved_llmwiki.weknora_enabled else None
|
||||
if local_index_config.enabled:
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
|
||||
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(local_index_config.embedding)
|
||||
app.state.llmwiki_vector_cache = WikiVectorCache(
|
||||
app.state.llmwiki_index_store,
|
||||
max_knowledge_bases=local_index_config.cache_max_knowledge_bases,
|
||||
revision_check_seconds=local_index_config.cache_revision_check_seconds,
|
||||
)
|
||||
app.state.llmwiki_vector_search = WikiVectorSearchService(
|
||||
local_index_config,
|
||||
app.state.llmwiki_embedding,
|
||||
app.state.llmwiki_vector_cache,
|
||||
)
|
||||
if weknora_client is not None:
|
||||
app.state.llmwiki_sync_service = WikiSyncService(
|
||||
local_index_config,
|
||||
app.state.llmwiki_index_store,
|
||||
app.state.llmwiki_embedding,
|
||||
app.state.llmwiki_vector_cache,
|
||||
weknora_client,
|
||||
)
|
||||
if weknora_client is not None or app.state.llmwiki_vector_search is not None:
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
|
||||
|
||||
app.state.llmwiki_retrieval_service = WikiRetrievalService(
|
||||
app.state.llmwiki_index_store,
|
||||
vector_search=(
|
||||
app.state.llmwiki_vector_search
|
||||
if local_index_config.internal_search_enabled
|
||||
else None
|
||||
),
|
||||
client=weknora_client,
|
||||
)
|
||||
register_local_wiki_index_runtime(
|
||||
store=app.state.llmwiki_index_store,
|
||||
search=app.state.llmwiki_vector_search,
|
||||
sync=app.state.llmwiki_sync_service,
|
||||
retrieval=app.state.llmwiki_retrieval_service,
|
||||
)
|
||||
app.state.llmwiki_store = make_llmwiki_store(sf)
|
||||
register_llmwiki_store(app.state.llmwiki_store)
|
||||
|
||||
# DeerFlow-local Wiki mirror/vector index. The store is always wired so
|
||||
# migrations and status inspection work while feature flags are off;
|
||||
# network/search services only exist when explicitly enabled.
|
||||
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
|
||||
from deerflow.persistence.llmwiki_index import make_llmwiki_index_store
|
||||
|
||||
local_index_config = config.llmwiki.local_wiki_index
|
||||
app.state.llmwiki_index_store = make_llmwiki_index_store(
|
||||
sf,
|
||||
write_batch_size=local_index_config.database_batch_size,
|
||||
)
|
||||
app.state.llmwiki_embedding = None
|
||||
app.state.llmwiki_vector_cache = None
|
||||
app.state.llmwiki_vector_search = None
|
||||
app.state.llmwiki_sync_service = None
|
||||
app.state.llmwiki_retrieval_service = None
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
|
||||
resolved_llmwiki = get_resolved_llmwiki_runtime(config)
|
||||
weknora_client = build_weknora_client(resolved_llmwiki) if resolved_llmwiki.weknora_enabled else None
|
||||
if local_index_config.enabled:
|
||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
|
||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
|
||||
|
||||
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(local_index_config.embedding)
|
||||
app.state.llmwiki_vector_cache = WikiVectorCache(
|
||||
app.state.llmwiki_index_store,
|
||||
max_knowledge_bases=local_index_config.cache_max_knowledge_bases,
|
||||
revision_check_seconds=local_index_config.cache_revision_check_seconds,
|
||||
)
|
||||
app.state.llmwiki_vector_search = WikiVectorSearchService(
|
||||
local_index_config,
|
||||
app.state.llmwiki_embedding,
|
||||
app.state.llmwiki_vector_cache,
|
||||
)
|
||||
if weknora_client is not None:
|
||||
app.state.llmwiki_sync_service = WikiSyncService(
|
||||
local_index_config,
|
||||
app.state.llmwiki_index_store,
|
||||
app.state.llmwiki_embedding,
|
||||
app.state.llmwiki_vector_cache,
|
||||
weknora_client,
|
||||
)
|
||||
if weknora_client is not None or app.state.llmwiki_vector_search is not None:
|
||||
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
|
||||
|
||||
app.state.llmwiki_retrieval_service = WikiRetrievalService(
|
||||
app.state.llmwiki_index_store,
|
||||
vector_search=(
|
||||
app.state.llmwiki_vector_search
|
||||
if local_index_config.internal_search_enabled
|
||||
else None
|
||||
),
|
||||
client=weknora_client,
|
||||
)
|
||||
register_local_wiki_index_runtime(
|
||||
store=app.state.llmwiki_index_store,
|
||||
search=app.state.llmwiki_vector_search,
|
||||
sync=app.state.llmwiki_sync_service,
|
||||
retrieval=app.state.llmwiki_retrieval_service,
|
||||
)
|
||||
|
||||
from deerflow.persistence.taskcop_tasks import make_taskcop_task_store
|
||||
|
||||
@@ -555,6 +567,47 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app.state.deep_research_executor = None
|
||||
app.state.deep_research_dispatcher = None
|
||||
|
||||
# ── Workflow Studio runtime: live hub + executor + lease dispatcher ──
|
||||
# Every worker runs a dispatcher; the atomic claim decides ownership and
|
||||
# an expired lease is how a crashed worker's run gets picked back up.
|
||||
workflow_cfg = getattr(config, "workflows", None)
|
||||
if workflow_cfg is not None and workflow_cfg.enabled:
|
||||
from app.gateway.workflow_dispatcher import WorkflowRunDispatcher
|
||||
from app.gateway.workflow_executor import WorkflowRunExecutor
|
||||
from app.gateway.workflow_live_hub import WorkflowLiveHub
|
||||
|
||||
app.state.workflow_live_hub = WorkflowLiveHub()
|
||||
app.state.workflow_executor = WorkflowRunExecutor(
|
||||
app,
|
||||
run_store=app.state.workflow_run_store,
|
||||
event_store=app.state.workflow_event_store,
|
||||
workflow_store=app.state.workflow_store,
|
||||
data_source_store=app.state.workflow_data_source_store,
|
||||
live_publisher=app.state.workflow_live_hub.publish,
|
||||
config=workflow_cfg,
|
||||
)
|
||||
app.state.workflow_dispatcher = WorkflowRunDispatcher(
|
||||
app.state.workflow_run_store,
|
||||
app.state.workflow_executor,
|
||||
event_store=app.state.workflow_event_store,
|
||||
live_publisher=app.state.workflow_live_hub.publish,
|
||||
config=workflow_cfg,
|
||||
)
|
||||
app.state.workflow_dispatcher.start()
|
||||
from app.gateway.workflow_retention import WorkflowRetentionCleaner
|
||||
|
||||
app.state.workflow_retention = WorkflowRetentionCleaner(
|
||||
app.state.workflow_run_store,
|
||||
app.state.workflow_event_store,
|
||||
config=workflow_cfg,
|
||||
)
|
||||
app.state.workflow_retention.start()
|
||||
else:
|
||||
app.state.workflow_live_hub = None
|
||||
app.state.workflow_executor = None
|
||||
app.state.workflow_dispatcher = None
|
||||
app.state.workflow_retention = None
|
||||
|
||||
# Run event store (has its own factory with config-driven backend selection)
|
||||
run_events_config = getattr(config, "run_events", None)
|
||||
app.state.run_event_store = make_run_event_store(run_events_config)
|
||||
@@ -565,16 +618,16 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
app.state.concurrency_gate = make_redis_concurrency_gate(config)
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
|
||||
|
||||
register_local_wiki_index_runtime(store=None, search=None, sync=None, retrieval=None)
|
||||
llmwiki_embedding = getattr(app.state, "llmwiki_embedding", None)
|
||||
if llmwiki_embedding is not None:
|
||||
await llmwiki_embedding.close()
|
||||
from deerflow.persistence.notifications import set_notification_store
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
|
||||
|
||||
register_local_wiki_index_runtime(store=None, search=None, sync=None, retrieval=None)
|
||||
llmwiki_embedding = getattr(app.state, "llmwiki_embedding", None)
|
||||
if llmwiki_embedding is not None:
|
||||
await llmwiki_embedding.close()
|
||||
from deerflow.persistence.notifications import set_notification_store
|
||||
|
||||
set_notification_store(None)
|
||||
dispatcher = getattr(app.state, "roundtable_job_dispatcher", None)
|
||||
@@ -583,6 +636,15 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
dr_dispatcher = getattr(app.state, "deep_research_dispatcher", None)
|
||||
if dr_dispatcher is not None:
|
||||
await dr_dispatcher.stop()
|
||||
wf_dispatcher = getattr(app.state, "workflow_dispatcher", None)
|
||||
if wf_dispatcher is not None:
|
||||
await wf_dispatcher.stop()
|
||||
wf_retention = getattr(app.state, "workflow_retention", None)
|
||||
if wf_retention is not None:
|
||||
await wf_retention.stop()
|
||||
from deerflow.workflows.runtime.sql_runner import dispose_engines
|
||||
|
||||
await dispose_engines()
|
||||
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
|
||||
if queue is not None:
|
||||
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
|
||||
@@ -795,17 +857,17 @@ _cached_local_provider: LocalAuthProvider | None = None
|
||||
_cached_repo: SQLiteUserRepository | None = None
|
||||
|
||||
|
||||
def get_request_access_token(request: Request) -> str | None:
|
||||
"""Extract the caller identity, preferring the explicit Bearer header.
|
||||
|
||||
An iframe or another tab can rewrite the origin-wide cookie, while the
|
||||
frontend deliberately sources its Bearer token from this tab's auth store.
|
||||
"""
|
||||
from app.gateway.auth.token_source import get_explicit_or_session_token
|
||||
|
||||
request_token = get_explicit_or_session_token(request)
|
||||
if request_token:
|
||||
return request_token
|
||||
def get_request_access_token(request: Request) -> str | None:
|
||||
"""Extract the caller identity, preferring the explicit Bearer header.
|
||||
|
||||
An iframe or another tab can rewrite the origin-wide cookie, while the
|
||||
frontend deliberately sources its Bearer token from this tab's auth store.
|
||||
"""
|
||||
from app.gateway.auth.token_source import get_explicit_or_session_token
|
||||
|
||||
request_token = get_explicit_or_session_token(request)
|
||||
if request_token:
|
||||
return request_token
|
||||
|
||||
from app.gateway.proxy_path import app_relative_path
|
||||
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Workflow data sources and HTTP credentials.
|
||||
|
||||
Write-only secrets: the DSN (or the credential header blob) is accepted on
|
||||
create/update, encrypted immediately, and never returned. Reads expose only a
|
||||
masked target so an operator can tell two sources apart.
|
||||
|
||||
Shared sources (``ownerId`` null) are admin-only; a regular user may manage
|
||||
their own.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_optional_user_from_request
|
||||
from app.gateway.workflow_audit import audit_workflow
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.workflows.security.secret_box import SecretKeyMissingError, secrets_available
|
||||
from deerflow.workflows.security.sql_policy import assert_read_only, assert_tables_allowed
|
||||
|
||||
router = APIRouter(prefix="/api/workflows/data-sources", tags=["workflow-data-sources"])
|
||||
|
||||
_HTTP_METHODS = frozenset({"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"})
|
||||
|
||||
|
||||
class DataSourceCreate(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=191)
|
||||
description: str = ""
|
||||
kind: str = Field(default="sql", pattern="^(sql|http)$")
|
||||
dsn: str | None = None
|
||||
headers: dict[str, str] | None = None
|
||||
base_url: str = Field(default="", max_length=2048, alias="baseUrl")
|
||||
allowed_methods: list[str] = Field(default_factory=list, alias="allowedMethods")
|
||||
max_rows: int = Field(default=1000, ge=1, le=100_000, alias="maxRows")
|
||||
allowed_tables: list[str] = Field(default_factory=list, alias="allowedTables")
|
||||
enabled: bool = True
|
||||
shared: bool = False
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class QueryBody(BaseModel):
|
||||
statement: str = Field(min_length=1)
|
||||
|
||||
|
||||
class DataSourceUpdate(BaseModel):
|
||||
name: str | None = Field(default=None, min_length=1, max_length=191)
|
||||
description: str | None = None
|
||||
dsn: str | None = None
|
||||
headers: dict[str, str] | None = None
|
||||
base_url: str | None = Field(default=None, max_length=2048, alias="baseUrl")
|
||||
allowed_methods: list[str] | None = Field(default=None, alias="allowedMethods")
|
||||
max_rows: int | None = Field(default=None, ge=1, le=100_000, alias="maxRows")
|
||||
allowed_tables: list[str] | None = Field(default=None, alias="allowedTables")
|
||||
enabled: bool | None = None
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
if not get_app_config().workflows.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
def _store(request: Request):
|
||||
store = getattr(request.app.state, "workflow_data_source_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow data source store not available")
|
||||
return store
|
||||
|
||||
|
||||
async def _is_admin(request: Request) -> bool:
|
||||
user = await get_optional_user_from_request(request)
|
||||
return user is not None and getattr(user, "system_role", None) == "admin"
|
||||
|
||||
|
||||
def _normalise_http_metadata(
|
||||
base_url: str,
|
||||
allowed_methods: list[str],
|
||||
) -> tuple[str, list[str]]:
|
||||
target = base_url.strip()
|
||||
try:
|
||||
parsed = urlsplit(target)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail="HTTP 接口地址格式无效") from exc
|
||||
if (
|
||||
parsed.scheme not in {"http", "https"}
|
||||
or not parsed.netloc
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="HTTP 接口地址必须是无凭据、无查询参数的 http(s) 地址",
|
||||
)
|
||||
|
||||
normalised_methods: list[str] = []
|
||||
for raw_method in allowed_methods:
|
||||
method = str(raw_method).strip().upper()
|
||||
if method not in _HTTP_METHODS:
|
||||
raise HTTPException(status_code=400, detail=f"不支持的 HTTP 方法:{method}")
|
||||
if method not in normalised_methods:
|
||||
normalised_methods.append(method)
|
||||
if not normalised_methods:
|
||||
raise HTTPException(status_code=400, detail="至少选择一种 HTTP 方法")
|
||||
return target, normalised_methods
|
||||
|
||||
|
||||
def _normalise_headers(headers: dict[str, str] | None) -> dict[str, str]:
|
||||
normalised: dict[str, str] = {}
|
||||
for raw_name, raw_value in (headers or {}).items():
|
||||
name = str(raw_name).strip()
|
||||
value = str(raw_value)
|
||||
if not name or any(char in name + value for char in "\r\n"):
|
||||
raise HTTPException(status_code=400, detail="HTTP 请求头格式无效")
|
||||
normalised[name] = value
|
||||
return normalised
|
||||
|
||||
|
||||
def _secret_from(kind: str, dsn: str | None, headers: dict[str, str] | None) -> str:
|
||||
if kind == "http":
|
||||
return json.dumps({"headers": _normalise_headers(headers)}, ensure_ascii=False)
|
||||
if not dsn:
|
||||
raise HTTPException(status_code=400, detail="SQL 数据源需要提供 dsn")
|
||||
if "://" not in dsn:
|
||||
raise HTTPException(status_code=400, detail="dsn 格式无效")
|
||||
return dsn
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_data_sources(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
rows = await _store(request).list_sources(owner_id=user_id)
|
||||
return {"dataSources": rows, "secretsEncryptionAvailable": secrets_available()}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def create_data_source(request: Request, body: DataSourceCreate) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
is_admin = await _is_admin(request)
|
||||
if body.shared and not is_admin:
|
||||
raise HTTPException(status_code=403, detail="仅管理员可创建共享数据源")
|
||||
secret = _secret_from(body.kind, body.dsn, body.headers)
|
||||
base_url = ""
|
||||
allowed_methods: list[str] = []
|
||||
if body.kind == "http":
|
||||
base_url, allowed_methods = _normalise_http_metadata(
|
||||
body.base_url,
|
||||
body.allowed_methods,
|
||||
)
|
||||
try:
|
||||
created = await _store(request).create_source(
|
||||
{
|
||||
"name": body.name,
|
||||
"description": body.description,
|
||||
"kind": body.kind,
|
||||
"dsn": secret,
|
||||
"base_url": base_url,
|
||||
"allowed_methods": allowed_methods,
|
||||
"max_rows": body.max_rows,
|
||||
"allowed_tables": body.allowed_tables,
|
||||
"enabled": body.enabled,
|
||||
"owner_id": None if body.shared else user_id,
|
||||
"created_by": user_id,
|
||||
}
|
||||
)
|
||||
except SecretKeyMissingError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail={
|
||||
"code": "WORKFLOW_RESOURCE_MISSING",
|
||||
"message": "服务端未配置 WORKFLOW_SECRET_KEY,无法安全保存连接信息",
|
||||
},
|
||||
) from exc
|
||||
audit_workflow("data_source.create", user_id=user_id, dataSourceId=created["id"], kind=body.kind, shared=body.shared)
|
||||
return created
|
||||
|
||||
|
||||
async def _owned(request: Request, source_id: str) -> dict[str, Any]:
|
||||
row = await _store(request).get_source(source_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="数据源不存在")
|
||||
user_id = await get_current_user(request)
|
||||
if row.get("owner_id") == user_id:
|
||||
return row
|
||||
if await _is_admin(request):
|
||||
return row
|
||||
raise HTTPException(status_code=403, detail="无权操作该数据源")
|
||||
|
||||
|
||||
async def _visible(request: Request, source_id: str) -> dict[str, Any]:
|
||||
row = await _store(request).get_source(source_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="数据源不存在")
|
||||
user_id = await get_current_user(request)
|
||||
if row.get("owner_id") in (user_id, None):
|
||||
return row
|
||||
if await _is_admin(request):
|
||||
return row
|
||||
raise HTTPException(status_code=403, detail="无权查看该数据源")
|
||||
|
||||
|
||||
@router.get("/{source_id}")
|
||||
async def get_data_source(request: Request, source_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
return await _visible(request, source_id)
|
||||
|
||||
|
||||
@router.post("/{source_id}/introspect")
|
||||
async def introspect_data_source(request: Request, source_id: str) -> dict[str, Any]:
|
||||
"""Return the configured table allowlist only — never sample rows."""
|
||||
_require_enabled()
|
||||
row = await _visible(request, source_id)
|
||||
tables = list(row.get("allowed_tables") or [])
|
||||
return {
|
||||
"dataSourceId": row["id"],
|
||||
"kind": row.get("kind"),
|
||||
"tables": [{"name": name} for name in tables],
|
||||
"schemas": [],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/validate-query")
|
||||
async def validate_data_source_query(request: Request, source_id: str, body: QueryBody) -> dict[str, Any]:
|
||||
"""Validate SQL against the read-only policy without executing it."""
|
||||
_require_enabled()
|
||||
row = await _visible(request, source_id)
|
||||
try:
|
||||
statement = assert_read_only(body.statement)
|
||||
assert_tables_allowed(statement, list(row.get("allowed_tables") or []))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
if isinstance(exc, WorkflowError):
|
||||
raise HTTPException(status_code=400, detail=exc.to_body().model_dump(by_alias=True)) from exc
|
||||
raise
|
||||
audit_workflow("data_source.validate_query", user_id=await get_current_user(request), dataSourceId=source_id)
|
||||
return {"ok": True, "statement": statement}
|
||||
|
||||
|
||||
@router.put("/{source_id}")
|
||||
async def update_data_source(request: Request, source_id: str, body: DataSourceUpdate) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
row = await _owned(request, source_id)
|
||||
payload = body.model_dump(exclude_unset=True, by_alias=False)
|
||||
kind = str(row.get("kind") or "sql")
|
||||
if body.dsn is not None or body.headers is not None:
|
||||
payload["dsn"] = _secret_from(kind, body.dsn, body.headers)
|
||||
payload.pop("headers", None)
|
||||
if kind == "http":
|
||||
base_url, allowed_methods = _normalise_http_metadata(
|
||||
body.base_url if body.base_url is not None else str(row.get("base_url") or ""),
|
||||
body.allowed_methods
|
||||
if body.allowed_methods is not None
|
||||
else list(row.get("allowed_methods") or []),
|
||||
)
|
||||
payload["base_url"] = base_url
|
||||
payload["allowed_methods"] = allowed_methods
|
||||
else:
|
||||
payload.pop("base_url", None)
|
||||
payload.pop("allowed_methods", None)
|
||||
try:
|
||||
updated = await _store(request).update_source(source_id, payload)
|
||||
except SecretKeyMissingError as exc:
|
||||
raise HTTPException(status_code=503, detail="服务端未配置 WORKFLOW_SECRET_KEY") from exc
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail="数据源不存在")
|
||||
audit_workflow("data_source.update", user_id=await get_current_user(request), dataSourceId=source_id)
|
||||
return updated
|
||||
|
||||
|
||||
@router.delete("/{source_id}")
|
||||
async def delete_data_source(request: Request, source_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _owned(request, source_id)
|
||||
deleted = await _store(request).delete_source(source_id)
|
||||
audit_workflow("data_source.delete", user_id=await get_current_user(request), dataSourceId=source_id)
|
||||
return {"deleted": deleted}
|
||||
@@ -0,0 +1,186 @@
|
||||
"""iframe embedding support: origin allowlist + short-lived signed tickets.
|
||||
|
||||
Scope, stated plainly: a ticket proves *"this workflow may be embedded by this
|
||||
origin, for this user, right now"*. It is **not** an API credential — embedded
|
||||
pages still authenticate their API calls with the normal session, exactly like
|
||||
the rest of the app's iframe surfaces. What the ticket adds is an explicit
|
||||
origin decision that can be audited, plus the ``frame-ancestors`` value the host
|
||||
should send.
|
||||
|
||||
Tickets are stateless HS256 JWTs (so any worker can verify them) with a 60s
|
||||
default TTL. Single-use is enforced best-effort per process via a replay cache;
|
||||
the short TTL is the real protection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import jwt
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user
|
||||
from deerflow.config.app_config import get_app_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/workflows/embed", tags=["workflow-embed"])
|
||||
|
||||
_ALGORITHM = "HS256"
|
||||
_AUDIENCE = "workflow-embed"
|
||||
_replay_cache: dict[str, float] = {}
|
||||
_REPLAY_CACHE_MAX = 4096
|
||||
|
||||
|
||||
class TicketRequest(BaseModel):
|
||||
workflow_id: str = Field(alias="workflowId")
|
||||
origin: str
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class VerifyRequest(BaseModel):
|
||||
ticket: str
|
||||
origin: str | None = None
|
||||
|
||||
|
||||
def _signing_key() -> str:
|
||||
for name in ("WORKFLOW_SECRET_KEY", "DEERFLOW_SECRET_KEY", "JWT_SECRET_KEY"):
|
||||
value = os.environ.get(name)
|
||||
if value and value.strip():
|
||||
return hashlib.sha256(value.strip().encode("utf-8")).hexdigest()
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail={
|
||||
"code": "WORKFLOW_EMBED_TICKET_INVALID",
|
||||
"message": "服务端未配置签名密钥,无法签发嵌入票据",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _normalize_origin(raw: str) -> str:
|
||||
parsed = urlparse(raw.strip())
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_EMBED_ORIGIN_DENIED", "message": "origin 格式无效"},
|
||||
)
|
||||
return f"{parsed.scheme.lower()}://{parsed.netloc.lower()}"
|
||||
|
||||
|
||||
def _assert_origin_allowed(origin: str) -> str:
|
||||
cfg = get_app_config().workflows.embed
|
||||
normalized = _normalize_origin(origin)
|
||||
allowed = {_normalize_origin(o) for o in cfg.allowed_origins if o.strip()}
|
||||
if not allowed:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={
|
||||
"code": "WORKFLOW_EMBED_ORIGIN_DENIED",
|
||||
"message": "未配置任何允许的嵌入来源(workflows.embed.allowed_origins)",
|
||||
},
|
||||
)
|
||||
if normalized not in allowed:
|
||||
logger.warning("workflow embed origin denied: %s", normalized)
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={"code": "WORKFLOW_EMBED_ORIGIN_DENIED", "message": "该来源不允许嵌入"},
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def _prune_replay_cache(now: float) -> None:
|
||||
for key, expiry in list(_replay_cache.items()):
|
||||
if expiry <= now:
|
||||
_replay_cache.pop(key, None)
|
||||
if len(_replay_cache) > _REPLAY_CACHE_MAX:
|
||||
_replay_cache.clear()
|
||||
|
||||
|
||||
@router.post("/tickets")
|
||||
async def issue_ticket(request: Request, body: TicketRequest) -> dict[str, Any]:
|
||||
cfg = get_app_config().workflows
|
||||
if not cfg.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
user_id = await get_current_user(request)
|
||||
origin = _assert_origin_allowed(body.origin)
|
||||
|
||||
definitions = getattr(request.app.state, "workflow_store", None)
|
||||
if definitions is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
definition = await definitions.get_definition(body.workflow_id, include_draft=False)
|
||||
if definition is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
if definition.get("owner_id") != user_id:
|
||||
raise HTTPException(status_code=403, detail={"code": "WORKFLOW_FORBIDDEN", "message": "无权嵌入该工作流"})
|
||||
|
||||
now = int(time.time())
|
||||
ttl = cfg.embed.ticket_ttl_seconds
|
||||
payload = {
|
||||
"aud": _AUDIENCE,
|
||||
"sub": user_id,
|
||||
"wf": body.workflow_id,
|
||||
"org": origin,
|
||||
"iat": now,
|
||||
"exp": now + ttl,
|
||||
"jti": hashlib.sha256(f"{user_id}{body.workflow_id}{origin}{now}".encode()).hexdigest()[:32],
|
||||
}
|
||||
ticket = jwt.encode(payload, _signing_key(), algorithm=_ALGORITHM)
|
||||
logger.info(
|
||||
"workflow embed ticket issued workflow=%s user=%s origin=%s ttl=%ss",
|
||||
body.workflow_id,
|
||||
user_id,
|
||||
origin,
|
||||
ttl,
|
||||
)
|
||||
return {
|
||||
"ticket": ticket,
|
||||
"expiresIn": ttl,
|
||||
"origin": origin,
|
||||
"frameAncestors": sorted({_normalize_origin(o) for o in cfg.embed.allowed_origins if o.strip()}),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/verify")
|
||||
async def verify_ticket(body: VerifyRequest) -> dict[str, Any]:
|
||||
cfg = get_app_config().workflows
|
||||
if not cfg.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
try:
|
||||
payload = jwt.decode(body.ticket, _signing_key(), algorithms=[_ALGORITHM], audience=_AUDIENCE)
|
||||
except jwt.PyJWTError as exc:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail={"code": "WORKFLOW_EMBED_TICKET_INVALID", "message": "票据无效或已过期"},
|
||||
) from exc
|
||||
|
||||
if body.origin and _normalize_origin(body.origin) != payload.get("org"):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={"code": "WORKFLOW_EMBED_ORIGIN_DENIED", "message": "票据与当前来源不匹配"},
|
||||
)
|
||||
|
||||
now = time.time()
|
||||
_prune_replay_cache(now)
|
||||
jti = str(payload.get("jti") or "")
|
||||
if jti and jti in _replay_cache:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail={"code": "WORKFLOW_EMBED_TICKET_INVALID", "message": "票据已被使用"},
|
||||
)
|
||||
if jti:
|
||||
_replay_cache[jti] = float(payload.get("exp") or now + 60)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"workflowId": payload.get("wf"),
|
||||
"userId": payload.get("sub"),
|
||||
"origin": payload.get("org"),
|
||||
"expiresAt": payload.get("exp"),
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Workflow node-type and resource catalog APIs (phase 1)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
|
||||
from app.gateway.deps import get_current_user, get_local_provider
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.persistence.workflows import WorkflowStore
|
||||
from deerflow.workflows.schemas import ALL_NODE_TYPES
|
||||
|
||||
router = APIRouter(prefix="/api/workflows", tags=["workflow-resources"])
|
||||
|
||||
_NODE_META: dict[str, dict[str, str]] = {
|
||||
"start": {"label": "开始", "category": "control"},
|
||||
"output": {"label": "输出", "category": "control"},
|
||||
"agent": {"label": "智能体", "category": "ai"},
|
||||
"skill": {"label": "技能", "category": "ai"},
|
||||
"http": {"label": "HTTP", "category": "data"},
|
||||
"sql_read": {"label": "SQL 只读", "category": "data"},
|
||||
"code": {"label": "代码", "category": "data"},
|
||||
"transform": {"label": "模板/转换", "category": "data"},
|
||||
"condition": {"label": "条件", "category": "control"},
|
||||
"merge": {"label": "合并", "category": "control"},
|
||||
"human_input": {"label": "人工介入", "category": "control"},
|
||||
"subworkflow": {"label": "子工作流", "category": "control"},
|
||||
"loop": {"label": "循环", "category": "control"},
|
||||
}
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
if not get_app_config().workflows.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
def _get_store(request: Request) -> WorkflowStore:
|
||||
store = getattr(request.app.state, "workflow_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
return store
|
||||
|
||||
|
||||
async def _resolve_creator_names(user_ids: set[str]) -> dict[str, str]:
|
||||
"""Resolve resource-owner ids to short display names, best-effort.
|
||||
|
||||
The workflow catalog is a small display projection over the agent and
|
||||
skill stores. Those stores intentionally retain immutable owner ids for
|
||||
access control; this resolver exposes a friendly creator name without
|
||||
changing that ownership model. If an account has since disappeared, the
|
||||
stable id remains a useful audit fallback.
|
||||
"""
|
||||
if not user_ids:
|
||||
return {}
|
||||
try:
|
||||
provider = get_local_provider()
|
||||
except Exception: # noqa: BLE001
|
||||
return {user_id: user_id for user_id in user_ids}
|
||||
|
||||
names: dict[str, str] = {}
|
||||
for user_id in user_ids:
|
||||
try:
|
||||
user = await provider.get_user(user_id)
|
||||
except Exception: # noqa: BLE001
|
||||
user = None
|
||||
email = getattr(user, "email", None) if user is not None else None
|
||||
if isinstance(email, str) and email:
|
||||
names[user_id] = email.split("@", 1)[0] or user_id
|
||||
else:
|
||||
names[user_id] = user_id
|
||||
return names
|
||||
|
||||
|
||||
def _creator_label(owner_id: object, creator_names: dict[str, str]) -> str:
|
||||
"""Return the catalog label for a user-owned or built-in resource."""
|
||||
if owner_id is None or not str(owner_id).strip():
|
||||
return "系统内置"
|
||||
owner_key = str(owner_id)
|
||||
return creator_names.get(owner_key, owner_key)
|
||||
|
||||
|
||||
@router.get("/node-types")
|
||||
async def list_node_types(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await get_current_user(request)
|
||||
items = []
|
||||
for node_type in ALL_NODE_TYPES:
|
||||
meta = _NODE_META.get(node_type, {"label": node_type, "category": "other"})
|
||||
items.append({"type": node_type, **meta})
|
||||
return {"nodeTypes": items}
|
||||
|
||||
|
||||
@router.get("/resources/agents")
|
||||
async def list_agent_resources(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
agent_store = getattr(request.app.state, "agent_store", None)
|
||||
agents: list[dict[str, Any]] = []
|
||||
if agent_store is not None:
|
||||
try:
|
||||
if hasattr(agent_store, "list_visible"):
|
||||
rows = await agent_store.list_visible(user_id)
|
||||
elif hasattr(agent_store, "list_all"):
|
||||
rows = await agent_store.list_all()
|
||||
else:
|
||||
rows = []
|
||||
creator_names = await _resolve_creator_names({str(row["user_id"]) for row in rows or [] if row.get("user_id") is not None})
|
||||
agent_ids = [str(row.get("id") or row.get("agent_id") or "") for row in rows or []]
|
||||
extras_by_agent: dict[str, dict[str, Any]] = {}
|
||||
if hasattr(agent_store, "get_extras_for"):
|
||||
extras_by_agent = await agent_store.get_extras_for([agent_id for agent_id in agent_ids if agent_id])
|
||||
keyword = request.query_params.get("keyword", "").strip().casefold()
|
||||
for row in rows or []:
|
||||
agent_id = str(row.get("id") or row.get("agent_id") or "")
|
||||
name = str(row.get("name") or agent_id)
|
||||
description = str(row.get("description") or "")
|
||||
if keyword and keyword not in f"{agent_id} {name} {description}".casefold():
|
||||
continue
|
||||
cached_skills = (extras_by_agent.get(agent_id) or {}).get("skills") or []
|
||||
agents.append(
|
||||
{
|
||||
"agentId": agent_id,
|
||||
"name": name,
|
||||
"description": description,
|
||||
"skills": [str(skill) for skill in cached_skills],
|
||||
"createdBy": _creator_label(row.get("user_id"), creator_names),
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
agents = []
|
||||
return {"agents": agents}
|
||||
|
||||
|
||||
@router.get("/resources/skills")
|
||||
async def list_skill_resources(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
skills: list[dict[str, Any]] = []
|
||||
try:
|
||||
from deerflow.skills import get_or_new_skill_storage
|
||||
|
||||
skill_store = getattr(request.app.state, "skill_store", None)
|
||||
ownership_by_name: dict[str, dict[str, Any]] = {}
|
||||
if skill_store is not None and hasattr(skill_store, "list_visible"):
|
||||
records = await skill_store.list_visible(user_id)
|
||||
ownership_by_name = {str(record["name"]): record for record in records if record.get("name")}
|
||||
creator_names = await _resolve_creator_names({str(record["owner_user_id"]) for record in ownership_by_name.values() if record.get("owner_user_id") is not None})
|
||||
storage = get_or_new_skill_storage()
|
||||
for skill in storage.load_skills(enabled_only=True) or []:
|
||||
name = getattr(skill, "name", None)
|
||||
if not name:
|
||||
continue
|
||||
record = ownership_by_name.get(str(name), {})
|
||||
skills.append(
|
||||
{
|
||||
"skillId": name,
|
||||
"name": name,
|
||||
"description": getattr(skill, "description", "") or "",
|
||||
# Workflow direct-skill execution remains opt-in. Older
|
||||
# Skill definitions do not advertise this capability and
|
||||
# are therefore correctly shown as agent-only resources.
|
||||
"callable": bool(getattr(skill, "callable", False)),
|
||||
"createdBy": _creator_label(record.get("owner_user_id"), creator_names),
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
skills = []
|
||||
return {"skills": skills}
|
||||
|
||||
|
||||
@router.get("/resources/data-sources")
|
||||
async def list_data_source_resources(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
store = getattr(request.app.state, "workflow_data_source_store", None)
|
||||
if store is None:
|
||||
return {"dataSources": []}
|
||||
rows = await store.list_sources(owner_id=user_id)
|
||||
return {
|
||||
"dataSources": [
|
||||
{
|
||||
"id": row["id"],
|
||||
"dataSourceId": row["id"],
|
||||
"name": row["name"],
|
||||
"description": row.get("description") or "",
|
||||
"kind": row.get("kind"),
|
||||
"engine": row.get("driver") or "",
|
||||
"baseUrl": row.get("base_url") or "",
|
||||
"maskedTarget": row.get("masked_target"),
|
||||
"enabled": row.get("enabled"),
|
||||
"readOnly": row.get("kind") == "sql",
|
||||
"allowedMethods": row.get("allowed_methods") or [],
|
||||
"allowedTables": row.get("allowed_tables") or [],
|
||||
}
|
||||
for row in rows
|
||||
if row.get("enabled", True)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.get("/resources/credentials")
|
||||
async def list_credential_resources(request: Request) -> dict[str, Any]:
|
||||
"""Read-only credential directory: ids and labels only.
|
||||
|
||||
HTTP credentials are ``http``-kind data sources; the encrypted header blob
|
||||
is resolved exclusively by the run executor. This endpoint must never
|
||||
return headers, tokens, DSNs or ciphertext — a 404-free catalog so the
|
||||
canvas credential picker does not fail the whole resource panel.
|
||||
"""
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
store = getattr(request.app.state, "workflow_data_source_store", None)
|
||||
if store is None:
|
||||
return {"credentials": []}
|
||||
rows = await store.list_sources(owner_id=user_id, include_shared=True)
|
||||
return {
|
||||
"credentials": [
|
||||
{
|
||||
"credentialId": row["id"],
|
||||
"name": row["name"],
|
||||
"type": "http_headers",
|
||||
"sourceKind": "http",
|
||||
}
|
||||
for row in rows
|
||||
if str(row.get("kind") or "") == "http" and row.get("enabled", True)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.get("/resources/subworkflows")
|
||||
async def list_subworkflow_resources(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await get_current_user(request)
|
||||
store = _get_store(request)
|
||||
return {"subworkflows": await store.list_published_summaries()}
|
||||
@@ -0,0 +1,700 @@
|
||||
"""Workflow run APIs: start / inspect / cancel / resume / retry / events / SSE stream."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_optional_user_from_request
|
||||
from app.gateway.workflow_audit import audit_workflow
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
from deerflow.workflows.runtime.sink import envelope_from_row
|
||||
from deerflow.workflows.schema_validation import schema_issues
|
||||
from deerflow.workflows.schemas import WorkflowGraph
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/workflows", tags=["workflow-runs"])
|
||||
|
||||
_TERMINAL_STATUSES = ("completed", "failed", "cancelled")
|
||||
_TERMINAL_EVENTS = ("run.completed", "run.failed", "run.cancelled")
|
||||
_REPLAY_PAGE = 200
|
||||
_DB_POLL_SECONDS = 0.8
|
||||
_LIVE_WAIT_SECONDS = 0.25
|
||||
_HEARTBEAT_SECONDS = 15.0
|
||||
_STREAM_DEADLINE_SECONDS = 60 * 60
|
||||
|
||||
|
||||
class StartRunBody(BaseModel):
|
||||
version_id: str | None = Field(default=None, alias="versionId")
|
||||
inputs: dict[str, Any] = Field(default_factory=dict)
|
||||
# 独立 Coze 工作流页早期使用 input/target;保留兼容入口,内部仍只使用 inputs。
|
||||
input: dict[str, Any] | None = None
|
||||
target: str | None = None
|
||||
idempotency_key: str | None = Field(default=None, alias="idempotencyKey", max_length=128)
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
def resolved_inputs(self) -> dict[str, Any]:
|
||||
return self.inputs if self.inputs or self.input is None else self.input
|
||||
|
||||
|
||||
class ResumeRunBody(BaseModel):
|
||||
resume_token: str = Field(alias="resumeToken")
|
||||
node_id: str | None = Field(default=None, alias="nodeId")
|
||||
action: str = "submit"
|
||||
values: dict[str, Any] = Field(default_factory=dict)
|
||||
# 兼容独立页的 ResumeRunRequest:payload/message。
|
||||
payload: dict[str, Any] | None = None
|
||||
message: str | None = None
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
def resolved_values(self) -> dict[str, Any]:
|
||||
return self.values if self.values or self.payload is None else self.payload
|
||||
|
||||
|
||||
# ── shared plumbing ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _cfg():
|
||||
return get_app_config().workflows
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
if not _cfg().enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
def _runs(request: Request) -> WorkflowRunStore:
|
||||
store = getattr(request.app.state, "workflow_run_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow run store not available")
|
||||
return store
|
||||
|
||||
|
||||
def _events(request: Request) -> WorkflowEventStore:
|
||||
store = getattr(request.app.state, "workflow_event_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow event store not available")
|
||||
return store
|
||||
|
||||
|
||||
def _definitions(request: Request):
|
||||
store = getattr(request.app.state, "workflow_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
return store
|
||||
|
||||
|
||||
def _nudge(request: Request) -> None:
|
||||
dispatcher = getattr(request.app.state, "workflow_dispatcher", None)
|
||||
if dispatcher is not None:
|
||||
dispatcher.nudge()
|
||||
|
||||
|
||||
async def _is_admin(request: Request) -> bool:
|
||||
user = await get_optional_user_from_request(request)
|
||||
return user is not None and getattr(user, "system_role", None) == "admin"
|
||||
|
||||
|
||||
async def _authorized_run(request: Request, run_id: str) -> dict[str, Any]:
|
||||
run = await _runs(request).get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail={"code": "WORKFLOW_RUN_NOT_FOUND", "message": "运行不存在"})
|
||||
user_id = await get_current_user(request)
|
||||
if run.get("owner_id") != user_id and not await _is_admin(request):
|
||||
raise HTTPException(status_code=403, detail={"code": "WORKFLOW_FORBIDDEN", "message": "无权访问该运行"})
|
||||
return run
|
||||
|
||||
|
||||
def _public_run(run: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Run view for HTTP clients: no lease internals, no resume token."""
|
||||
return {
|
||||
"id": run["id"],
|
||||
"runId": run["id"],
|
||||
"workflowId": run["workflow_id"],
|
||||
"versionId": run["workflow_version_id"],
|
||||
"status": run["status"],
|
||||
"ownerId": run["owner_id"],
|
||||
"input": run.get("input") or {},
|
||||
"output": run.get("output"),
|
||||
"error": run.get("error"),
|
||||
"pendingInput": run.get("pending_input"),
|
||||
"attempt": run.get("attempt"),
|
||||
"retryOfRunId": run.get("retry_of_run_id"),
|
||||
"startedAt": run.get("started_at"),
|
||||
"finishedAt": run.get("finished_at"),
|
||||
"createdAt": run.get("created_at"),
|
||||
"updatedAt": run.get("updated_at"),
|
||||
# The run row's counter holds the last seq handed out by the event log
|
||||
# (the event store bumps it in the insert transaction); 0 = no events.
|
||||
"lastSeq": int(run.get("next_event_seq") or 0),
|
||||
}
|
||||
|
||||
|
||||
# ── start / list / get / cancel / resume ────────────────────────────────
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/runs")
|
||||
async def start_run(request: Request, workflow_id: str, body: StartRunBody) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
if str(body.target or "").strip().lower() == "draft":
|
||||
# Never silently fall back to the published version when the client
|
||||
# thinks it is exercising the draft — draft snapshots need the explicit
|
||||
# test-run API (not built yet).
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_RUN_UNSUPPORTED",
|
||||
"message": "暂不支持运行草稿,请先发布或指定 versionId",
|
||||
},
|
||||
)
|
||||
definitions = _definitions(request)
|
||||
definition = await definitions.get_definition(workflow_id, include_draft=False)
|
||||
if definition is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
if definition.get("owner_id") != user_id and not await _is_admin(request):
|
||||
raise HTTPException(status_code=403, detail={"code": "WORKFLOW_FORBIDDEN", "message": "无权运行该工作流"})
|
||||
|
||||
version_id = body.version_id
|
||||
if not version_id:
|
||||
versions = await definitions.list_versions(workflow_id)
|
||||
if not versions:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_VERSION_NOT_FOUND", "message": "工作流尚未发布任何版本"},
|
||||
)
|
||||
version_id = versions[0]["id"]
|
||||
version = await definitions.get_version(version_id)
|
||||
if version is None or version.get("workflow_id") != workflow_id:
|
||||
raise HTTPException(status_code=404, detail={"code": "WORKFLOW_VERSION_NOT_FOUND", "message": "版本不存在"})
|
||||
|
||||
graph = WorkflowGraph.model_validate(version.get("graph") or {})
|
||||
inputs = body.resolved_inputs()
|
||||
missing = [key for key in (graph.input_schema.get("required") or []) if inputs.get(str(key)) in (None, "")]
|
||||
if missing:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_INPUT_INVALID",
|
||||
"message": f"缺少必填输入:{'、'.join(str(m) for m in missing)}",
|
||||
"details": {"missing": missing},
|
||||
},
|
||||
)
|
||||
# Full JSON Schema pass so type/enum/nested mistakes fail here with a
|
||||
# structured 400 instead of surfacing mid-run as a node error.
|
||||
schema_errors = schema_issues(inputs, graph.input_schema or {})
|
||||
if schema_errors:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_INPUT_INVALID",
|
||||
"message": "输入不符合 schema:" + ";".join(e["message"] for e in schema_errors[:3]),
|
||||
"details": {"errors": schema_errors},
|
||||
},
|
||||
)
|
||||
|
||||
runs = _runs(request)
|
||||
active = [r for r in await runs.list_runs(owner_id=user_id, limit=200) if r["status"] in ("queued", "running")]
|
||||
if len(active) >= _cfg().max_concurrent_runs_per_user:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail={
|
||||
"code": "WORKFLOW_LIMIT_EXCEEDED",
|
||||
"message": f"同时运行的工作流已达上限({_cfg().max_concurrent_runs_per_user})",
|
||||
},
|
||||
)
|
||||
|
||||
# Scope the idempotency key per workflow so two workflows can share a client key.
|
||||
key = f"{workflow_id}:{body.idempotency_key}" if body.idempotency_key else None
|
||||
run, created = await runs.create_run(
|
||||
{
|
||||
"workflow_id": workflow_id,
|
||||
"workflow_version_id": version_id,
|
||||
"owner_id": user_id,
|
||||
"input": inputs,
|
||||
"idempotency_key": key,
|
||||
"context": {"depth": 0},
|
||||
}
|
||||
)
|
||||
if created:
|
||||
await _events(request).append(
|
||||
run_id=run["id"],
|
||||
workflow_id=workflow_id,
|
||||
version_id=version_id,
|
||||
event_type="run.created",
|
||||
payload={"inputs": inputs},
|
||||
)
|
||||
audit_workflow("run.start", user_id=user_id, workflowId=workflow_id, runId=run["id"], versionId=version_id)
|
||||
_nudge(request)
|
||||
return {**_public_run(run), "created": created}
|
||||
|
||||
|
||||
@router.get("/{workflow_id}/runs")
|
||||
async def list_workflow_runs(
|
||||
request: Request,
|
||||
workflow_id: str,
|
||||
status: str | None = None,
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
owner = None if await _is_admin(request) else user_id
|
||||
rows = await _runs(request).list_runs(workflow_id=workflow_id, owner_id=owner, status=status, limit=limit, offset=offset)
|
||||
return {"runs": [_public_run(r) for r in rows], "hasMore": len(rows) >= limit}
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}")
|
||||
async def get_run(request: Request, run_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
run = await _authorized_run(request, run_id)
|
||||
node_runs = await _runs(request).list_node_runs(run_id, current_only=True)
|
||||
return {
|
||||
**_public_run(run),
|
||||
"nodeRuns": [
|
||||
{
|
||||
"nodeId": r["node_id"],
|
||||
"nodeType": r["node_type"],
|
||||
"status": r["status"],
|
||||
"attempt": r["attempt"],
|
||||
"iteration": r["iteration"],
|
||||
"durationMs": r["duration_ms"],
|
||||
"error": r["error"],
|
||||
"startedAt": r["started_at"],
|
||||
"finishedAt": r["finished_at"],
|
||||
}
|
||||
for r in node_runs
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/artifacts")
|
||||
async def list_run_artifacts(request: Request, run_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _authorized_run(request, run_id)
|
||||
rows = await _runs(request).list_artifacts(run_id)
|
||||
return {
|
||||
"artifacts": [
|
||||
{
|
||||
"artifactId": r["id"],
|
||||
"nodeId": r["node_id"],
|
||||
"name": r["name"],
|
||||
"mimeType": r["mime_type"],
|
||||
"sizeBytes": r["size_bytes"],
|
||||
"preview": r["preview"],
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# ── artifact content / download ─────────────────────────────────────────
|
||||
|
||||
_PREVIEW_MAX_BYTES = 256 * 1024
|
||||
_TEXT_MIME_PREFIXES = ("text/", "application/json", "application/javascript", "application/x-yaml", "application/xml", "application/csv")
|
||||
|
||||
|
||||
def _is_textual(mime_type: str) -> bool:
|
||||
return (mime_type or "").lower().startswith(_TEXT_MIME_PREFIXES)
|
||||
|
||||
|
||||
def _read_prefix(path: str, limit: int) -> tuple[bytes, bool]:
|
||||
"""Read up to ``limit + 1`` bytes; the extra byte detects truncation."""
|
||||
with open(path, "rb") as handle:
|
||||
data = handle.read(limit + 1)
|
||||
return data[:limit], len(data) > limit
|
||||
|
||||
|
||||
def _file_exists(path: str) -> bool:
|
||||
import os
|
||||
|
||||
return bool(path) and os.path.isfile(path)
|
||||
|
||||
|
||||
async def _authorized_artifact(request: Request, run_id: str, artifact_id: str) -> dict[str, Any]:
|
||||
"""Run-level authorization first, then artifact-must-belong-to-run.
|
||||
|
||||
The file path is resolved exclusively from the registered artifact row —
|
||||
clients address artifacts by id and can never pass a sandbox path in.
|
||||
"""
|
||||
await _authorized_run(request, run_id)
|
||||
artifact = await _runs(request).get_artifact(run_id, artifact_id)
|
||||
if artifact is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"code": "WORKFLOW_ARTIFACT_NOT_FOUND", "message": "制品不存在"},
|
||||
)
|
||||
return artifact
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/artifacts/{artifact_id}")
|
||||
async def get_run_artifact_content(request: Request, run_id: str, artifact_id: str) -> dict[str, Any]:
|
||||
"""Size-bounded text preview for file cards; binaries stay download-only."""
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from deerflow.workflows.events import artifact_kind
|
||||
|
||||
_require_enabled()
|
||||
artifact = await _authorized_artifact(request, run_id, artifact_id)
|
||||
mime = str(artifact.get("mime_type") or "application/octet-stream")
|
||||
kind = artifact_kind(mime)
|
||||
base = {
|
||||
"artifactId": artifact["id"],
|
||||
"name": artifact["name"],
|
||||
"kind": kind,
|
||||
"mimeType": mime,
|
||||
"sizeBytes": artifact.get("size_bytes"),
|
||||
}
|
||||
|
||||
path = str(artifact.get("path") or "")
|
||||
file_ok = bool(path) and await run_in_threadpool(_file_exists, path)
|
||||
if file_ok:
|
||||
raw, truncated = (b"", False)
|
||||
if _is_textual(mime):
|
||||
raw, truncated = await run_in_threadpool(_read_prefix, path, _PREVIEW_MAX_BYTES)
|
||||
return {
|
||||
**base,
|
||||
"text": raw.decode("utf-8", errors="replace"),
|
||||
"truncated": truncated,
|
||||
"source": "file",
|
||||
}
|
||||
return {
|
||||
**base,
|
||||
"binary": True,
|
||||
"downloadUrl": f"/api/workflows/runs/{run_id}/artifacts/{artifact_id}/download",
|
||||
"source": "file",
|
||||
}
|
||||
|
||||
preview = artifact.get("preview")
|
||||
if preview:
|
||||
return {**base, "text": str(preview), "truncated": True, "source": "preview"}
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"code": "WORKFLOW_ARTIFACT_CONTENT_UNAVAILABLE", "message": "制品内容不可用(文件已过期)"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/artifacts/{artifact_id}/download")
|
||||
async def download_run_artifact(request: Request, run_id: str, artifact_id: str):
|
||||
from urllib.parse import quote
|
||||
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.responses import FileResponse, Response
|
||||
|
||||
_require_enabled()
|
||||
artifact = await _authorized_artifact(request, run_id, artifact_id)
|
||||
mime = str(artifact.get("mime_type") or "application/octet-stream")
|
||||
name = str(artifact.get("name") or artifact_id)
|
||||
disposition = f"attachment; filename*=UTF-8''{quote(name)}"
|
||||
path = str(artifact.get("path") or "")
|
||||
|
||||
if path and await run_in_threadpool(_file_exists, path):
|
||||
return FileResponse(path, media_type=mime, headers={"Content-Disposition": disposition})
|
||||
preview = artifact.get("preview")
|
||||
if preview:
|
||||
return Response(
|
||||
content=str(preview),
|
||||
media_type=mime if _is_textual(mime) else "text/plain",
|
||||
headers={"Content-Disposition": disposition},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"code": "WORKFLOW_ARTIFACT_CONTENT_UNAVAILABLE", "message": "制品文件已过期,无法下载"},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/cancel")
|
||||
async def cancel_run(request: Request, run_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
run = await _authorized_run(request, run_id)
|
||||
if run["status"] in _TERMINAL_STATUSES:
|
||||
return {**_public_run(run), "cancelled": False}
|
||||
updated = await _runs(request).request_cancel(run_id)
|
||||
if updated is None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "WORKFLOW_RUN_CONFLICT", "message": "当前状态无法取消"},
|
||||
)
|
||||
await _events(request).append(
|
||||
run_id=run_id,
|
||||
workflow_id=run["workflow_id"],
|
||||
version_id=run["workflow_version_id"],
|
||||
event_type="run.cancel_requested",
|
||||
payload={},
|
||||
)
|
||||
_nudge(request)
|
||||
audit_workflow("run.cancel", user_id=await get_current_user(request), runId=run_id, workflowId=run["workflow_id"])
|
||||
return {**_public_run(updated), "cancelled": True}
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/resume")
|
||||
async def resume_run(request: Request, run_id: str, body: ResumeRunBody) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
run = await _authorized_run(request, run_id)
|
||||
if run["status"] != "awaiting_input":
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "WORKFLOW_RUN_NOT_RESUMABLE", "message": "运行当前不在等待人工输入"},
|
||||
)
|
||||
pending = run.get("pending_input") or {}
|
||||
node_id = body.node_id or str(pending.get("nodeId") or "")
|
||||
if not node_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_HUMAN_INPUT_INVALID", "message": "缺少 nodeId"},
|
||||
)
|
||||
|
||||
runs = _runs(request)
|
||||
# Single conditional update: token validation + payload write + token
|
||||
# consumption + status flip all in one statement, so two concurrent resume
|
||||
# requests with the same token cannot interleave — exactly one succeeds and
|
||||
# the loser gets 409 without ever touching the run.
|
||||
resumed = await runs.resume_with_payload(
|
||||
run_id,
|
||||
body.resume_token,
|
||||
{
|
||||
"nodeId": node_id,
|
||||
"action": body.action,
|
||||
"values": body.resolved_values(),
|
||||
**({"message": body.message} if body.message else {}),
|
||||
},
|
||||
context=dict(run.get("context") or {}),
|
||||
)
|
||||
if resumed is None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "WORKFLOW_RESUME_TOKEN_INVALID", "message": "恢复令牌无效或已被使用"},
|
||||
)
|
||||
await _events(request).append(
|
||||
run_id=run_id,
|
||||
workflow_id=run["workflow_id"],
|
||||
version_id=run["workflow_version_id"],
|
||||
event_type="run.resumed",
|
||||
payload={"nodeId": node_id, "action": body.action},
|
||||
node_id=node_id,
|
||||
)
|
||||
_nudge(request)
|
||||
audit_workflow("run.resume", user_id=await get_current_user(request), runId=run_id, nodeId=node_id)
|
||||
return _public_run(resumed)
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/retry", status_code=201)
|
||||
async def retry_run(request: Request, run_id: str) -> dict[str, Any]:
|
||||
"""Start a new run from a failed one.
|
||||
|
||||
Completed node outputs are copied onto the new run so the engine resumes
|
||||
from the first unfinished node. If there is nothing to replay, the new run
|
||||
is a full replay of the same version and inputs, still marked
|
||||
``retry_of_run_id``.
|
||||
"""
|
||||
_require_enabled()
|
||||
run = await _authorized_run(request, run_id)
|
||||
if run["status"] != "failed":
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "WORKFLOW_RUN_NOT_RETRYABLE", "message": "只有失败的运行可以重试"},
|
||||
)
|
||||
|
||||
user_id = await get_current_user(request)
|
||||
runs = _runs(request)
|
||||
active = [r for r in await runs.list_runs(owner_id=user_id, limit=200) if r["status"] in ("queued", "running")]
|
||||
if len(active) >= _cfg().max_concurrent_runs_per_user:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail={
|
||||
"code": "WORKFLOW_LIMIT_EXCEEDED",
|
||||
"message": f"同时运行的工作流已达上限({_cfg().max_concurrent_runs_per_user})",
|
||||
},
|
||||
)
|
||||
|
||||
context = dict(run.get("context") or {})
|
||||
context.pop("resume_payload", None)
|
||||
context["retry_of_run_id"] = run_id
|
||||
new_run, _created = await runs.create_run(
|
||||
{
|
||||
"workflow_id": run["workflow_id"],
|
||||
"workflow_version_id": run["workflow_version_id"],
|
||||
"owner_id": user_id,
|
||||
"input": run.get("input") or {},
|
||||
"retry_of_run_id": run_id,
|
||||
"context": context,
|
||||
}
|
||||
)
|
||||
for row in await runs.list_node_runs(run_id, current_only=True):
|
||||
if row.get("status") != "completed":
|
||||
continue
|
||||
await runs.record_node_run(
|
||||
{
|
||||
"run_id": new_run["id"],
|
||||
"node_id": row["node_id"],
|
||||
"node_type": row.get("node_type") or "unknown",
|
||||
"status": "completed",
|
||||
"attempt": 1,
|
||||
"iteration": int(row.get("iteration") or 0),
|
||||
"output": row.get("output"),
|
||||
"duration_ms": row.get("duration_ms"),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
await _events(request).append(
|
||||
run_id=new_run["id"],
|
||||
workflow_id=new_run["workflow_id"],
|
||||
version_id=new_run["workflow_version_id"],
|
||||
event_type="run.created",
|
||||
payload={"retryOfRunId": run_id, "inputs": new_run.get("input") or {}},
|
||||
)
|
||||
_nudge(request)
|
||||
audit_workflow("run.retry", user_id=user_id, runId=new_run["id"], retryOfRunId=run_id)
|
||||
return {**_public_run(new_run), "created": True}
|
||||
|
||||
|
||||
# ── events / SSE ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/events")
|
||||
async def list_run_events(
|
||||
request: Request,
|
||||
run_id: str,
|
||||
after: int = Query(default=0, ge=0),
|
||||
after_seq: int | None = Query(default=None, alias="after_seq", ge=0),
|
||||
limit: int = Query(default=_REPLAY_PAGE, ge=1, le=1000),
|
||||
) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _authorized_run(request, run_id)
|
||||
cursor = after_seq if after_seq is not None else after
|
||||
rows = await _events(request).list_after(run_id, after_seq=cursor, limit=limit)
|
||||
return {"events": [envelope_from_row(r) for r in rows], "lastSeq": rows[-1]["seq"] if rows else cursor}
|
||||
|
||||
|
||||
def _frame(payload: dict[str, Any]) -> str:
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
return f"id: {payload.get('seq', 0)}\nevent: {payload.get('event', 'message')}\ndata: {body}\n\n"
|
||||
|
||||
|
||||
def _replay_cursor(request: Request, after: int) -> int:
|
||||
"""``Last-Event-ID`` wins over ``?after=``; invalid headers fall back."""
|
||||
raw = request.headers.get("last-event-id")
|
||||
if raw:
|
||||
try:
|
||||
return max(0, int(str(raw).strip()))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return after
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/stream")
|
||||
async def stream_run(
|
||||
request: Request,
|
||||
run_id: str,
|
||||
after: int = Query(default=0, ge=0),
|
||||
after_seq: int | None = Query(default=None, alias="after_seq", ge=0),
|
||||
) -> StreamingResponse:
|
||||
"""Replay from ``after`` then tail live.
|
||||
|
||||
The database is authoritative for ordering: live frames whose ``seq`` we have
|
||||
not yet replayed are held back and picked up by the next DB poll, so a
|
||||
reconnecting client never sees a gap or an out-of-order ``seq``.
|
||||
``Last-Event-ID`` takes priority over ``after``.
|
||||
"""
|
||||
_require_enabled()
|
||||
run = await _authorized_run(request, run_id)
|
||||
events = _events(request)
|
||||
runs = _runs(request)
|
||||
hub = getattr(request.app.state, "workflow_live_hub", None)
|
||||
cursor_start = _replay_cursor(request, after_seq if after_seq is not None else after)
|
||||
|
||||
async def generator():
|
||||
cursor = cursor_start
|
||||
live_queue = hub.subscribe(run_id) if hub is not None else None
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + _STREAM_DEADLINE_SECONDS
|
||||
next_poll = 0.0
|
||||
last_beat = loop.time()
|
||||
try:
|
||||
while loop.time() < deadline:
|
||||
if await request.is_disconnected():
|
||||
return
|
||||
|
||||
if loop.time() >= next_poll:
|
||||
while True:
|
||||
rows = await events.list_after(run_id, after_seq=cursor, limit=_REPLAY_PAGE)
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
yield _frame(envelope_from_row(row))
|
||||
cursor = max(cursor, int(row["seq"]))
|
||||
last_beat = loop.time()
|
||||
if row["event_type"] in _TERMINAL_EVENTS:
|
||||
return
|
||||
if len(rows) < _REPLAY_PAGE:
|
||||
break
|
||||
current = await runs.get_run(run_id)
|
||||
if current is None:
|
||||
return
|
||||
if current["status"] in _TERMINAL_STATUSES:
|
||||
# Terminal state with no terminal event (e.g. a crash
|
||||
# before the event landed): synthesise a closing frame
|
||||
# so the client is never left hanging.
|
||||
yield _frame(
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"runId": run_id,
|
||||
"workflowId": current["workflow_id"],
|
||||
"versionId": current["workflow_version_id"],
|
||||
"seq": cursor,
|
||||
"event": f"run.{current['status']}",
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"data": {
|
||||
"status": current["status"],
|
||||
"output": current.get("output"),
|
||||
"error": current.get("error"),
|
||||
"synthesised": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
return
|
||||
next_poll = loop.time() + _DB_POLL_SECONDS
|
||||
|
||||
if live_queue is not None:
|
||||
try:
|
||||
event = await asyncio.wait_for(live_queue.get(), timeout=_LIVE_WAIT_SECONDS)
|
||||
except TimeoutError:
|
||||
event = None
|
||||
if event is not None and int(event.get("seq") or 0) > cursor:
|
||||
yield _frame(event)
|
||||
cursor = int(event["seq"])
|
||||
last_beat = loop.time()
|
||||
if event.get("event") in _TERMINAL_EVENTS:
|
||||
return
|
||||
else:
|
||||
await asyncio.sleep(_LIVE_WAIT_SECONDS)
|
||||
|
||||
if loop.time() - last_beat >= _HEARTBEAT_SECONDS:
|
||||
yield ": heartbeat\n\n"
|
||||
last_beat = loop.time()
|
||||
finally:
|
||||
if live_queue is not None and hub is not None:
|
||||
hub.unsubscribe(run_id, live_queue)
|
||||
|
||||
return StreamingResponse(
|
||||
generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
"X-Workflow-Run-Status": run["status"],
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,351 @@
|
||||
"""Studio facade: camelCase DTOs for the standalone Coze canvas page.
|
||||
|
||||
Four endpoints in front of the standard workflow store:
|
||||
|
||||
GET /api/workflows/{id}/studio-document
|
||||
PUT /api/workflows/{id}/studio-draft
|
||||
POST /api/workflows/{id}/studio-validate
|
||||
POST /api/workflows/{id}/studio-publish
|
||||
|
||||
Design contract (docs/WORKFLOW_STUDIO_COZE_FRONTEND_ADAPTATION_ZH.md §3-§4):
|
||||
|
||||
- One workflow, two documents. ``canvasSchemaJson`` is an opaque Coze canvas
|
||||
saved for editing; ``executionGraph`` is the WorkflowGraph v1.0 the runtime
|
||||
executes. Both land in the SAME CAS update, so they can never disagree on
|
||||
revision.
|
||||
- The facade only maps DTO naming and delegates to the standard store /
|
||||
validator / publish service — it never forks versioning logic.
|
||||
- Publishing a workflow whose draft has no valid execution graph is a 400, not
|
||||
a silent empty-graph publish.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user
|
||||
from app.gateway.workflow_audit import audit_workflow
|
||||
from app.gateway.workflow_resource_checks import build_resource_catalog, make_resource_checker
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.persistence.workflows import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
from deerflow.workflows.schemas import WorkflowGraph
|
||||
from deerflow.workflows.validator import validate_workflow_graph
|
||||
|
||||
router = APIRouter(prefix="/api/workflows", tags=["workflow-studio"])
|
||||
|
||||
# Canvas document guards: it is stored and replayed as an opaque document, so
|
||||
# the only checks are size / JSON shape / depth / obvious credential keys.
|
||||
_CANVAS_MAX_CHARS = 2_000_000
|
||||
_CANVAS_MAX_DEPTH = 64
|
||||
_CANVAS_SECRET_KEYS = frozenset(
|
||||
{
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
"cookie",
|
||||
"x-api-key",
|
||||
"apikey",
|
||||
"api_key",
|
||||
"secret",
|
||||
"client_secret",
|
||||
"password",
|
||||
"dsn",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"resumetoken",
|
||||
"resume_token",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class StudioDraftBody(BaseModel):
|
||||
expected_revision: int = Field(alias="expectedRevision", ge=0)
|
||||
# Canvas may arrive as a JSON string or an embedded object; both are
|
||||
# canonicalised to compact JSON text before storage.
|
||||
canvas_schema_json: Any = Field(default=None, alias="canvasSchemaJson")
|
||||
execution_graph: dict[str, Any] = Field(alias="executionGraph")
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class StudioValidateBody(BaseModel):
|
||||
execution_graph: dict[str, Any] | None = Field(default=None, alias="executionGraph")
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class StudioPublishBody(BaseModel):
|
||||
expected_revision: int | None = Field(default=None, alias="expectedRevision", ge=0)
|
||||
change_note: str = Field(default="", alias="changeNote", max_length=2000)
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
if not get_app_config().workflows.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
def _get_store(request: Request) -> WorkflowStore:
|
||||
store = getattr(request.app.state, "workflow_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
return store
|
||||
|
||||
|
||||
async def _load_owned(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
"""Owner or admin only — the studio edits the draft, so read == write."""
|
||||
from app.gateway.routers.workflows import _load_owned as _standard_load_owned
|
||||
|
||||
return await _standard_load_owned(request, workflow_id, write=True)
|
||||
|
||||
|
||||
def _limits() -> dict[str, int]:
|
||||
cfg = get_app_config().workflows
|
||||
return {
|
||||
"system_max_steps": cfg.max_steps,
|
||||
"system_max_loop_iterations": cfg.max_loop_iterations,
|
||||
"system_max_parallelism": cfg.max_parallelism,
|
||||
}
|
||||
|
||||
|
||||
def _parse_canvas(raw: Any) -> str:
|
||||
"""Validate + canonicalise the canvas document into compact JSON text."""
|
||||
if raw is None or raw == "":
|
||||
return "{}"
|
||||
if isinstance(raw, str):
|
||||
text = raw
|
||||
try:
|
||||
document = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_CANVAS_INVALID", "message": "画布 JSON 无法解析"},
|
||||
) from exc
|
||||
elif isinstance(raw, dict):
|
||||
document = raw
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_CANVAS_INVALID", "message": "画布必须是 JSON 对象或 JSON 字符串"},
|
||||
)
|
||||
text = json.dumps(document, ensure_ascii=False, separators=(",", ":"))
|
||||
if len(text) > _CANVAS_MAX_CHARS:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_CANVAS_INVALID", "message": f"画布文档超过 {_CANVAS_MAX_CHARS} 字符上限"},
|
||||
)
|
||||
offending = _find_secret_key(document, 0)
|
||||
if offending:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_CANVAS_INVALID",
|
||||
"message": f"画布中出现凭据类字段 {offending},请改用凭据引用",
|
||||
"field": offending,
|
||||
},
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def _find_secret_key(node: Any, depth: int) -> str | None:
|
||||
if depth > _CANVAS_MAX_DEPTH:
|
||||
return "<max-depth-exceeded>"
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
name = str(key).strip().lower()
|
||||
if name in _CANVAS_SECRET_KEYS and value not in (None, "", {}, []):
|
||||
return str(key)
|
||||
found = _find_secret_key(value, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
found = _find_secret_key(item, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _studio_issues(issues: list[Any]) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
for issue in issues:
|
||||
body = issue.model_dump(by_alias=True)
|
||||
details = body.get("details") or {}
|
||||
out.append(
|
||||
{
|
||||
"code": body.get("code"),
|
||||
"nodeId": body.get("nodeId"),
|
||||
"field": details.get("field") if isinstance(details, dict) else None,
|
||||
"message": body.get("message"),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
async def _validate_with_resources(request: Request, workflow_id: str, raw_graph: dict[str, Any], user_id: str) -> list[Any]:
|
||||
catalog = await build_resource_catalog(request, user_id=user_id)
|
||||
checker = make_resource_checker(catalog, config=get_app_config().workflows, current_workflow_id=workflow_id)
|
||||
return validate_workflow_graph(raw_graph, resource_checker=checker, **_limits())
|
||||
|
||||
|
||||
@router.get("/{workflow_id}/studio-document")
|
||||
async def get_studio_document(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id)
|
||||
store = _get_store(request)
|
||||
graph = definition.get("draft_graph") or {}
|
||||
versions = await store.list_versions(workflow_id)
|
||||
return {
|
||||
"workflowId": definition["id"],
|
||||
"name": definition.get("name"),
|
||||
"description": definition.get("description") or "",
|
||||
"status": definition.get("status"),
|
||||
"draftRevision": int(definition.get("draft_revision") or 0),
|
||||
"publishedVersionId": versions[0]["id"] if versions else None,
|
||||
# Canvas stays an opaque JSON document (string); never null.
|
||||
"canvasSchemaJson": definition.get("draft_canvas_schema") or "{}",
|
||||
"executionGraph": graph,
|
||||
"inputSchema": graph.get("inputSchema") or graph.get("input_schema") or {"type": "object", "properties": {}},
|
||||
"outputSchema": graph.get("outputSchema") or graph.get("output_schema") or {"type": "object", "properties": {}},
|
||||
"updatedAt": definition.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
@router.put("/{workflow_id}/studio-draft")
|
||||
async def put_studio_draft(request: Request, workflow_id: str, body: StudioDraftBody) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id)
|
||||
store = _get_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
|
||||
try:
|
||||
WorkflowGraph.model_validate(body.execution_graph)
|
||||
except Exception as exc: # noqa: BLE001 - surface pydantic as 400
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_SCHEMA_INVALID", "message": f"执行图不符合 WorkflowGraph v1.0: {exc}"},
|
||||
) from exc
|
||||
|
||||
canvas_text = _parse_canvas(body.canvas_schema_json)
|
||||
try:
|
||||
saved = await store.save_studio_draft(
|
||||
workflow_id,
|
||||
expected_revision=body.expected_revision,
|
||||
graph=body.execution_graph,
|
||||
canvas_schema=canvas_text,
|
||||
updated_by=user_id,
|
||||
)
|
||||
except WorkflowDraftConflictError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"message": "草稿版本冲突,请刷新后重试或另存副本",
|
||||
"currentRevision": exc.current_revision,
|
||||
},
|
||||
) from exc
|
||||
except WorkflowNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在") from exc
|
||||
audit_workflow("workflow.studio_draft", user_id=user_id, workflowId=workflow_id, revision=saved.get("draft_revision"))
|
||||
return {"revision": int(saved.get("draft_revision") or 0), "updatedAt": saved.get("updated_at")}
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/studio-validate")
|
||||
async def studio_validate(request: Request, workflow_id: str, body: StudioValidateBody | None = None) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id)
|
||||
user_id = await get_current_user(request)
|
||||
graph = (body.execution_graph if body and body.execution_graph is not None else definition.get("draft_graph")) or {}
|
||||
try:
|
||||
issues = await _validate_with_resources(request, workflow_id, graph, user_id)
|
||||
except Exception as exc: # noqa: BLE001 - unparsable graph is a validation result, not a 500
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_SCHEMA_INVALID", "message": f"图结构无效: {exc}"},
|
||||
) from exc
|
||||
valid = not issues
|
||||
graph_hash = None
|
||||
if valid:
|
||||
try:
|
||||
graph_hash = WorkflowGraph.model_validate(graph).graph_hash()
|
||||
except Exception: # noqa: BLE001
|
||||
valid = False
|
||||
return {"valid": valid, "issues": _studio_issues(issues), "graphHash": graph_hash}
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/studio-publish")
|
||||
async def studio_publish(request: Request, workflow_id: str, body: StudioPublishBody | None = None) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id)
|
||||
store = _get_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
body = body or StudioPublishBody()
|
||||
|
||||
graph_raw = definition.get("draft_graph") or {}
|
||||
if not graph_raw or not (graph_raw.get("nodes") or []):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_SCHEMA_INVALID",
|
||||
"message": "草稿没有可发布的执行图(只有画布文档不能发布)",
|
||||
},
|
||||
)
|
||||
try:
|
||||
issues = await _validate_with_resources(request, workflow_id, graph_raw, user_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_SCHEMA_INVALID", "message": f"图结构无效: {exc}"},
|
||||
) from exc
|
||||
if issues:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_SCHEMA_INVALID",
|
||||
"message": "发布校验未通过",
|
||||
"issues": _studio_issues(issues),
|
||||
},
|
||||
)
|
||||
|
||||
if body.expected_revision is not None and body.expected_revision != definition.get("draft_revision"):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"message": "发布前草稿版本已变化",
|
||||
"currentRevision": definition.get("draft_revision"),
|
||||
},
|
||||
)
|
||||
|
||||
graph = WorkflowGraph.model_validate(graph_raw)
|
||||
canvas_text = str(definition.get("draft_canvas_schema") or "{}")
|
||||
canvas_hash = sha256(canvas_text.encode("utf-8")).hexdigest()
|
||||
version = await store.publish_version(
|
||||
workflow_id,
|
||||
graph=graph.model_dump(by_alias=True),
|
||||
graph_hash=graph.graph_hash(),
|
||||
published_by=user_id,
|
||||
change_note=body.change_note,
|
||||
canvas_schema_hash=canvas_hash,
|
||||
)
|
||||
audit_workflow(
|
||||
"workflow.studio_publish",
|
||||
user_id=user_id,
|
||||
workflowId=workflow_id,
|
||||
versionId=version.get("id"),
|
||||
graphHash=graph.graph_hash(),
|
||||
)
|
||||
return {
|
||||
"versionId": version.get("id"),
|
||||
"versionNumber": version.get("version_number"),
|
||||
"publishedAt": version.get("published_at"),
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Workflow definition / draft / publish API (phase 1 — no execution yet)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_optional_user_from_request
|
||||
from app.gateway.workflow_audit import audit_workflow
|
||||
from app.gateway.workflow_resource_checks import build_resource_catalog, make_resource_checker
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.persistence.workflows import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
from deerflow.workflows.schemas import WorkflowGraph
|
||||
from deerflow.workflows.validator import validate_workflow_graph
|
||||
|
||||
router = APIRouter(prefix="/api/workflows", tags=["workflows"])
|
||||
|
||||
|
||||
class WorkflowCreateRequest(BaseModel):
|
||||
name: str = Field(default="未命名工作流", min_length=1, max_length=255)
|
||||
description: str = ""
|
||||
graph: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class WorkflowPatchRequest(BaseModel):
|
||||
name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
description: str | None = None
|
||||
|
||||
|
||||
class DraftPutRequest(BaseModel):
|
||||
expected_revision: int = Field(alias="expectedRevision", ge=0)
|
||||
graph: dict[str, Any]
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class PublishRequest(BaseModel):
|
||||
change_note: str = Field(default="", alias="changeNote")
|
||||
expected_revision: int | None = Field(default=None, alias="expectedRevision")
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class ValidateRequest(BaseModel):
|
||||
graph: dict[str, Any] | None = None
|
||||
|
||||
|
||||
def _get_store(request: Request) -> WorkflowStore:
|
||||
store = getattr(request.app.state, "workflow_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
return store
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
cfg = get_app_config().workflows
|
||||
if not cfg.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
async def _current_user_id(request: Request) -> str:
|
||||
return await get_current_user(request)
|
||||
|
||||
|
||||
def _can_access(definition: dict[str, Any], user_id: str, *, write: bool = False) -> bool:
|
||||
# Phase 1: owner-only. Shared/org ACLs land later.
|
||||
owner = definition.get("owner_id")
|
||||
if owner == user_id:
|
||||
return True
|
||||
# Auth disabled environments often use "default" / anonymous — still owner-scoped.
|
||||
if write:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
async def _load_owned(request: Request, workflow_id: str, *, write: bool = False) -> dict[str, Any]:
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
definition = await store.get_definition(workflow_id, include_draft=True)
|
||||
if definition is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
if not _can_access(definition, user_id, write=write):
|
||||
# Soft admin bypass for list/read when system_role=admin
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is not None and getattr(user, "system_role", None) == "admin" and not write:
|
||||
return definition
|
||||
if user is not None and getattr(user, "system_role", None) == "admin" and write:
|
||||
return definition
|
||||
raise HTTPException(status_code=403, detail="无权访问该工作流")
|
||||
return definition
|
||||
|
||||
|
||||
def _limits() -> dict[str, int]:
|
||||
cfg = get_app_config().workflows
|
||||
return {
|
||||
"system_max_steps": cfg.max_steps,
|
||||
"system_max_loop_iterations": cfg.max_loop_iterations,
|
||||
"system_max_parallelism": cfg.max_parallelism,
|
||||
}
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_workflows(
|
||||
request: Request,
|
||||
include_archived: bool = Query(False, alias="includeArchived"),
|
||||
status: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
user = await get_optional_user_from_request(request)
|
||||
is_admin = user is not None and getattr(user, "system_role", None) == "admin"
|
||||
rows = await store.list_definitions(
|
||||
owner_id=None if is_admin else user_id,
|
||||
status=status,
|
||||
include_archived=include_archived,
|
||||
)
|
||||
if is_admin:
|
||||
# Still hide other users' drafts from non-explicit admin listing? Phase 1: admin sees all.
|
||||
pass
|
||||
else:
|
||||
rows = [r for r in rows if r.get("owner_id") == user_id]
|
||||
return {"workflows": rows}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def create_workflow(request: Request, body: WorkflowCreateRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
created = await store.create_definition(
|
||||
{
|
||||
"name": body.name,
|
||||
"description": body.description,
|
||||
"owner_id": user_id,
|
||||
"draft_graph": body.graph,
|
||||
}
|
||||
)
|
||||
audit_workflow("workflow.create", user_id=user_id, workflowId=created["id"])
|
||||
return created
|
||||
|
||||
|
||||
@router.get("/{workflow_id}")
|
||||
async def get_workflow(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
return await _load_owned(request, workflow_id, write=False)
|
||||
|
||||
|
||||
@router.patch("/{workflow_id}")
|
||||
async def patch_workflow(request: Request, workflow_id: str, body: WorkflowPatchRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id, write=True)
|
||||
store = _get_store(request)
|
||||
updated = await store.update_definition(
|
||||
workflow_id,
|
||||
body.model_dump(exclude_unset=True),
|
||||
)
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
return updated
|
||||
|
||||
|
||||
@router.delete("/{workflow_id}")
|
||||
async def archive_workflow(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id, write=True)
|
||||
store = _get_store(request)
|
||||
archived = await store.archive_definition(workflow_id)
|
||||
if archived is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
audit_workflow("workflow.archive", user_id=await _current_user_id(request), workflowId=workflow_id)
|
||||
return archived
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/copy")
|
||||
async def copy_workflow(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id, write=False)
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
created = await store.create_definition(
|
||||
{
|
||||
"name": f"{definition.get('name') or '未命名工作流'} 副本",
|
||||
"description": definition.get("description") or "",
|
||||
"owner_id": user_id,
|
||||
"draft_graph": definition.get("draft_graph"),
|
||||
}
|
||||
)
|
||||
audit_workflow("workflow.copy", user_id=user_id, workflowId=created["id"], sourceWorkflowId=workflow_id)
|
||||
return created
|
||||
|
||||
|
||||
@router.put("/{workflow_id}/draft")
|
||||
async def put_draft(request: Request, workflow_id: str, body: DraftPutRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id, write=True)
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
try:
|
||||
# Light structural parse so obviously broken drafts fail early; full publish
|
||||
# checks run on /validate and /publish.
|
||||
WorkflowGraph.model_validate(body.graph)
|
||||
except Exception as exc: # noqa: BLE001 — surface pydantic as 400
|
||||
raise HTTPException(status_code=400, detail=f"图结构无效: {exc}") from exc
|
||||
try:
|
||||
return await store.save_draft(
|
||||
workflow_id,
|
||||
expected_revision=body.expected_revision,
|
||||
graph=body.graph,
|
||||
updated_by=user_id,
|
||||
)
|
||||
except WorkflowDraftConflictError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"message": "草稿版本冲突,请刷新后重试或另存副本",
|
||||
"currentRevision": exc.current_revision,
|
||||
},
|
||||
) from exc
|
||||
except WorkflowNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在") from exc
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/validate")
|
||||
async def validate_workflow(request: Request, workflow_id: str, body: ValidateRequest | None = None) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id, write=False)
|
||||
graph = (body.graph if body and body.graph is not None else definition.get("draft_graph")) or {}
|
||||
issues = validate_workflow_graph(graph, **_limits())
|
||||
return {
|
||||
"ok": len(issues) == 0,
|
||||
"issues": [i.model_dump(by_alias=True) for i in issues],
|
||||
"graphHash": WorkflowGraph.model_validate(graph).graph_hash() if not issues else None,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/publish")
|
||||
async def publish_workflow(request: Request, workflow_id: str, body: PublishRequest | None = None) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id, write=True)
|
||||
body = body or PublishRequest()
|
||||
if body.expected_revision is not None and body.expected_revision != definition.get("draft_revision"):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"message": "发布前草稿版本已变化",
|
||||
"currentRevision": definition.get("draft_revision"),
|
||||
},
|
||||
)
|
||||
graph_raw = definition.get("draft_graph") or {}
|
||||
# Same resource/permission checks as the studio facade (§5.3): the standard
|
||||
# publish path must not accept graphs referencing invisible agents or
|
||||
# forbidden resources.
|
||||
catalog = await build_resource_catalog(request, user_id=await _current_user_id(request))
|
||||
checker = make_resource_checker(catalog, config=get_app_config().workflows, current_workflow_id=workflow_id)
|
||||
issues = validate_workflow_graph(graph_raw, resource_checker=checker, **_limits())
|
||||
if issues:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_SCHEMA_INVALID",
|
||||
"message": "发布校验未通过",
|
||||
"issues": [i.model_dump(by_alias=True) for i in issues],
|
||||
},
|
||||
)
|
||||
graph = WorkflowGraph.model_validate(graph_raw)
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
canvas_text = str(definition.get("draft_canvas_schema") or "{}")
|
||||
version = await store.publish_version(
|
||||
workflow_id,
|
||||
graph=graph.model_dump(by_alias=True),
|
||||
graph_hash=graph.graph_hash(),
|
||||
published_by=user_id,
|
||||
change_note=body.change_note,
|
||||
canvas_schema_hash=sha256(canvas_text.encode("utf-8")).hexdigest(),
|
||||
)
|
||||
audit_workflow(
|
||||
"workflow.publish",
|
||||
user_id=user_id,
|
||||
workflowId=workflow_id,
|
||||
versionId=version.get("id"),
|
||||
graphHash=graph.graph_hash(),
|
||||
)
|
||||
return version
|
||||
|
||||
|
||||
@router.get("/{workflow_id}/versions")
|
||||
async def list_versions(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id, write=False)
|
||||
store = _get_store(request)
|
||||
return {"versions": await store.list_versions(workflow_id)}
|
||||
|
||||
|
||||
@router.get("/{workflow_id}/versions/{version_id}")
|
||||
async def get_version(request: Request, workflow_id: str, version_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id, write=False)
|
||||
store = _get_store(request)
|
||||
version = await store.get_version(version_id)
|
||||
if version is None or version.get("workflow_id") != workflow_id:
|
||||
raise HTTPException(status_code=404, detail="版本不存在")
|
||||
return version
|
||||
@@ -0,0 +1,368 @@
|
||||
"""Minimal Coze Studio workflow_api compatibility layer (phase 1).
|
||||
|
||||
Internal services always use DeerFlow draft DTOs. This module only translates
|
||||
request/response shapes for the standalone canvas page: ``/canvas`` replies
|
||||
with the Coze playground envelope and ``/save`` persists the Coze canvas into
|
||||
``save_studio_draft`` without letting it anywhere near ``WorkflowGraph``
|
||||
validation (the canvas is an opaque editing document; the executable graph is
|
||||
compiled client-side and saved via the studio facade).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_optional_user_from_request
|
||||
from app.gateway.routers.workflow_studio import _parse_canvas
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.persistence.workflows import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/workflow_api", tags=["workflow-coze-compat"])
|
||||
# Playground 启动还会打这些 Coze 壳接口;缺了它们画布和节点资源都不会挂载。
|
||||
chrome_router = APIRouter(tags=["workflow-coze-compat"])
|
||||
|
||||
_DEFAULT_SPACE: dict[str, Any] = {
|
||||
"id": "1",
|
||||
"space_id": 1,
|
||||
"name": "默认空间",
|
||||
"description": "",
|
||||
# SpaceType.Personal = 1;Playground 找不到个人空间会去调 space/save。
|
||||
"space_type": 1,
|
||||
"space_mode": 0,
|
||||
"icon_uri": "",
|
||||
"icon_url": "",
|
||||
"owner_id": "1",
|
||||
"owner_user_id": "1",
|
||||
"role_type": 1,
|
||||
"has_permission": True,
|
||||
}
|
||||
|
||||
# Coze playground node_template_list payload. The standalone canvas maps
|
||||
# DeerFlow node kinds onto these StandardNodeType ids (see the frontend's
|
||||
# node-catalog.ts); entries keep the Coze field names (node_type / icon_url /
|
||||
# color) so the playground's template list renders without adaptation.
|
||||
_COZE_NODE_TEMPLATES: list[dict[str, Any]] = [
|
||||
{"node_type": "1", "name": "开始", "icon_url": "", "color": "#5C62FF", "desc": "工作流入口,定义输入参数"},
|
||||
{"node_type": "2", "name": "结束", "icon_url": "", "color": "#5C62FF", "desc": "工作流出口,汇总输出"},
|
||||
{"node_type": "3", "name": "大模型", "icon_url": "", "color": "#5C62FF", "desc": "调用智能体处理任务", "support_batch": 1},
|
||||
{"node_type": "5", "name": "代码", "icon_url": "", "color": "#00B2B2", "desc": "在沙箱中执行代码", "support_batch": 0},
|
||||
{"node_type": "8", "name": "选择器", "icon_url": "", "color": "#00B2B2", "desc": "按条件分支执行", "support_batch": 0},
|
||||
{"node_type": "9", "name": "工作流", "icon_url": "", "color": "#00B83E", "desc": "调用已发布子工作流", "support_batch": 1},
|
||||
{"node_type": "13", "name": "输出", "icon_url": "", "color": "#5C62FF", "desc": "定义输出字段", "support_batch": 0},
|
||||
{"node_type": "15", "name": "文本处理", "icon_url": "", "color": "#3071F2", "desc": "模板/文本转换", "support_batch": 0},
|
||||
{"node_type": "18", "name": "问答", "icon_url": "", "color": "#3071F2", "desc": "人工介入确认", "support_batch": 0},
|
||||
{"node_type": "21", "name": "循环", "icon_url": "", "color": "#00B2B2", "desc": "循环执行子图", "support_batch": 0},
|
||||
{"node_type": "32", "name": "变量聚合", "icon_url": "", "color": "#00B2B2", "desc": "合并多分支变量", "support_batch": 0},
|
||||
{"node_type": "43", "name": "查询数据", "icon_url": "", "color": "#F2B600", "desc": "SQL 只读查询", "support_batch": 0},
|
||||
{"node_type": "45", "name": "HTTP 请求", "icon_url": "", "color": "#3071F2", "desc": "调用外部接口", "support_batch": 0},
|
||||
]
|
||||
|
||||
_COZE_NODE_CATEGORIES: list[dict[str, Any]] = [
|
||||
{"name": "", "node_type_list": ["3", "9"]},
|
||||
{"name": "业务逻辑", "node_type_list": ["5", "8", "21", "32"]},
|
||||
{"name": "输入&输出", "node_type_list": ["1", "2", "13"]},
|
||||
{"name": "组件", "node_type_list": ["15", "18", "45"]},
|
||||
{"name": "数据库", "node_type_list": ["43"]},
|
||||
]
|
||||
|
||||
# Mirrors the standalone page's built-in empty canvas (start → end), used when
|
||||
# a workflow has never been opened in the Coze editor.
|
||||
_DEFAULT_CANVAS_SCHEMA = json.dumps(
|
||||
{
|
||||
"nodes": [
|
||||
{
|
||||
"id": "100001",
|
||||
"type": "1",
|
||||
"meta": {"position": {"x": 0, "y": 0}},
|
||||
"data": {"outputs": [{"type": "string", "name": "", "required": True}]},
|
||||
},
|
||||
{
|
||||
"id": "900001",
|
||||
"type": "2",
|
||||
"meta": {"position": {"x": 1000, "y": 0}},
|
||||
},
|
||||
],
|
||||
"edges": [{"sourceNodeID": "100001", "targetNodeID": "900001"}],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
|
||||
class CozeCreateRequest(BaseModel):
|
||||
name: str = Field(default="未命名工作流")
|
||||
description: str = ""
|
||||
graph: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class CozeCanvasRequest(BaseModel):
|
||||
workflow_id: str | None = Field(default=None, alias="workflowId")
|
||||
workflow_id_snake: str | None = Field(default=None, alias="workflow_id")
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
def resolved_id(self) -> str | None:
|
||||
return self.workflow_id or self.workflow_id_snake
|
||||
|
||||
|
||||
class CozeSaveRequest(BaseModel):
|
||||
workflow_id: str | None = Field(default=None, alias="workflowId")
|
||||
workflow_id_snake: str | None = Field(default=None, alias="workflow_id")
|
||||
expected_revision: int | None = Field(default=None, alias="expectedRevision")
|
||||
expected_revision_snake: int | None = Field(default=None, alias="expected_revision")
|
||||
# Coze posts the canvas as a JSON string under ``schema``.
|
||||
schema_: str | dict[str, Any] | None = Field(default=None, alias="schema")
|
||||
graph: str | dict[str, Any] | None = None
|
||||
data: dict[str, Any] | None = None
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
def resolved_id(self) -> str | None:
|
||||
return self.workflow_id or self.workflow_id_snake
|
||||
|
||||
def resolved_revision(self) -> int | None:
|
||||
if self.expected_revision is not None:
|
||||
return self.expected_revision
|
||||
return self.expected_revision_snake
|
||||
|
||||
def resolved_canvas(self) -> str | dict[str, Any] | None:
|
||||
if self.schema_ is not None:
|
||||
return self.schema_
|
||||
if isinstance(self.graph, (str, dict)):
|
||||
return self.graph
|
||||
if self.data is not None:
|
||||
for key in ("schema", "graph"):
|
||||
value = self.data.get(key)
|
||||
if isinstance(value, (str, dict)):
|
||||
return value
|
||||
return self.data
|
||||
return None
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
if not get_app_config().workflows.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
def _get_store(request: Request) -> WorkflowStore:
|
||||
store = getattr(request.app.state, "workflow_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
return store
|
||||
|
||||
|
||||
def _canvas_schema_text(definition: dict[str, Any]) -> str:
|
||||
raw = definition.get("draft_canvas_schema")
|
||||
if isinstance(raw, str) and raw.strip() not in ("", "{}", "null"):
|
||||
return raw
|
||||
if isinstance(raw, dict) and raw:
|
||||
return json.dumps(raw, ensure_ascii=False)
|
||||
return _DEFAULT_CANVAS_SCHEMA
|
||||
|
||||
|
||||
def _to_coze_canvas(definition: dict[str, Any], *, is_published: bool = False) -> dict[str, Any]:
|
||||
owner = str(definition.get("owner_id") or "1")
|
||||
return {
|
||||
"code": 0,
|
||||
"msg": "success",
|
||||
"data": {
|
||||
"workflow": {
|
||||
"workflow_id": definition["id"],
|
||||
"space_id": "1",
|
||||
"name": definition.get("name"),
|
||||
"desc": definition.get("description") or "",
|
||||
"schema_json": _canvas_schema_text(definition),
|
||||
"status": 1,
|
||||
"collaborator_mode": "Open",
|
||||
"creator": {"user_id": owner, "nickname": owner},
|
||||
"is_published": is_published,
|
||||
},
|
||||
"vcs_data": {"can_edit": True},
|
||||
"db_data": {"status": 1},
|
||||
"operation_info": {},
|
||||
"is_bind_agent": False,
|
||||
"workflow_version": "1",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def _load_owned_definition(request: Request, store: WorkflowStore, workflow_id: str, user_id: str) -> dict[str, Any]:
|
||||
definition = await store.get_definition(workflow_id, include_draft=True)
|
||||
if definition is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
if definition.get("owner_id") == user_id:
|
||||
return definition
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is not None and getattr(user, "system_role", None) == "admin":
|
||||
return definition
|
||||
raise HTTPException(status_code=403, detail="无权访问该工作流")
|
||||
|
||||
|
||||
def _coze_ok(data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"code": 0, "msg": "success", "data": data}
|
||||
|
||||
|
||||
def _account_payload(user: Any, user_id: str) -> dict[str, Any]:
|
||||
email = str(getattr(user, "email", "") or "")
|
||||
nickname = email.split("@", 1)[0] if email else user_id
|
||||
return {
|
||||
"user_id_str": user_id,
|
||||
"user_id": user_id,
|
||||
"nickname": nickname,
|
||||
"name": nickname,
|
||||
"email": email,
|
||||
"avatar_url": "",
|
||||
"login_status": "logined",
|
||||
}
|
||||
|
||||
|
||||
async def _require_studio_user(request: Request) -> tuple[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=401, detail="未登录")
|
||||
user = await get_optional_user_from_request(request)
|
||||
return user_id, user
|
||||
|
||||
|
||||
def _space_list_payload() -> dict[str, Any]:
|
||||
return {
|
||||
"has_personal_space": True,
|
||||
"bot_space_list": [_DEFAULT_SPACE],
|
||||
"team_space_num": 0,
|
||||
"max_team_space_num": 10,
|
||||
"recently_used_space_list": [],
|
||||
}
|
||||
|
||||
|
||||
@chrome_router.api_route("/api/passport/account/info/v2/", methods=["GET", "POST"])
|
||||
async def coze_account_info(request: Request) -> dict[str, Any]:
|
||||
user_id, user = await _require_studio_user(request)
|
||||
return _coze_ok(_account_payload(user, user_id))
|
||||
|
||||
|
||||
@chrome_router.api_route("/api/space/list", methods=["GET", "POST"])
|
||||
async def coze_space_list(request: Request) -> dict[str, Any]:
|
||||
await _require_studio_user(request)
|
||||
payload = _space_list_payload()
|
||||
# developer_api.SpaceList 把列表放在顶层;playground v2 读 data。
|
||||
return {"code": 0, "msg": "success", "data": payload, **payload}
|
||||
|
||||
|
||||
@chrome_router.api_route(
|
||||
"/api/playground_api/space/list",
|
||||
methods=["GET", "POST", "PUT", "PATCH"],
|
||||
)
|
||||
@chrome_router.api_route(
|
||||
"/api/playground_api/space/list/",
|
||||
methods=["GET", "POST", "PUT", "PATCH"],
|
||||
)
|
||||
async def coze_playground_space_list(request: Request) -> dict[str, Any]:
|
||||
await _require_studio_user(request)
|
||||
return _coze_ok(_space_list_payload())
|
||||
|
||||
|
||||
@chrome_router.api_route(
|
||||
"/api/playground_api/space/save",
|
||||
methods=["GET", "POST"],
|
||||
)
|
||||
async def coze_playground_space_save(request: Request) -> dict[str, Any]:
|
||||
await _require_studio_user(request)
|
||||
return _coze_ok({"id": _DEFAULT_SPACE["id"], "check_not_pass": False})
|
||||
|
||||
|
||||
@chrome_router.api_route("/api/bot/get_type_list", methods=["GET", "POST"])
|
||||
async def coze_model_type_list(request: Request) -> dict[str, Any]:
|
||||
await _require_studio_user(request)
|
||||
return _coze_ok({"model_list": []})
|
||||
|
||||
|
||||
@router.post("/create")
|
||||
async def coze_create(request: Request, body: CozeCreateRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
store = _get_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
created = await store.create_definition(
|
||||
{
|
||||
"name": body.name,
|
||||
"description": body.description,
|
||||
"owner_id": user_id,
|
||||
}
|
||||
)
|
||||
return {"code": 0, "msg": "success", "data": {"workflow_id": created["id"]}}
|
||||
|
||||
|
||||
@router.post("/canvas")
|
||||
async def coze_canvas(request: Request, body: CozeCanvasRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
workflow_id = body.resolved_id()
|
||||
if not workflow_id:
|
||||
raise HTTPException(status_code=400, detail="workflow_id required")
|
||||
store = _get_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
definition = await _load_owned_definition(request, store, workflow_id, user_id)
|
||||
versions = await store.list_versions(workflow_id)
|
||||
return _to_coze_canvas(definition, is_published=bool(versions))
|
||||
|
||||
|
||||
@router.post("/save")
|
||||
async def coze_save(request: Request, body: CozeSaveRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
workflow_id = body.resolved_id()
|
||||
if not workflow_id:
|
||||
raise HTTPException(status_code=400, detail="workflow_id required")
|
||||
canvas_raw = body.resolved_canvas()
|
||||
if canvas_raw is None or canvas_raw == "" or canvas_raw == {}:
|
||||
raise HTTPException(status_code=400, detail="schema required")
|
||||
canvas_text = _parse_canvas(canvas_raw)
|
||||
store = _get_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
definition = await _load_owned_definition(request, store, workflow_id, user_id)
|
||||
expected = body.resolved_revision()
|
||||
if expected is None:
|
||||
expected = int(definition.get("draft_revision") or 0)
|
||||
try:
|
||||
await store.save_studio_draft(
|
||||
workflow_id,
|
||||
expected_revision=expected,
|
||||
# The execution graph is owned by the studio facade; the Coze save
|
||||
# only persists the canvas, so the current graph passes through.
|
||||
graph=definition.get("draft_graph") or {},
|
||||
canvas_schema=canvas_text,
|
||||
updated_by=user_id,
|
||||
)
|
||||
except WorkflowDraftConflictError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"currentRevision": exc.current_revision,
|
||||
},
|
||||
) from exc
|
||||
except WorkflowNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在") from exc
|
||||
return {"code": 0, "msg": "success", "data": {}}
|
||||
|
||||
|
||||
@router.post("/node_template_list")
|
||||
async def coze_node_template_list(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await get_current_user(request)
|
||||
# Coze playground envelope: data.template_list / cate_list (§ workflow-playground-context).
|
||||
return {
|
||||
"code": 0,
|
||||
"msg": "success",
|
||||
"data": {
|
||||
"template_list": _COZE_NODE_TEMPLATES,
|
||||
"cate_list": _COZE_NODE_CATEGORIES,
|
||||
"plugin_api_list": [],
|
||||
"plugin_category_list": [],
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Runs a DeerFlow agent for one workflow ``agent`` / ``skill`` node.
|
||||
|
||||
This is the single place where the workflow runtime touches LangGraph. We build
|
||||
the lead agent with :func:`make_lead_agent` (so the node inherits the same
|
||||
middleware chain, SOUL, skills and model resolution as a normal chat turn) and
|
||||
stream ``stream_mode=["values", "messages"]`` so the node can forward token
|
||||
deltas into the workflow event log.
|
||||
|
||||
Each node gets its own thread id (``wf-{run_id}-{node_id}``) unless the node
|
||||
asks for ``shared_run_thread``, which keeps the whole run in one conversation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _extract_text(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict) and block.get("type") == "text":
|
||||
parts.append(str(block.get("text") or ""))
|
||||
return "".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def _thread_id(run_id: str, node_id: str, thread_mode: str) -> str:
|
||||
if thread_mode == "shared_run_thread":
|
||||
return f"wf-{run_id}"
|
||||
return f"wf-{run_id}-{node_id}"
|
||||
|
||||
|
||||
class WorkflowAgentRunner:
|
||||
def __init__(self, app: Any) -> None:
|
||||
self._app = app
|
||||
|
||||
async def run_agent(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
node_id: str,
|
||||
owner_id: str,
|
||||
agent_id: str,
|
||||
prompt: str,
|
||||
skill_names: list[str] | None = None,
|
||||
thread_mode: str = "isolated_per_node",
|
||||
cancel: Any = None,
|
||||
on_delta: Callable[[str], Awaitable[None]] | None = None,
|
||||
on_tool: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
|
||||
model_name: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
from deerflow.agents.lead_agent.agent import make_lead_agent
|
||||
|
||||
thread_id = _thread_id(run_id, node_id, thread_mode)
|
||||
configurable: dict[str, Any] = {
|
||||
"thread_id": thread_id,
|
||||
"agent_id": agent_id,
|
||||
"agent_name": agent_id,
|
||||
"user_id": owner_id,
|
||||
"model_name": model_name or "",
|
||||
# Workflow nodes are non-interactive: no plan mode, no clarification.
|
||||
"is_plan_mode": False,
|
||||
"workflow_run_id": run_id,
|
||||
"workflow_node_id": node_id,
|
||||
}
|
||||
if skill_names:
|
||||
configurable["skill_names"] = list(skill_names)
|
||||
config = {"configurable": configurable, "recursion_limit": 60}
|
||||
|
||||
try:
|
||||
agent = make_lead_agent(config)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("failed to build agent %s for workflow node %s", agent_id, node_id, exc_info=True)
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"引用的智能体不存在或配置无效",
|
||||
node_id=node_id,
|
||||
details={"agentId": agent_id},
|
||||
) from exc
|
||||
|
||||
checkpointer = getattr(self._app.state, "checkpointer", None)
|
||||
if checkpointer is not None:
|
||||
agent.checkpointer = checkpointer
|
||||
store = getattr(self._app.state, "store", None)
|
||||
if store is not None:
|
||||
agent.store = store
|
||||
|
||||
seen_tool_calls: set[str] = set()
|
||||
finished_tool_calls: set[str] = set()
|
||||
tool_names: dict[str, str] = {}
|
||||
final_state: dict[str, Any] | None = None
|
||||
streamed: dict[str, list[str]] = {}
|
||||
last_message_id = ""
|
||||
|
||||
async for mode, chunk in agent.astream(
|
||||
{"messages": [{"role": "user", "content": prompt}]},
|
||||
config=config,
|
||||
stream_mode=["values", "messages"],
|
||||
):
|
||||
if cancel is not None and getattr(cancel, "cancelled", False):
|
||||
raise WorkflowError("WORKFLOW_CANCELLED", "运行已被取消", node_id=node_id)
|
||||
if mode == "values":
|
||||
if isinstance(chunk, dict):
|
||||
final_state = chunk
|
||||
continue
|
||||
if mode != "messages":
|
||||
continue
|
||||
message = chunk[0] if isinstance(chunk, tuple) and chunk else chunk
|
||||
if isinstance(message, ToolMessage):
|
||||
# Tool result for an earlier call: close its step in the event log.
|
||||
call_id = str(getattr(message, "tool_call_id", "") or "")
|
||||
if call_id and call_id not in finished_tool_calls:
|
||||
finished_tool_calls.add(call_id)
|
||||
if on_tool is not None:
|
||||
content = getattr(message, "content", "")
|
||||
if not isinstance(content, str):
|
||||
content = str(content)
|
||||
await on_tool(
|
||||
{
|
||||
"phase": "finished",
|
||||
"id": call_id,
|
||||
"name": tool_names.get(call_id) or getattr(message, "name", "") or "",
|
||||
"ok": getattr(message, "status", None) != "error",
|
||||
"summary": content[:200],
|
||||
}
|
||||
)
|
||||
continue
|
||||
if not isinstance(message, AIMessage):
|
||||
continue
|
||||
text = _extract_text(getattr(message, "content", ""))
|
||||
if text:
|
||||
message_id = str(getattr(message, "id", "") or "")
|
||||
streamed.setdefault(message_id, []).append(text)
|
||||
last_message_id = message_id
|
||||
if on_delta is not None:
|
||||
await on_delta(text)
|
||||
for call in getattr(message, "tool_calls", None) or []:
|
||||
call_id = str(call.get("id") or "")
|
||||
if not call_id or call_id in seen_tool_calls:
|
||||
continue
|
||||
seen_tool_calls.add(call_id)
|
||||
tool_names[call_id] = str(call.get("name") or "")
|
||||
if on_tool is not None:
|
||||
await on_tool(
|
||||
{
|
||||
"phase": "started",
|
||||
"id": call_id,
|
||||
"name": tool_names[call_id],
|
||||
"args": call.get("args"),
|
||||
}
|
||||
)
|
||||
|
||||
final_text = "".join(streamed.get(last_message_id, ())) or self._final_text(final_state)
|
||||
if not final_text.strip():
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_AGENT_FAILED",
|
||||
"智能体没有返回任何内容",
|
||||
retryable=True,
|
||||
node_id=node_id,
|
||||
)
|
||||
return {
|
||||
"text": final_text,
|
||||
"thread_id": thread_id,
|
||||
"messages": [],
|
||||
"artifacts": self._artifacts(final_state),
|
||||
"usage": {},
|
||||
}
|
||||
|
||||
def _final_text(self, state: dict[str, Any] | None) -> str:
|
||||
for message in reversed((state or {}).get("messages") or []):
|
||||
if isinstance(message, AIMessage):
|
||||
text = _extract_text(getattr(message, "content", "")).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
def _artifacts(self, state: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
raw = (state or {}).get("artifacts")
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
for item in raw:
|
||||
if isinstance(item, dict) and item.get("name"):
|
||||
out.append(
|
||||
{
|
||||
"name": str(item.get("name")),
|
||||
"path": str(item.get("path") or ""),
|
||||
"mime_type": str(item.get("mime_type") or item.get("mimeType") or ""),
|
||||
"size_bytes": item.get("size_bytes") or item.get("sizeBytes"),
|
||||
"preview": item.get("preview"),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
__all__ = ["WorkflowAgentRunner"]
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Structured audit lines for Workflow Studio mutating APIs.
|
||||
|
||||
These are operator logs, not a separate audit table: every line is a single
|
||||
JSON object with a stable ``action`` plus the ids needed to reconstruct who
|
||||
did what. Secrets never go in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("deerflow.workflows.audit")
|
||||
|
||||
|
||||
def audit_workflow(action: str, *, user_id: str, **fields: Any) -> None:
|
||||
payload = {"action": action, "userId": user_id, **fields}
|
||||
logger.info("workflow.audit %s", json.dumps(payload, ensure_ascii=False, default=str))
|
||||
|
||||
|
||||
__all__ = ["audit_workflow"]
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Lease-based dispatcher for workflow runs.
|
||||
|
||||
A single background loop scans for claimable runs (``queued``, or ``running``
|
||||
with an expired lease) and hands each successfully claimed run to the executor.
|
||||
Because the claim is an atomic conditional UPDATE, several workers can run this
|
||||
loop concurrently and a run that dies with its worker is picked up by whoever
|
||||
notices the expired lease first — that is the restart-recovery story.
|
||||
|
||||
``nudge()`` is called by the start/resume routes so a fresh run does not wait
|
||||
for the next poll tick.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from app.gateway.workflow_executor import LeaseParams, WorkflowRunExecutor
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
from deerflow.workflows.runtime import PersistingWorkflowEventSink
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_POLL_INTERVAL_SECONDS = 2.0
|
||||
_CLAIM_BATCH = 5
|
||||
|
||||
|
||||
def _worker_id() -> str:
|
||||
return f"{socket.gethostname()[:24]}-{os.getpid()}-{uuid.uuid4().hex[:6]}"
|
||||
|
||||
|
||||
class WorkflowRunDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
store: WorkflowRunStore,
|
||||
executor: WorkflowRunExecutor,
|
||||
*,
|
||||
event_store: WorkflowEventStore | None = None,
|
||||
live_publisher: Any = None,
|
||||
config: WorkflowConfig | None = None,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._executor = executor
|
||||
self._events = event_store
|
||||
self._publish = live_publisher
|
||||
self._config = config or WorkflowConfig()
|
||||
self._worker_id = _worker_id()
|
||||
self._wake = asyncio.Event()
|
||||
self._task: asyncio.Task | None = None
|
||||
self._stopping = False
|
||||
|
||||
@property
|
||||
def worker_id(self) -> str:
|
||||
return self._worker_id
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is not None and not self._task.done():
|
||||
return
|
||||
self._stopping = False
|
||||
self._task = asyncio.create_task(self._loop(), name="workflow-run-dispatcher")
|
||||
logger.info("workflow dispatcher started (worker=%s)", self._worker_id)
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._stopping = True
|
||||
self._wake.set()
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
self._task = None
|
||||
await self._executor.drain()
|
||||
|
||||
def nudge(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
async def _loop(self) -> None:
|
||||
await self._safe_dispatch()
|
||||
while not self._stopping:
|
||||
try:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=_POLL_INTERVAL_SECONDS)
|
||||
except TimeoutError:
|
||||
pass
|
||||
self._wake.clear()
|
||||
if self._stopping:
|
||||
return
|
||||
await self._safe_dispatch()
|
||||
|
||||
async def _safe_dispatch(self) -> None:
|
||||
try:
|
||||
await self.dispatch_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - the loop must survive a bad round
|
||||
logger.exception("workflow dispatch round failed")
|
||||
|
||||
async def dispatch_once(self) -> int:
|
||||
now = datetime.now(UTC)
|
||||
await self._reap_cancel_requested()
|
||||
candidates = await self._store.list_claimable(now=now, limit=_CLAIM_BATCH)
|
||||
if not candidates:
|
||||
return 0
|
||||
ttl = max(30, self._config.lease_ttl_seconds)
|
||||
started = 0
|
||||
for run_id in candidates:
|
||||
row = await self._store.claim_run(
|
||||
run_id,
|
||||
lease_owner=self._worker_id,
|
||||
lease_until=now + timedelta(seconds=ttl),
|
||||
)
|
||||
if row is None:
|
||||
continue
|
||||
if self._executor.start(LeaseParams(run_id=run_id, lease_owner=self._worker_id, attempt=int(row.get("attempt") or 1))):
|
||||
started += 1
|
||||
return started
|
||||
|
||||
async def _reap_cancel_requested(self) -> None:
|
||||
"""Finalize cancels whose worker is gone (no live lease to notice them)."""
|
||||
rows = await self._store.list_runs(status="cancel_requested", limit=20)
|
||||
now = datetime.now(UTC)
|
||||
for row in rows:
|
||||
lease_until = row.get("lease_until")
|
||||
if lease_until:
|
||||
try:
|
||||
if datetime.fromisoformat(str(lease_until)) > now:
|
||||
continue # its worker is alive and will handle the cancel
|
||||
except ValueError:
|
||||
pass
|
||||
finalized = await self._store.finalize_cancel(row["id"])
|
||||
if finalized is not None and self._events is not None:
|
||||
# Nobody owns this run, so the dispatcher owes the stream its
|
||||
# terminal frame — otherwise open SSE connections hang.
|
||||
sink = PersistingWorkflowEventSink(
|
||||
self._events,
|
||||
run_id=row["id"],
|
||||
workflow_id=row["workflow_id"],
|
||||
version_id=row["workflow_version_id"],
|
||||
publish=self._publish,
|
||||
)
|
||||
await sink.emit("run.cancelled", data={"reapedBy": self._worker_id})
|
||||
|
||||
|
||||
__all__ = ["WorkflowRunDispatcher"]
|
||||
@@ -0,0 +1,444 @@
|
||||
"""Executes leased workflow runs and owns every terminal state transition.
|
||||
|
||||
Lifecycle for one lease:
|
||||
|
||||
1. resolve the published version → immutable graph snapshot,
|
||||
2. replay completed node results from ``workflow_node_runs`` (so a re-lease or a
|
||||
resume never re-runs finished work),
|
||||
3. drive :class:`WorkflowEngine` under a run-level timeout with a heartbeat
|
||||
renewing the lease and a watcher polling for ``cancel_requested``,
|
||||
4. write exactly one terminal outcome: ``completed`` / ``failed`` /
|
||||
``cancelled`` / ``awaiting_input``.
|
||||
|
||||
Every exit path emits a matching terminal event, because the SSE endpoint uses
|
||||
those events to decide when to close a stream.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.nodes import build_default_registry
|
||||
from deerflow.workflows.runtime import (
|
||||
CancelToken,
|
||||
PersistingWorkflowEventSink,
|
||||
RunContext,
|
||||
WorkflowEngine,
|
||||
WorkflowPaused,
|
||||
WorkflowRuntimeDeps,
|
||||
)
|
||||
from deerflow.workflows.schemas import WorkflowGraph
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HEARTBEAT_INTERVAL_SECONDS = 15
|
||||
_CANCEL_POLL_SECONDS = 2.0
|
||||
_MAX_SUBWORKFLOW_DEPTH = 3
|
||||
|
||||
|
||||
@dataclass
|
||||
class LeaseParams:
|
||||
run_id: str
|
||||
lease_owner: str
|
||||
attempt: int = 1
|
||||
|
||||
|
||||
class WorkflowRunExecutor:
|
||||
def __init__(
|
||||
self,
|
||||
app: Any,
|
||||
*,
|
||||
run_store: WorkflowRunStore,
|
||||
event_store: WorkflowEventStore,
|
||||
workflow_store: Any,
|
||||
data_source_store: Any = None,
|
||||
live_publisher: Any = None,
|
||||
config: WorkflowConfig | None = None,
|
||||
) -> None:
|
||||
self._app = app
|
||||
self._runs = run_store
|
||||
self._events = event_store
|
||||
self._workflows = workflow_store
|
||||
self._data_sources = data_source_store
|
||||
self._publish = live_publisher
|
||||
self._config = config or WorkflowConfig()
|
||||
self._registry = build_default_registry(self._config)
|
||||
self._tasks: dict[str, asyncio.Task] = {}
|
||||
|
||||
# ── public API ──────────────────────────────────────────────────────
|
||||
def start(self, params: LeaseParams) -> bool:
|
||||
"""Launch a background task for a leased run. Idempotent per run id."""
|
||||
existing = self._tasks.get(params.run_id)
|
||||
if existing is not None and not existing.done():
|
||||
return False
|
||||
task = asyncio.create_task(self._guarded(params), name=f"workflow-run-{params.run_id}")
|
||||
self._tasks[params.run_id] = task
|
||||
task.add_done_callback(lambda _t, rid=params.run_id: self._tasks.pop(rid, None))
|
||||
return True
|
||||
|
||||
async def drain(self, timeout: float = 5.0) -> None:
|
||||
tasks = [t for t in self._tasks.values() if not t.done()]
|
||||
if not tasks:
|
||||
return
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.wait(tasks, timeout=timeout)
|
||||
|
||||
# ── run driver ──────────────────────────────────────────────────────
|
||||
async def _guarded(self, params: LeaseParams) -> None:
|
||||
try:
|
||||
await self._execute_lease(params)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - never let a run kill the dispatcher
|
||||
logger.exception("workflow run %s crashed outside the engine", params.run_id)
|
||||
await self._fail(
|
||||
params,
|
||||
WorkflowError("WORKFLOW_INTERNAL", "运行发生内部错误,请查看服务端日志"),
|
||||
)
|
||||
|
||||
async def _execute_lease(self, params: LeaseParams) -> None:
|
||||
run = await self._runs.get_run(params.run_id)
|
||||
if run is None:
|
||||
return
|
||||
sink = self._sink(run)
|
||||
graph = await self._load_graph(run)
|
||||
if graph is None:
|
||||
await self._fail(
|
||||
params,
|
||||
WorkflowError("WORKFLOW_VERSION_NOT_FOUND", "运行引用的工作流版本不存在"),
|
||||
sink=sink,
|
||||
)
|
||||
return
|
||||
|
||||
cancel = CancelToken()
|
||||
resume_payload = (run.get("context") or {}).get("resume_payload")
|
||||
preloaded, loop_state = await self._replay_state(params.run_id)
|
||||
|
||||
ctx = RunContext(
|
||||
run_id=run["id"],
|
||||
workflow_id=run["workflow_id"],
|
||||
version_id=run["workflow_version_id"],
|
||||
owner_id=run["owner_id"],
|
||||
graph=graph,
|
||||
inputs=run.get("input") or {},
|
||||
deps=self._deps(depth=int((run.get("context") or {}).get("depth") or 0)),
|
||||
cancel=cancel,
|
||||
emit=sink.emit,
|
||||
resume_payload=resume_payload if isinstance(resume_payload, dict) else None,
|
||||
)
|
||||
|
||||
await sink.emit(
|
||||
"run.started" if not preloaded else "run.resumed",
|
||||
data={"attempt": params.attempt, "replayedNodes": sorted(preloaded)},
|
||||
)
|
||||
|
||||
engine = WorkflowEngine(self._registry, run_store=self._runs)
|
||||
main = asyncio.create_task(
|
||||
asyncio.wait_for(
|
||||
engine.run(ctx, preloaded=preloaded, loop_state=loop_state, attempt=params.attempt),
|
||||
timeout=min(graph.settings.run_timeout_seconds, self._config.run_timeout_seconds),
|
||||
)
|
||||
)
|
||||
heartbeat = asyncio.create_task(self._heartbeat(params, main))
|
||||
watcher = asyncio.create_task(self._watch_cancel(params, cancel, main))
|
||||
try:
|
||||
result = await main
|
||||
except WorkflowPaused as paused:
|
||||
await self._pause(params, paused, ctx, sink)
|
||||
return
|
||||
except TimeoutError:
|
||||
await self._fail(
|
||||
params,
|
||||
WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"运行超时({graph.settings.run_timeout_seconds}s)",
|
||||
details={"timeoutSeconds": graph.settings.run_timeout_seconds},
|
||||
),
|
||||
sink=sink,
|
||||
)
|
||||
return
|
||||
except WorkflowError as exc:
|
||||
if exc.code == "WORKFLOW_CANCELLED" or cancel.cancelled:
|
||||
await self._cancel(params, sink)
|
||||
else:
|
||||
await self._fail(params, exc, sink=sink)
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
# Lease lost or process shutting down: leave the run claimable again.
|
||||
await self._runs.update_run(params.run_id, lease_owner=params.lease_owner, status="queued")
|
||||
raise
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
watcher.cancel()
|
||||
for task in (heartbeat, watcher):
|
||||
try:
|
||||
await task
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
|
||||
await self._complete(params, result, sink)
|
||||
|
||||
# ── terminal transitions ────────────────────────────────────────────
|
||||
async def _complete(self, params: LeaseParams, result: Any, sink: Any) -> None:
|
||||
await self._runs.update_run(
|
||||
params.run_id,
|
||||
lease_owner=params.lease_owner,
|
||||
status="completed",
|
||||
output_json=result.output,
|
||||
context_json={"loop_state": result.loop_state, "steps": result.steps},
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
await sink.emit("run.completed", data={"output": result.output, "steps": result.steps})
|
||||
|
||||
async def _fail(self, params: LeaseParams, error: WorkflowError, *, sink: Any = None) -> None:
|
||||
run = await self._runs.get_run(params.run_id)
|
||||
if run is None:
|
||||
return
|
||||
body = error.to_body().model_dump(by_alias=True)
|
||||
retry = error.retryable and int(run.get("attempt") or 1) < int(run.get("max_attempts") or 1)
|
||||
sink = sink or self._sink(run)
|
||||
if retry:
|
||||
await self._runs.update_run(
|
||||
params.run_id,
|
||||
lease_owner=params.lease_owner,
|
||||
status="queued",
|
||||
error_json=body,
|
||||
)
|
||||
await sink.emit("run.queued", data={"retryAfterError": body, "attempt": run.get("attempt")})
|
||||
return
|
||||
await self._runs.update_run(
|
||||
params.run_id,
|
||||
lease_owner=params.lease_owner,
|
||||
status="failed",
|
||||
error_json=body,
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
await sink.emit("run.failed", data={"error": body})
|
||||
|
||||
async def _cancel(self, params: LeaseParams, sink: Any) -> None:
|
||||
await self._runs.request_cancel(params.run_id)
|
||||
finalized = await self._runs.finalize_cancel(params.run_id)
|
||||
if finalized is None:
|
||||
await self._runs.update_run(params.run_id, lease_owner=params.lease_owner, status="cancelled")
|
||||
await sink.emit("run.cancelled", data={})
|
||||
|
||||
async def _pause(self, params: LeaseParams, paused: WorkflowPaused, ctx: RunContext, sink: Any) -> None:
|
||||
token = secrets.token_urlsafe(24)
|
||||
# One conditional update writes status + token + pending descriptor +
|
||||
# run context together; a separate follow-up context write could clobber
|
||||
# a resume that lands in between.
|
||||
updated = await self._runs.set_awaiting_input(
|
||||
params.run_id,
|
||||
lease_owner=params.lease_owner,
|
||||
pending_input=paused.pending_input,
|
||||
resume_token=token,
|
||||
context={"loop_state": dict(ctx.loop_state), "resume_payload": None},
|
||||
)
|
||||
if updated is None:
|
||||
# Lost the lease (cancel or steal) — do not fabricate a pause.
|
||||
logger.info("workflow run %s could not enter awaiting_input", params.run_id)
|
||||
return
|
||||
await sink.emit(
|
||||
"run.awaiting_input",
|
||||
data={
|
||||
# Flattened wire contract: the full descriptor keeps living in
|
||||
# the run row (pending_input), the event exposes exactly what a
|
||||
# client needs to render the intervention card.
|
||||
"resumeToken": token,
|
||||
"prompt": paused.pending_input.get("prompt") or "",
|
||||
"formSchema": paused.pending_input.get("formSchema") or {},
|
||||
"actions": paused.pending_input.get("actions") or ["submit"],
|
||||
},
|
||||
node_id=paused.node_id,
|
||||
)
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────
|
||||
def _sink(self, run: dict[str, Any]) -> PersistingWorkflowEventSink:
|
||||
return PersistingWorkflowEventSink(
|
||||
self._events,
|
||||
run_id=run["id"],
|
||||
workflow_id=run["workflow_id"],
|
||||
version_id=run["workflow_version_id"],
|
||||
publish=self._publish,
|
||||
)
|
||||
|
||||
async def _load_graph(self, run: dict[str, Any]) -> WorkflowGraph | None:
|
||||
version = await self._workflows.get_version(run["workflow_version_id"])
|
||||
if version is None or version.get("workflow_id") != run["workflow_id"]:
|
||||
return None
|
||||
try:
|
||||
return WorkflowGraph.model_validate(version.get("graph") or {})
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("workflow version %s has an invalid graph", run["workflow_version_id"], exc_info=True)
|
||||
return None
|
||||
|
||||
async def _replay_state(self, run_id: str) -> tuple[dict[str, Any], dict[str, int]]:
|
||||
run = await self._runs.get_run(run_id)
|
||||
loop_state = {}
|
||||
raw_loop = (run or {}).get("context", {}).get("loop_state")
|
||||
if isinstance(raw_loop, dict):
|
||||
loop_state = {str(k): int(v) for k, v in raw_loop.items() if isinstance(v, int)}
|
||||
completed: dict[str, Any] = {}
|
||||
for row in await self._runs.list_node_runs(run_id, current_only=True):
|
||||
if row.get("status") != "completed":
|
||||
continue
|
||||
output = row.get("output")
|
||||
if isinstance(output, dict):
|
||||
completed[str(row["node_id"])] = output
|
||||
return completed, loop_state
|
||||
|
||||
def _deps(self, *, depth: int = 0) -> WorkflowRuntimeDeps:
|
||||
from app.gateway.workflow_agent_runner import WorkflowAgentRunner
|
||||
|
||||
agent_runner = WorkflowAgentRunner(self._app)
|
||||
|
||||
async def save_artifact(**kwargs: Any) -> dict[str, Any]:
|
||||
return await self._runs.add_artifact(kwargs)
|
||||
|
||||
async def resolve_data_source(source_id: str) -> dict[str, Any] | None:
|
||||
if self._data_sources is None:
|
||||
return None
|
||||
return await self._data_sources.resolve_dsn(source_id)
|
||||
|
||||
async def resolve_credential(ref: str) -> dict[str, Any] | None:
|
||||
"""HTTP credentials are data-source rows of kind ``http`` whose
|
||||
secret is a JSON blob of headers to inject."""
|
||||
if self._data_sources is None:
|
||||
return None
|
||||
resolved = await self._data_sources.resolve_dsn(ref)
|
||||
if not resolved:
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(resolved.get("dsn") or "{}")
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
headers = payload.get("headers") if isinstance(payload, dict) else None
|
||||
return {"headers": headers} if isinstance(headers, dict) else None
|
||||
|
||||
async def run_subworkflow(**kwargs: Any) -> dict[str, Any]:
|
||||
if depth + 1 > _MAX_SUBWORKFLOW_DEPTH:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_LIMIT_EXCEEDED",
|
||||
f"子工作流嵌套超过 {_MAX_SUBWORKFLOW_DEPTH} 层",
|
||||
details={"depth": depth + 1},
|
||||
)
|
||||
return await self._run_child(depth=depth + 1, **kwargs)
|
||||
|
||||
return WorkflowRuntimeDeps(
|
||||
run_agent=agent_runner.run_agent,
|
||||
save_artifact=save_artifact,
|
||||
resolve_data_source=resolve_data_source,
|
||||
resolve_credential=resolve_credential,
|
||||
run_subworkflow=run_subworkflow,
|
||||
)
|
||||
|
||||
async def _run_child(
|
||||
self,
|
||||
*,
|
||||
depth: int,
|
||||
parent_run_id: str,
|
||||
node_id: str,
|
||||
owner_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
inputs: dict[str, Any],
|
||||
cancel: CancelToken | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run a child workflow inline: same process, its own run row and events."""
|
||||
child, _ = await self._runs.create_run(
|
||||
{
|
||||
"workflow_id": workflow_id,
|
||||
"workflow_version_id": version_id,
|
||||
"owner_id": owner_id,
|
||||
"input": inputs,
|
||||
"status": "running",
|
||||
"context": {
|
||||
"depth": depth,
|
||||
"parent_run_id": parent_run_id,
|
||||
"parent_node_id": node_id,
|
||||
},
|
||||
}
|
||||
)
|
||||
graph = await self._load_graph(child)
|
||||
if graph is None:
|
||||
raise WorkflowError("WORKFLOW_VERSION_NOT_FOUND", "子工作流版本不存在", node_id=node_id)
|
||||
sink = self._sink(child)
|
||||
ctx = RunContext(
|
||||
run_id=child["id"],
|
||||
workflow_id=workflow_id,
|
||||
version_id=version_id,
|
||||
owner_id=owner_id,
|
||||
graph=graph,
|
||||
inputs=inputs,
|
||||
deps=self._deps(depth=depth),
|
||||
cancel=cancel or CancelToken(),
|
||||
emit=sink.emit,
|
||||
)
|
||||
await sink.emit("run.started", data={"parentRunId": parent_run_id, "depth": depth})
|
||||
engine = WorkflowEngine(self._registry, run_store=self._runs)
|
||||
try:
|
||||
result = await engine.run(ctx)
|
||||
except WorkflowPaused as exc:
|
||||
await self._runs.update_run(child["id"], status="failed")
|
||||
await sink.emit(
|
||||
"run.failed",
|
||||
data={
|
||||
"error": {
|
||||
"code": "WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
"message": "子工作流不支持人工节点",
|
||||
}
|
||||
},
|
||||
)
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
"子工作流不支持人工输入节点",
|
||||
node_id=node_id,
|
||||
details={"childRunId": child["id"], "pausedAt": exc.node_id},
|
||||
) from exc
|
||||
except WorkflowError as exc:
|
||||
await self._runs.update_run(child["id"], status="failed", error_json=exc.to_body().model_dump(by_alias=True))
|
||||
await sink.emit("run.failed", data={"error": exc.to_body().model_dump(by_alias=True)})
|
||||
raise
|
||||
await self._runs.update_run(child["id"], status="completed", output_json=result.output)
|
||||
await sink.emit("run.completed", data={"output": result.output})
|
||||
return {"run_id": child["id"], "status": "completed", "output": result.output}
|
||||
|
||||
async def _heartbeat(self, params: LeaseParams, main: asyncio.Task) -> None:
|
||||
ttl = max(30, self._config.lease_ttl_seconds)
|
||||
while not main.done():
|
||||
await asyncio.sleep(_HEARTBEAT_INTERVAL_SECONDS)
|
||||
if main.done():
|
||||
return
|
||||
ok = await self._runs.renew_lease(
|
||||
params.run_id,
|
||||
lease_owner=params.lease_owner,
|
||||
lease_until=datetime.now(UTC) + timedelta(seconds=ttl),
|
||||
)
|
||||
if not ok:
|
||||
logger.warning("workflow run %s lost its lease; aborting", params.run_id)
|
||||
main.cancel()
|
||||
return
|
||||
|
||||
async def _watch_cancel(self, params: LeaseParams, cancel: CancelToken, main: asyncio.Task) -> None:
|
||||
while not main.done():
|
||||
await asyncio.sleep(_CANCEL_POLL_SECONDS)
|
||||
run = await self._runs.get_run(params.run_id)
|
||||
if run is None:
|
||||
return
|
||||
if run.get("status") in ("cancel_requested", "cancelled"):
|
||||
cancel.cancel()
|
||||
return
|
||||
|
||||
|
||||
__all__ = ["LeaseParams", "WorkflowRunExecutor"]
|
||||
@@ -0,0 +1,59 @@
|
||||
"""In-process fan-out of workflow events to connected SSE clients.
|
||||
|
||||
The durable log in ``workflow_run_events`` is the source of truth; this hub only
|
||||
shortens latency for the tail. Slow consumers lose the *oldest* queued frame
|
||||
rather than blocking the executor, and the SSE endpoint reconciles any gap from
|
||||
the database using the ``seq`` cursor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WorkflowLiveHub:
|
||||
def __init__(self, *, queue_size: int = 512) -> None:
|
||||
self._subscribers: dict[str, set[asyncio.Queue[dict[str, Any]]]] = defaultdict(set)
|
||||
self._queue_size = queue_size
|
||||
|
||||
def subscribe(self, run_id: str) -> asyncio.Queue[dict[str, Any]]:
|
||||
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=self._queue_size)
|
||||
self._subscribers[run_id].add(queue)
|
||||
return queue
|
||||
|
||||
def unsubscribe(self, run_id: str, queue: asyncio.Queue[dict[str, Any]]) -> None:
|
||||
subscribers = self._subscribers.get(run_id)
|
||||
if not subscribers:
|
||||
return
|
||||
subscribers.discard(queue)
|
||||
if not subscribers:
|
||||
self._subscribers.pop(run_id, None)
|
||||
|
||||
def subscriber_count(self, run_id: str) -> int:
|
||||
return len(self._subscribers.get(run_id) or ())
|
||||
|
||||
async def publish(self, event: Mapping[str, Any]) -> None:
|
||||
run_id = str(event.get("runId") or "")
|
||||
subscribers = self._subscribers.get(run_id)
|
||||
if not run_id or not subscribers:
|
||||
return
|
||||
payload = dict(event)
|
||||
for queue in list(subscribers):
|
||||
if queue.full():
|
||||
try:
|
||||
queue.get_nowait()
|
||||
except asyncio.QueueEmpty: # pragma: no cover - race with a reader
|
||||
pass
|
||||
try:
|
||||
queue.put_nowait(payload)
|
||||
except asyncio.QueueFull: # pragma: no cover - race with a reader
|
||||
logger.debug("workflow live queue still full for run %s", run_id)
|
||||
|
||||
|
||||
__all__ = ["WorkflowLiveHub"]
|
||||
@@ -0,0 +1,303 @@
|
||||
"""App-layer publish-time resource checks for workflow graphs.
|
||||
|
||||
The harness validator stays store-agnostic via its ``resource_checker`` hook;
|
||||
this module is where the Gateway binds that hook to the real agent / skill /
|
||||
data-source / published-version stores. It pre-fetches the catalogs the caller
|
||||
is allowed to see, then exposes a **sync** checker so it can be passed straight
|
||||
into ``validate_workflow_graph``.
|
||||
|
||||
Never imports from ``packages/harness`` internals beyond the public workflow
|
||||
API, and the harness never imports this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.security.sql_policy import assert_read_only, assert_tables_allowed
|
||||
|
||||
# Header-ish keys that must never be configured as literal values on the canvas:
|
||||
# credentials belong in an http data source referenced by ``credentialRef``.
|
||||
_LITERAL_SECRET_KEYS = frozenset(
|
||||
{
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
"cookie",
|
||||
"x-api-key",
|
||||
"apikey",
|
||||
"api_key",
|
||||
"secret",
|
||||
"client_secret",
|
||||
"password",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
}
|
||||
)
|
||||
|
||||
_HTTP_METHODS = frozenset({"GET", "POST", "PUT", "PATCH", "DELETE"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResourceCatalog:
|
||||
"""Everything the checker is allowed to reference, prefetched."""
|
||||
|
||||
agent_ids: set[str] = field(default_factory=set)
|
||||
# agent_id -> cached skill whitelist from the agent store (empty = no
|
||||
# restriction recorded).
|
||||
agent_skills: dict[str, set[str]] = field(default_factory=dict)
|
||||
enabled_skills: set[str] = field(default_factory=set)
|
||||
# data source id -> public row (kind / enabled / allowed_* / max_rows).
|
||||
data_sources: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
# workflow_id -> set of published version ids.
|
||||
published_subworkflows: dict[str, set[str]] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _issue(message: str, field_path: str) -> dict[str, Any]:
|
||||
return {"message": message, "field": field_path}
|
||||
|
||||
|
||||
async def build_resource_catalog(request: Request, *, user_id: str) -> ResourceCatalog:
|
||||
catalog = ResourceCatalog()
|
||||
|
||||
agent_store = getattr(request.app.state, "agent_store", None)
|
||||
if agent_store is not None:
|
||||
try:
|
||||
if hasattr(agent_store, "list_visible"):
|
||||
rows = await agent_store.list_visible(user_id)
|
||||
elif hasattr(agent_store, "list_all"):
|
||||
rows = await agent_store.list_all()
|
||||
else:
|
||||
rows = []
|
||||
catalog.agent_ids = {
|
||||
str(row.get("id") or row.get("agent_id") or "") for row in rows or [] if row.get("id") or row.get("agent_id")
|
||||
}
|
||||
if hasattr(agent_store, "get_extras_for"):
|
||||
extras = await agent_store.get_extras_for([aid for aid in catalog.agent_ids if aid])
|
||||
for agent_id, extra in (extras or {}).items():
|
||||
skills = (extra or {}).get("skills") or []
|
||||
if skills:
|
||||
catalog.agent_skills[str(agent_id)] = {str(s) for s in skills}
|
||||
except Exception: # noqa: BLE001 - publish checks degrade to strict-by-empty
|
||||
catalog.agent_ids = set()
|
||||
|
||||
try:
|
||||
from deerflow.skills import get_or_new_skill_storage
|
||||
|
||||
catalog.enabled_skills = {
|
||||
str(getattr(skill, "name"))
|
||||
for skill in get_or_new_skill_storage().load_skills(enabled_only=True) or []
|
||||
if getattr(skill, "name", None)
|
||||
}
|
||||
except Exception: # noqa: BLE001
|
||||
catalog.enabled_skills = set()
|
||||
|
||||
ds_store = getattr(request.app.state, "workflow_data_source_store", None)
|
||||
if ds_store is not None:
|
||||
try:
|
||||
for row in await ds_store.list_sources(owner_id=user_id, include_shared=True) or []:
|
||||
catalog.data_sources[str(row.get("id"))] = row
|
||||
except Exception: # noqa: BLE001
|
||||
catalog.data_sources = {}
|
||||
|
||||
wf_store = getattr(request.app.state, "workflow_store", None)
|
||||
if wf_store is not None:
|
||||
try:
|
||||
for row in await wf_store.list_published_summaries() or []:
|
||||
versions = catalog.published_subworkflows.setdefault(str(row.get("workflow_id")), set())
|
||||
versions.add(str(row.get("version_id")))
|
||||
except Exception: # noqa: BLE001
|
||||
catalog.published_subworkflows = {}
|
||||
|
||||
return catalog
|
||||
|
||||
|
||||
def _check_agent(config: dict[str, Any], catalog: ResourceCatalog) -> dict[str, Any] | None:
|
||||
agent_id = str(config.get("agentId") or config.get("agent_id") or "")
|
||||
if not agent_id:
|
||||
return _issue("未选择智能体", "config.agentId")
|
||||
if agent_id not in catalog.agent_ids:
|
||||
return _issue("未选择可访问的智能体", "config.agentId")
|
||||
if not str(config.get("promptTemplate") or config.get("prompt_template") or "").strip():
|
||||
return _issue("智能体节点的提示词不能为空", "config.promptTemplate")
|
||||
mode = str(config.get("responseMode") or config.get("response_mode") or "text")
|
||||
schema = config.get("responseSchema") or config.get("response_schema")
|
||||
if mode == "json":
|
||||
if not isinstance(schema, dict) or not schema:
|
||||
return _issue("JSON 输出模式必须配置 responseSchema 对象", "config.responseSchema")
|
||||
if schema.get("type") not in (None, "object"):
|
||||
return _issue("responseSchema 目前仅支持 object 类型", "config.responseSchema")
|
||||
return None
|
||||
|
||||
|
||||
def _check_skill(config: dict[str, Any], catalog: ResourceCatalog) -> dict[str, Any] | None:
|
||||
mode = str(config.get("mode") or "agent_skill")
|
||||
if mode != "agent_skill":
|
||||
return _issue("callable_skill 模式暂未开放执行,请改用 agent_skill", "config.mode")
|
||||
agent_id = str(config.get("agentId") or config.get("agent_id") or "")
|
||||
if not agent_id:
|
||||
return _issue("技能节点未选择智能体", "config.agentId")
|
||||
if agent_id not in catalog.agent_ids:
|
||||
return _issue("技能节点引用的智能体不可访问", "config.agentId")
|
||||
skill_names = [str(s) for s in (config.get("skillNames") or config.get("skill_names") or [])]
|
||||
if not skill_names:
|
||||
return _issue("技能节点至少需要选择一个技能", "config.skillNames")
|
||||
for name in skill_names:
|
||||
if name not in catalog.enabled_skills:
|
||||
return _issue(f"技能不存在或未启用:{name}", "config.skillNames")
|
||||
whitelist = catalog.agent_skills.get(agent_id)
|
||||
if whitelist:
|
||||
missing = [name for name in skill_names if name not in whitelist]
|
||||
if missing:
|
||||
return _issue(f"技能不在智能体白名单内:{'、'.join(missing)}", "config.skillNames")
|
||||
return None
|
||||
|
||||
|
||||
def _check_sql(config: dict[str, Any], catalog: ResourceCatalog, cfg: WorkflowConfig) -> dict[str, Any] | None:
|
||||
source_id = str(config.get("dataSourceId") or config.get("data_source_id") or "")
|
||||
if not source_id:
|
||||
return _issue("SQL 节点未选择数据源", "config.dataSourceId")
|
||||
source = catalog.data_sources.get(source_id)
|
||||
if source is None:
|
||||
return _issue("数据源不存在或无权访问", "config.dataSourceId")
|
||||
if not source.get("enabled", True):
|
||||
return _issue("数据源已被禁用", "config.dataSourceId")
|
||||
if str(source.get("kind") or "sql") != "sql":
|
||||
return _issue("该资源不是 SQL 数据源", "config.dataSourceId")
|
||||
statement = str(config.get("statement") or config.get("queryTemplate") or config.get("query_template") or "")
|
||||
if not statement.strip():
|
||||
return _issue("SQL 节点缺少查询语句", "config.statement")
|
||||
try:
|
||||
assert_read_only(statement)
|
||||
assert_tables_allowed(statement, list(source.get("allowed_tables") or []))
|
||||
except WorkflowError as exc:
|
||||
return _issue(str(exc.message if hasattr(exc, "message") else exc), "config.statement")
|
||||
parameters = config.get("parameters")
|
||||
if parameters is not None and not isinstance(parameters, dict):
|
||||
return _issue("SQL 参数必须是对象", "config.parameters")
|
||||
max_rows = config.get("maxRows") or config.get("max_rows")
|
||||
if max_rows is not None:
|
||||
try:
|
||||
rows = int(max_rows)
|
||||
except (TypeError, ValueError):
|
||||
return _issue("maxRows 必须是整数", "config.maxRows")
|
||||
ceiling = min(int(cfg.sql.max_rows), int(source.get("max_rows") or cfg.sql.max_rows))
|
||||
if rows < 1 or rows > ceiling:
|
||||
return _issue(f"maxRows 超过上限 {ceiling}", "config.maxRows")
|
||||
return None
|
||||
|
||||
|
||||
def _check_http(config: dict[str, Any], catalog: ResourceCatalog) -> dict[str, Any] | None:
|
||||
method = str(config.get("method") or "GET").upper()
|
||||
if method not in _HTTP_METHODS:
|
||||
return _issue(f"不支持的 HTTP 方法:{method}", "config.method")
|
||||
headers = config.get("headers")
|
||||
if headers is not None:
|
||||
if not isinstance(headers, dict):
|
||||
return _issue("请求头必须是字符串到字符串的映射", "config.headers")
|
||||
for name in headers:
|
||||
if str(name).strip().lower() in _LITERAL_SECRET_KEYS:
|
||||
return _issue(f"请勿在节点上明文配置 {name},改用凭据引用", "config.headers")
|
||||
url = str(config.get("url") or "").strip()
|
||||
credential_ref = str(config.get("credentialRef") or config.get("credential_ref") or "").strip()
|
||||
if not url and not credential_ref:
|
||||
return _issue("HTTP 节点缺少 URL 或凭据引用", "config.url")
|
||||
if credential_ref:
|
||||
source = catalog.data_sources.get(credential_ref)
|
||||
if source is None:
|
||||
return _issue("引用的凭据(HTTP 数据源)不存在或无权访问", "config.credentialRef")
|
||||
if not source.get("enabled", True):
|
||||
return _issue("引用的凭据已被禁用", "config.credentialRef")
|
||||
if str(source.get("kind") or "") != "http":
|
||||
return _issue("引用的凭据不是 HTTP 类型数据源", "config.credentialRef")
|
||||
allowed = {str(m).upper() for m in (source.get("allowed_methods") or [])}
|
||||
if allowed and method not in allowed:
|
||||
return _issue(f"方法 {method} 不在凭据资源允许的方法列表内", "config.method")
|
||||
return None
|
||||
|
||||
|
||||
def _check_code(config: dict[str, Any], cfg: WorkflowConfig) -> dict[str, Any] | None:
|
||||
if not cfg.code.enabled:
|
||||
return _issue("代码节点未启用(需管理员在 config.yaml 中开启 workflows.code.enabled)", "")
|
||||
language = config.get("language")
|
||||
if language is not None and str(language) != "python":
|
||||
return _issue("代码节点目前仅支持 python", "config.language")
|
||||
source = str(config.get("source") or "")
|
||||
if not source.strip():
|
||||
return _issue("代码节点没有源代码", "config.source")
|
||||
timeout = config.get("timeoutSeconds") or config.get("timeout_seconds")
|
||||
if timeout is not None:
|
||||
try:
|
||||
seconds = int(timeout)
|
||||
except (TypeError, ValueError):
|
||||
return _issue("timeoutSeconds 必须是整数", "config.timeoutSeconds")
|
||||
if seconds < 1 or seconds > int(cfg.code.timeout_seconds):
|
||||
return _issue(f"timeoutSeconds 超过上限 {cfg.code.timeout_seconds}", "config.timeoutSeconds")
|
||||
return None
|
||||
|
||||
|
||||
def _check_output(config: dict[str, Any]) -> dict[str, Any] | None:
|
||||
mapping = config.get("mapping")
|
||||
if mapping is None:
|
||||
return None
|
||||
if not isinstance(mapping, dict):
|
||||
return _issue("输出映射必须是对象", "config.mapping")
|
||||
for key, template in mapping.items():
|
||||
if not isinstance(template, str) or not template.strip():
|
||||
return _issue(f"输出字段 {key} 的映射模板为空", f"config.mapping.{key}")
|
||||
return None
|
||||
|
||||
|
||||
def _check_subworkflow(config: dict[str, Any], catalog: ResourceCatalog, *, current_workflow_id: str) -> dict[str, Any] | None:
|
||||
workflow_id = str(config.get("workflowId") or config.get("workflow_id") or "")
|
||||
if not workflow_id:
|
||||
return _issue("子工作流节点缺少 workflowId", "config.workflowId")
|
||||
version_id = str(config.get("versionId") or config.get("version_id") or "")
|
||||
if not version_id:
|
||||
return _issue("子工作流节点缺少 versionId", "config.versionId")
|
||||
if workflow_id == current_workflow_id:
|
||||
return _issue("子工作流不能引用自身", "config.workflowId")
|
||||
versions = catalog.published_subworkflows.get(workflow_id)
|
||||
if not versions:
|
||||
return _issue("子工作流未发布或无权访问", "config.workflowId")
|
||||
if version_id not in versions:
|
||||
return _issue("子工作流版本不存在或已下线", "config.versionId")
|
||||
return None
|
||||
|
||||
|
||||
def make_resource_checker(
|
||||
catalog: ResourceCatalog,
|
||||
*,
|
||||
config: WorkflowConfig,
|
||||
current_workflow_id: str = "",
|
||||
):
|
||||
"""Build the sync ``resource_checker`` consumed by ``validate_workflow_graph``."""
|
||||
|
||||
def checker(node_type: str, node_config: dict[str, Any]):
|
||||
try:
|
||||
if node_type == "agent":
|
||||
return _check_agent(node_config, catalog)
|
||||
if node_type == "skill":
|
||||
return _check_skill(node_config, catalog)
|
||||
if node_type == "sql_read":
|
||||
return _check_sql(node_config, catalog, config)
|
||||
if node_type == "http":
|
||||
return _check_http(node_config, catalog)
|
||||
if node_type == "code":
|
||||
return _check_code(node_config, config)
|
||||
if node_type == "output":
|
||||
return _check_output(node_config)
|
||||
if node_type == "subworkflow":
|
||||
return _check_subworkflow(node_config, catalog, current_workflow_id=current_workflow_id)
|
||||
except Exception: # noqa: BLE001 - a broken check must fail closed
|
||||
return _issue("资源配置校验失败,请检查节点配置", "")
|
||||
return None
|
||||
|
||||
return checker
|
||||
|
||||
|
||||
__all__ = ["ResourceCatalog", "build_resource_catalog", "make_resource_checker"]
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Periodic retention sweep for workflow runs, events, and artifacts.
|
||||
|
||||
Consumes ``workflows.retention.event_retention_days`` /
|
||||
``run_retention_days``. Events older than the event window are dropped first;
|
||||
terminal runs older than the run window are deleted with their node-run and
|
||||
artifact rows, and any leftover events for those run ids are cleaned up so the
|
||||
event log cannot outlive the run it describes.
|
||||
|
||||
Artifact *files* stay on the existing sandbox/artifact lifecycle — this job
|
||||
only removes the workflow catalog rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_INTERVAL_SECONDS = 60 * 60
|
||||
_MAX_BATCHES_PER_SWEEP = 8
|
||||
_SQL_PURGE_BATCH = 500
|
||||
|
||||
|
||||
class WorkflowRetentionCleaner:
|
||||
def __init__(
|
||||
self,
|
||||
runs: WorkflowRunStore,
|
||||
events: WorkflowEventStore,
|
||||
*,
|
||||
config: WorkflowConfig | None = None,
|
||||
) -> None:
|
||||
self._runs = runs
|
||||
self._events = events
|
||||
self._config = config or WorkflowConfig()
|
||||
self._task: asyncio.Task | None = None
|
||||
self._stopping = False
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is not None and not self._task.done():
|
||||
return
|
||||
self._stopping = False
|
||||
self._task = asyncio.create_task(self._loop(), name="workflow-retention")
|
||||
logger.info(
|
||||
"workflow retention cleaner started (events=%sd runs=%sd)",
|
||||
self._config.retention.event_retention_days,
|
||||
self._config.retention.run_retention_days,
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._stopping = True
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
self._task = None
|
||||
|
||||
async def sweep(self) -> dict[str, int]:
|
||||
"""One pass. Safe to call from tests without starting the loop."""
|
||||
event_days = max(1, self._config.retention.event_retention_days)
|
||||
run_days = max(1, self._config.retention.run_retention_days)
|
||||
events_deleted = await self._events.purge_before(older_than_days=event_days)
|
||||
runs_deleted = 0
|
||||
for _ in range(_MAX_BATCHES_PER_SWEEP):
|
||||
ids = await self._runs.purge_terminal_before(older_than_days=run_days)
|
||||
if not ids:
|
||||
break
|
||||
runs_deleted += len(ids)
|
||||
events_deleted += await self._events.purge_for_runs(ids)
|
||||
if len(ids) < _SQL_PURGE_BATCH:
|
||||
break
|
||||
if events_deleted or runs_deleted:
|
||||
logger.info("workflow retention sweep: events=%s runs=%s", events_deleted, runs_deleted)
|
||||
return {"events": events_deleted, "runs": runs_deleted}
|
||||
|
||||
async def _loop(self) -> None:
|
||||
try:
|
||||
await self.sweep()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - the loop must survive a bad pass
|
||||
logger.exception("workflow retention sweep failed")
|
||||
while not self._stopping:
|
||||
try:
|
||||
await asyncio.sleep(_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
if self._stopping:
|
||||
return
|
||||
try:
|
||||
await self.sweep()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("workflow retention sweep failed")
|
||||
|
||||
|
||||
__all__ = ["WorkflowRetentionCleaner"]
|
||||
Reference in new issue
Block a user