代码提交

This commit is contained in:
董宏凯 committed 2026-08-30 01:25:20 +08:00
1 parent 4ac7bc21cd
commit 17cf849361
110 files changed
+16754 -265

No files matched your search

+8
View File
@@ -119,6 +119,14 @@ offline-backend-20260512/backend/tests/*
!offline-backend-20260512/backend/tests/test_enterprise_research_tasks.py
# Position-roundtable per-turn write/present cap (weak-model duplicate artifacts).
!offline-backend-20260512/backend/tests/test_position_artifact_cap.py
# Workflow Studio: schemas / definitions / engine / run-lifecycle regressions
# plus the concurrency remediation suite (atomic resume, draft CAS, publish
# counter, event seq) — required in CI per docs/WORKFLOW_STUDIO_BACKEND_REMEDIATION_ZH.md P1-06.
!offline-backend-20260512/backend/tests/test_workflow_schemas.py
!offline-backend-20260512/backend/tests/test_workflow_definitions.py
!offline-backend-20260512/backend/tests/test_workflow_engine.py
!offline-backend-20260512/backend/tests/test_workflow_runs.py
!offline-backend-20260512/backend/tests/test_workflow_concurrency.py
deploy
.history
+2
View File
@@ -73,6 +73,8 @@ The **Lead Agent** (`deerflow/agents/lead_agent/`) is the single LangGraph graph
**Harness → App import firewall** is enforced by `tests/test_harness_boundary.py` (runs in CI).
**Workflow Studio** (`deerflow/workflows/` + `/api/workflows*`) is a second, independent execution path beside the lead agent: a persisted DAG scheduler that runs published workflow graphs, streams a durable event log over SSE, and supports human-input pauses, cancellation, and crash recovery via lease reclaim. Coze Studio supplies the canvas UI only; LangGraph is used only *inside* an `agent`/`skill` node. Master switch and all limits live in `config.yaml` → `workflows`. See the backend `AGENTS.md`/`CLAUDE.md`「Workflow Studio」and `docs/WORKFLOW_STUDIO_BACKEND_DEV_ZH.md`.
### Frontend
The frontend (`frontend-web/`) is a React 19 + Vite SPA:
+2
View File
@@ -73,6 +73,8 @@ The **Lead Agent** (`deerflow/agents/lead_agent/`) is the single LangGraph graph
**Harness → App import firewall** is enforced by `tests/test_harness_boundary.py` (runs in CI).
**Workflow Studio** (`deerflow/workflows/` + `/api/workflows*`) is a second, independent execution path beside the lead agent: a persisted DAG scheduler that runs published workflow graphs, streams a durable event log over SSE, and supports human-input pauses, cancellation, and crash recovery via lease reclaim. Coze Studio supplies the canvas UI only; LangGraph is used only *inside* an `agent`/`skill` node. Master switch and all limits live in `config.yaml` → `workflows`. See the backend `CLAUDE.md`「Workflow Studio」and `docs/WORKFLOW_STUDIO_BACKEND_DEV_ZH.md`.
### Frontend
The frontend (`frontend-web/`) is a React 19 + Vite SPA:
+37 -78
View File
@@ -31,6 +31,7 @@ import {
import {
collectHumanMessageIds,
hasServerEchoedOptimisticHuman,
isStaleStreamFinish,
mergeStreamDisplayValues,
shouldDiscardTerminalOverlay,
} from "./stream-display";
@@ -190,11 +191,9 @@ function getRunMetadataStorage(): {
/**
* SDK ``useStream`` stores the in-flight run id at ``lg:stream:{threadId}``
* but **deletes that key on stream onSuccess**. A premature SSE close
* (proxy timeout / false-positive disconnect) looks like success, so the
* reconnect key vanishes while the backend run is still occupying the
* thread. We keep a second key the SDK never touches and only clear it
* after ``runs.get`` confirms a terminal status.
* but **deletes that key on stream onSuccess**. We keep a second key the SDK
* never touches so a 409 / dropped-SSE exception can still rejoin. Normal
* SSE close clears it immediately — do not wait for ``runs.get``.
*/
const ACTIVE_RUN_STORAGE_PREFIX = "df:active-run:";
@@ -276,36 +275,18 @@ export async function stopThreadRun(
/**
* 协助卡提交前:若上一轮还在流式(弱模型出卡后偶发未停干净),先取消服务端 run
* 并断开本地 SSE,避免 409「已有活跃 run」或用户点了选项却发不出去。
*
* 不在发送前 GET /runs —— 协助卡出现时图多半已经结束,为兜底去查会让每次
* 点选项都卡住。未在流式则直接发;真遇到 409 再走 confirmOrRejoin。
*/
export async function stopActiveThreadBeforeClarificationReply(
threadId: string | null | undefined,
thread: { isLoading?: boolean; stop: () => void | Promise<void> },
): Promise<void> {
let storedRunId: string | null = null;
if (threadId && typeof window !== "undefined") {
try {
storedRunId =
normalizeStoredRunId(
window.sessionStorage.getItem(`lg:stream:${threadId}`),
) || readActiveRunId(threadId);
} catch {
storedRunId = null;
}
}
// 协助卡出现时图已经 jump_to=end,run 多半已是 terminal。再 stop()/cancel
// 会再触发 onFinish → getState overlay,把刚发出的用户回复冻住、整段攒着出。
if (storedRunId && threadId) {
try {
const run = await getAPIClient().runs.get(threadId, storedRunId);
if (run && !isInflightRunStatus(run.status)) {
clearActiveRunId(threadId);
if (!thread.isLoading) {
if (threadId) clearActiveRunId(threadId);
return;
}
} catch {
/* 404 / 网络:按下面的 best-effort stop 走 */
}
}
if (!thread.isLoading && !storedRunId) return;
try {
await stopThreadRun(threadId);
} catch {
@@ -363,12 +344,9 @@ const TERMINAL_RUN_STATUSES = new Set([
"interrupted",
]);
// A completed server run normally sends a terminal SSE event. In local
// development, however, a Vite hot update, a proxy disconnect, or a brief
// network interruption can make the browser miss that final event while the
// backend continues (and successfully completes) the run. Do not probe on
// every render: only treat the stream as suspicious after it has been quiet
// for a while, then reconcile it from the authoritative run/state endpoints.
// Only probe /runs after an *exception* path has already set streamHoldOpen
// (dropped SSE, 409, React #185). Normal Q&A must not GET /runs on finish or
// every 5s — that made every send wait for a status round-trip.
const STREAM_RECONCILIATION_IDLE_MS = 12_000;
const STREAM_RECONCILIATION_POLL_MS = 5_000;
// A run can legitimately be quiet while a long tool call / slow model write is
@@ -608,14 +586,9 @@ export function useThreadStream({
"terminal" | "rejoined" | "unknown"
>
>(async () => "unknown");
// Keep the send button in "stop" until we confirm the backend run is
// terminal. A premature SSE close must not look like the turn ended.
const [streamHoldOpen, setStreamHoldOpen] = useState(
() =>
Boolean(
threadId && typeof window !== "undefined" && readActiveRunId(threadId),
),
);
// Held only on exception recovery (dropped SSE / 409). Normal turns rely
// on SDK `isLoading` so the send button unlocks as soon as SSE closes.
const [streamHoldOpen, setStreamHoldOpen] = useState(false);
const sendInFlightRef = useRef(false);
const sendEpochRef = useRef(0);
// Filled after useStream; onFinish/onError may read it asynchronously.
@@ -707,7 +680,6 @@ export function useThreadStream({
if (meta.run_id) {
activeRunIdRef.current = meta.run_id;
persistActiveRunId(meta.thread_id, meta.run_id);
setStreamHoldOpen(true);
}
const metadataAgentId = context.agent_id ?? context.agent_name;
const extraMetadata =
@@ -944,34 +916,30 @@ export function useThreadStream({
}
toast.error(message);
},
onFinish(state) {
onFinish(state, runMeta) {
lastStreamActivityAtRef.current = Date.now();
// Do not trust a clean SSE close as "the run is done". streamResumable
// chats keep the backend run alive on disconnect; the SDK still fires
// onFinish and would otherwise restore the send button.
//
// If a newer turn is already streaming (common: 用户协助卡提交后,上一轮
// interrupt 的迟到 onFinish 才到), do not freeze/rejoin — that would
// steal the live SSE and dump tokens only at the end.
if (threadStreamRef.current?.isLoading || sendInFlightRef.current) {
// Trust a clean SSE close. Querying /runs here blocked every send for
// ~5s to paper over a rare premature disconnect; that case is handled
// when the next send hits 409 (or onError for a dropped stream).
if (
isStaleStreamFinish({
finishingRunId: runMeta?.run_id,
currentRunId: activeRunIdRef.current,
})
) {
return;
}
setStreamHoldOpen(true);
void confirmOrRejoinRef.current("finish").then((result) => {
if (result === "rejoined") {
return;
}
if (threadStreamRef.current?.isLoading || sendInFlightRef.current) {
return;
}
setStreamHoldOpen(false);
// run 真正终结(非 rejoin):若深度研究报告流仍 active(tool_finished
// 帧丢失/被取消),标记中断,避免进度条永久转圈。
const finThreadId = threadIdRef.current ?? onStreamThreadId ?? null;
if (finThreadId) finalizeResearchReportStream(finThreadId);
if (finThreadId) {
clearActiveRunId(finThreadId);
finalizeResearchReportStream(finThreadId);
}
activeRunIdRef.current = null;
setStreamHoldOpen(false);
if (state?.values) {
listeners.current.onFinish?.(state.values);
}
void queryClient.invalidateQueries({ queryKey: ["threads", "search"] });
});
},
});
@@ -1009,14 +977,6 @@ export function useThreadStream({
if (streamReconciliationInFlightRef.current && reason === "poll") {
return "unknown";
}
// Stale finish/error from a prior interrupt must not joinStream over a
// turn the user already started (协助卡提交后的实时流会被打断并攒批)。
if (
(reason === "finish" || reason === "error") &&
(sendInFlightRef.current || threadStreamRef.current?.isLoading)
) {
return "rejoined";
}
streamReconciliationInFlightRef.current = true;
const epochAtStart = sendEpochRef.current;
try {
@@ -1192,11 +1152,10 @@ export function useThreadStream({
if (dropped && idleFor < 800) {
return;
}
const stored = activeRunIdRef.current ?? readActiveRunId(activeThreadId);
if (!stored && !streamHoldOpen && threadStreamRef.current?.isLoading) {
return;
}
if (!stored && !streamHoldOpen) {
// Do not probe just because a run id is stored. Only the exception
// path sets streamHoldOpen; polling otherwise made every turn wait
// on GET /runs before the user could send again.
if (!streamHoldOpen) {
return;
}
await confirmOrRejoin("poll");
@@ -5,6 +5,7 @@ import type { Message } from "@langchain/langgraph-sdk";
import {
collectHumanMessageIds,
hasServerEchoedOptimisticHuman,
isStaleStreamFinish,
mergeStreamDisplayValues,
shouldDiscardTerminalOverlay,
} from "./stream-display";
@@ -28,6 +29,30 @@ test("live streaming ignores frozen checkpoint messages so tokens can paint", ()
assert.deepEqual(afterStream?.messages, overrides.messages);
});
test("late onFinish of a prior run is stale once a newer run is live", () => {
assert.equal(
isStaleStreamFinish({
finishingRunId: "old-run",
currentRunId: "new-run",
}),
true,
);
assert.equal(
isStaleStreamFinish({
finishingRunId: "run-1",
currentRunId: "run-1",
}),
false,
);
assert.equal(
isStaleStreamFinish({
finishingRunId: "run-1",
currentRunId: null,
}),
false,
);
});
test("a later send discards the previous run's getState overlay", () => {
assert.equal(
shouldDiscardTerminalOverlay({
@@ -51,6 +51,25 @@ export function shouldDiscardTerminalOverlay(opts: {
return false;
}
/**
* True when an SSE onFinish belongs to a *previous* run, after the user
* already started a newer turn (协助卡提交是典型场景).
*
* Do **not** use `sendInFlight` / `isLoading` for this: `submit()` stays
* pending until the current stream ends, so those flags are still true on a
* normal same-turn finish and would skip unlock for every Q&A.
*/
export function isStaleStreamFinish(opts: {
finishingRunId?: string | null;
currentRunId?: string | null;
}): boolean {
return Boolean(
opts.finishingRunId &&
opts.currentRunId &&
opts.finishingRunId !== opts.currentRunId,
);
}
export function collectHumanMessageIds(
...groups: Array<Message[] | undefined>
): Set<string> {
+65 -25
View File
@@ -1,21 +1,23 @@
"use client";
import {
Bot,
ChevronLeft,
ChevronRight,
Compass,
Clock,
DownloadIcon,
FilePenLine,
MessageSquare,
PenTool,
Pin,
PinOff,
Search,
Star,
Users2,
type LucideIcon,
X,
} from "lucide-react";
import Link from "next/link";
import { motion } from "motion/react";
import { useCallback, useEffect, useMemo, useState } from "react";
import { Button } from "@/components/ui/button";
@@ -48,6 +50,9 @@ import {
import { formatTimeAgo } from "@/core/utils/datetime";
import { desensitize } from "@/lib/desensitize";
import { cn } from "@/lib/utils";
import { useAIWritingSessions } from "@/open-canvas/hooks/useAIWritingSessions";
import { useRoundtableDrafts } from "@/roundtable-planning/hooks/useRoundtableDrafts";
import { useResearchSessions } from "@/strategy-components/api/deep-research";
import { RoundtableHistoryTab } from "./chats-history/RoundtableHistoryTab";
import { WritingHistoryTab } from "./chats-history/WritingHistoryTab";
@@ -79,19 +84,19 @@ const HISTORY_TABS: {
id: "roundtable",
label: "多智能体会商",
description: "多角色协作研讨",
icon: Bot,
icon: Users2,
},
{
id: "writing",
label: "AI写作",
description: "写作任务与草稿",
icon: FilePenLine,
icon: PenTool,
},
{
id: "research",
label: "深度研究",
description: "研究任务与报告",
icon: Search,
icon: Compass,
},
];
@@ -126,11 +131,20 @@ export default function ChatsPage({
const [importOpen, setImportOpen] = useState(false);
// 非对话 Tab 的记录总数由 Tab 组件回传(客户端过滤后的条数)。
const [tabTotal, setTabTotal] = useState<number | null>(null);
// 顶部的数字角标与参考页一致,始终显示各类记录的实际总数,而不是当前筛选结果。
const { drafts: roundtableTabDrafts, refresh: refreshRoundtableTabDrafts } = useRoundtableDrafts();
const { sessions: writingTabSessions, refresh: refreshWritingTabSessions } = useAIWritingSessions();
const { data: researchTabData } = useResearchSessions(100);
useEffect(() => {
document.title = `${t.pages.chats} - ${t.pages.appName}`;
}, [t.pages.chats, t.pages.appName]);
useEffect(() => {
void refreshRoundtableTabDrafts();
void refreshWritingTabSessions();
}, [refreshRoundtableTabDrafts, refreshWritingTabSessions]);
// Debounce the search box so each keystroke doesn't fire a backend query.
useEffect(() => {
const timer = setTimeout(() => setDebouncedSearch(search), 300);
@@ -224,7 +238,7 @@ export default function ChatsPage({
"group/chat-row relative flex min-w-0 items-start gap-3 rounded-[10px] border bg-card p-3 transition-[border-color,box-shadow,background-color] duration-150 sm:items-center sm:p-3.5",
pinned
? "border-red-200 bg-gradient-to-r from-red-50/50 via-white to-white shadow-xs hover:border-red-300 hover:shadow-sm"
: "border-neutral-200/80 bg-white hover:border-red-200 hover:bg-red-50/20 hover:shadow-xs",
: "border-neutral-200/80 bg-white hover:!border-red-300 hover:!bg-gradient-to-r hover:!from-red-50/50 hover:!via-white hover:!to-white hover:shadow-xs",
)}
>
<Button
@@ -248,12 +262,12 @@ export default function ChatsPage({
>
<div className="flex min-w-0 items-center gap-2">
{pinned && (
<span className="inline-flex items-center gap-1 rounded border border-red-200/50 bg-red-100/80 px-1.5 py-0.5 text-[10px] font-semibold text-red-700">
<span className="inline-flex items-center gap-1 rounded border !border-red-200 bg-red-100/80 px-1.5 py-0.5 text-[10px] font-semibold text-red-700">
<Pin className="size-3 rotate-45 text-red-600" />
置顶
</span>
)}
<span className="inline-flex items-center gap-1 rounded-full border border-red-200/70 bg-red-50 px-2 py-0.5 text-[11px] font-semibold text-red-700">
<span className="inline-flex items-center gap-1 rounded-full border !border-red-200 bg-red-50 px-2 py-0.5 text-[11px] font-semibold text-red-700">
<MessageSquare className="size-3 text-red-600" />
对话
</span>
@@ -298,6 +312,12 @@ export default function ChatsPage({
// 头部计数:对话 Tab 用后端总数,其余 Tab 用组件回传的过滤后条数。
const displayTotal = activeTab === "chats" ? totalCount : tabTotal;
const activeTabConfig = HISTORY_TABS.find((tab) => tab.id === activeTab)!;
const historyTabCounts: Record<HistoryTab, number> = {
chats: allThreadsCount ?? 0,
roundtable: roundtableTabDrafts.length,
writing: writingTabSessions.length,
research: researchTabData?.total ?? 0,
};
return (
<WorkspaceContainer>
@@ -315,15 +335,18 @@ export default function ChatsPage({
<WorkspaceBody>
<div className="flex size-full min-w-0 flex-col">
<header className="flex shrink-0 flex-col items-center gap-3 pt-8">
<div className="relative flex w-full max-w-5xl items-center justify-center px-4 sm:px-6">
<div className="relative w-full max-w-5xl px-4 sm:px-6">
<div className="flex w-full flex-col items-center">
<div
className="inline-flex max-w-full overflow-x-auto rounded-2xl border border-red-100 bg-white p-1.5 shadow-xs backdrop-blur-sm"
id="primary-tabs-nav"
className="inline-flex rounded-2xl border border-red-100 bg-white p-1.5 shadow-xs backdrop-blur-sm"
role="tablist"
aria-label="历史记录类型"
>
{HISTORY_TABS.map((tab) => {
const isActive = activeTab === tab.id;
const Icon = tab.icon;
const count = historyTabCounts[tab.id];
return (
<button
@@ -331,7 +354,7 @@ export default function ChatsPage({
id={`history-tab-${tab.id}`}
type="button"
className={cn(
"relative flex shrink-0 select-none items-center gap-2 rounded-xl px-4 py-2 text-sm font-medium transition-all duration-200 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-red-500/30",
"relative flex items-center gap-2 rounded-xl px-4 py-2 text-sm font-medium transition-all duration-200 select-none focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-red-500/30",
isActive
? "font-semibold text-red-700 shadow-xs"
: "text-neutral-600 hover:bg-red-50/50 hover:text-red-600",
@@ -343,24 +366,46 @@ export default function ChatsPage({
onClick={() => setActiveTab(tab.id)}
>
{isActive && (
<span
className="absolute inset-0 rounded-xl border border-red-200 bg-red-50/90 shadow-xs"
<motion.div
layoutId="activeTabBadge"
className="absolute inset-0 rounded-xl border !border-red-200 bg-red-50/90 shadow-xs"
transition={{ type: "spring", stiffness: 450, damping: 35 }}
aria-hidden="true"
/>
)}
<span className="relative z-10 flex items-center gap-2">
<span
className={cn(
"relative z-10 flex items-center gap-2",
isActive
? "font-semibold text-red-700"
: "text-neutral-600",
)}
>
<Icon className="size-4" aria-hidden="true" />
<span>{tab.label}</span>
</span>
{count > 0 && (
<span
className={cn(
"relative z-10 rounded-full px-1.5 py-[0.05rem] font-mono text-[11px] transition-colors",
isActive
? "bg-red-600 font-bold text-white"
: "bg-neutral-100 text-neutral-500",
)}
>
{count}
</span>
)}
</button>
);
})}
</div>
</div>
{activeTab === "chats" && (
<Button
variant="outline"
size="sm"
className="absolute right-4 sm:right-6"
className="absolute right-4 top-1/2 -translate-y-1/2 sm:right-6"
onClick={() => setImportOpen(true)}
>
<DownloadIcon className="size-4" />
@@ -376,9 +421,9 @@ export default function ChatsPage({
id={`history-panel-${activeTab}`}
role="tabpanel"
aria-labelledby={`history-tab-${activeTab}`}
className="space-y-4 rounded-2xl border border-red-100 bg-white p-4 shadow-xs sm:p-5"
className="space-y-4 rounded-2xl border border-red-200 bg-white p-4 shadow-xs sm:p-5"
>
<div className="w-full space-y-3.5">
{activeTab !== "roundtable" && <div className="w-full space-y-3.5">
<div className="flex items-center justify-between gap-3">
{activeTab === "chats" ? (
<div className="flex items-center gap-1.5 overflow-x-auto pb-1 md:pb-0" role="group" aria-label="对话筛选">
@@ -387,7 +432,7 @@ export default function ChatsPage({
className={cn(
"inline-flex shrink-0 items-center gap-1.5 rounded-lg px-3.5 py-1.5 text-xs font-medium transition-all focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-red-500/30",
chatFilter === "all"
? "bg-red-600 font-semibold text-white shadow-xs"
? "border !border-red-600 bg-red-600 font-semibold text-white shadow-xs"
: "border border-neutral-200 bg-white text-neutral-600 hover:border-red-200 hover:bg-red-50/60 hover:text-red-700",
)}
aria-pressed={chatFilter === "all"}
@@ -499,7 +544,7 @@ export default function ChatsPage({
</span>
)}
</div>
</div>
</div>}
{activeTab === "chats" && (
<div className="space-y-3 pt-5">
{currentPage === 1 && pinnedThreads.map(renderThread)}
@@ -543,12 +588,7 @@ export default function ChatsPage({
</div>
)}
{activeTab === "roundtable" && (
<div className="pt-5">
<RoundtableHistoryTab
query={search}
onTotalChange={handleTabTotalChange}
/>
</div>
<RoundtableHistoryTab onTotalChange={handleTabTotalChange} />
)}
{activeTab === "writing" && (
<div className="pt-5">
@@ -3,113 +3,326 @@
/**
* 「对话历史」页 —— 多智能体会商 Tab。
*
* 数据与原侧栏「多智能体会商」列表同源(当前用户的个人草稿 roundtable_drafts,
* useRoundtableDrafts 不带 taskId);点击行为也一致:暂存 sessionStorage
* 「roundtable:pending-draft」后跳转会商页,由页面挂载时消费并加载该草稿。
* 任务维度会商(task-draft,按 taskId 全任务共享、不分权)不随个人历史走,
* 仍只在对应任务工作区里查看。
* 卡片与筛选区按最新「智能工作区管理 (1)」的 ConversationRow /
* FilterAndSearchBar 适配。会商步骤、状态、模式、互动数和模型都来自实际草稿快照。
*/
import { useEffect, useState } from "react";
import { useEffect, useMemo, useState } from "react";
import { useNavigate } from "react-router-dom";
import { Bot, Share2 } from "lucide-react";
import {
ArrowUpDown,
Clock,
Cpu,
ExternalLink,
Search,
Tag,
Trash2,
Users2,
X,
} from "lucide-react";
import { toast } from "sonner";
import { ShareRoundtableDraftDialog } from "@/components/workspace/roundtable-draft-share-dialogs";
import { formatRelative } from "@/open-canvas/components/ai-writing/AIWritingHistoryDropdown";
import { useRoundtableDrafts } from "@/roundtable-planning/hooks/useRoundtableDrafts";
import type { DraftMeta } from "@/roundtable-planning/api/drafts";
import { desensitize } from "@/lib/desensitize";
import {
DeleteIcon,
HistoryRenameDialog,
HistoryRow,
HistoryRowIconButton,
HistoryStatusPill,
HistoryTypeBadge,
RenameIcon,
} from "./history-tab-shared";
getDraft,
type DraftMeta,
type DraftRecord,
} from "@/roundtable-planning/api/drafts";
import { useRoundtableDrafts } from "@/roundtable-planning/hooks/useRoundtableDrafts";
import { desensitize } from "@/lib/desensitize";
const ROUNDTABLE_PATH = "/page/workspace/roundtable/planning";
const ROUNDTABLE_STEP_LABEL: Record<DraftMeta["furthestStep"], string> = {
1: "议题梳理",
2: "多智能体会商",
3: "报告生成",
};
type SortType = "updated_desc" | "updated_asc" | "messages_desc";
interface RoundtableItem {
draft: DraftMeta;
record?: DraftRecord;
status: ReturnType<typeof progressOf>;
businessChainName: string;
messageCount: number;
model?: string;
tags: string[];
}
/**
* 列表接口会返回草稿已推进到的工作台步骤以及最新后台作业状态。两者须同时展示:
* `furthestStep` 告诉用户做到哪一环,`status` 告诉用户是否已真正完成或被中断。
*/
function progressOf(draft: DraftMeta): {
label: string;
className: string;
stepLabel: string;
tone: "completed" | "active" | "waiting" | "failed" | "cancelled";
} {
const stepLabel = `第 ${draft.furthestStep} 步 · ${ROUNDTABLE_STEP_LABEL[draft.furthestStep]}`;
const stepLabel = `第 ${draft.furthestStep} 步`;
if (draft.status === "done" || (!draft.status && draft.furthestStep === 3)) {
return { label: "已完成", className: "text-emerald-600", stepLabel };
return { label: "已完成", tone: "completed", stepLabel };
}
if (draft.status === "error") {
return { label: "执行失败", className: "text-rose-500", stepLabel: `停在${stepLabel}` };
return { label: "执行失败", tone: "failed", stepLabel: `停在${stepLabel}` };
}
if (draft.status === "cancelled") {
return { label: "已取消", className: "text-muted-foreground", stepLabel: `停在${stepLabel}` };
return { label: "已取消", tone: "cancelled", stepLabel: `停在${stepLabel}` };
}
if (draft.status === "awaiting_input") {
return { label: "等待确认", className: "text-amber-500", stepLabel: `当前${stepLabel}` };
return { label: "等待确认", tone: "waiting", stepLabel: `当前${stepLabel}` };
}
return { label: "进行中", className: "text-primary", stepLabel: `当前${stepLabel}` };
return { label: "进行中", tone: "active", stepLabel: `当前${stepLabel}` };
}
function modeOf(record?: DraftRecord): string {
switch (record?.step2?.orchestrationMode) {
case "chain":
return "业务链会商";
case "dag":
return "DAG 编排";
default:
return "自由模式";
}
}
function businessChainNameOf(draft: DraftMeta, record?: DraftRecord): string {
return record?.step2?.chain?.title ?? draft.chainTitle ?? modeOf(record);
}
function interactionCount(record?: DraftRecord): number {
if (!record) return 0;
return (
(record.step1?.interactiveMessages?.length ?? 0) +
(record.step2?.step2RoundtableDialogues?.length ?? 0) +
(record.step3?.summaryReport?.dialogues?.length ?? 0) +
(record.step3?.reportDialogues?.length ?? 0) +
(record.step3?.dashboardDialogues?.length ?? 0)
);
}
function modelOf(record?: DraftRecord): string | undefined {
return record?.step3?.summaryReport?.model ?? record?.step3?.model;
}
function tagsOf(draft: DraftMeta, record?: DraftRecord): string[] {
const chain = record?.step2?.chain?.title ?? draft.chainTitle;
return chain ? [chain] : [];
}
function timestamp(value: string): number {
const parsed = Date.parse(value);
return Number.isFinite(parsed) ? parsed : 0;
}
function renderHighlightedTitle(text: string, query: string) {
if (!query.trim()) return text;
const escaped = query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const parts = text.split(new RegExp(`(${escaped})`, "gi"));
return parts.map((part, index) =>
part.toLowerCase() === query.toLowerCase() ? (
<mark key={index} className="rounded-xs bg-red-100 px-0.5 font-semibold text-red-900">
{part}
</mark>
) : (
part
),
);
}
function RoundtableHistoryRow({
item,
searchQuery,
onDelete,
onOpen,
}: {
item: RoundtableItem;
searchQuery: string;
onDelete: () => void;
onOpen: () => void;
}) {
const { draft, status } = item;
const statusClass = {
completed: "border-emerald-200/80 bg-emerald-50 text-emerald-700",
active: "border-red-200/80 bg-red-50 text-red-600",
waiting: "border-amber-200/80 bg-amber-50 text-amber-700",
failed: "border-rose-200/80 bg-rose-50 text-rose-600",
cancelled: "border-neutral-200/80 bg-neutral-50 text-neutral-500",
}[status.tone];
const statusDotClass = {
completed: "bg-emerald-500",
active: "bg-red-500",
waiting: "bg-amber-500",
failed: "bg-rose-500",
cancelled: "bg-neutral-400",
}[status.tone];
return (
<div
id={`conversation-item-${draft.id}`}
onClick={(event) => {
if ((event.target as HTMLElement).closest("button, input")) return;
onOpen();
}}
className="group relative flex cursor-pointer items-start justify-between rounded-xl border border-neutral-200/80 bg-white p-4 transition-all duration-150 hover:!border-red-300 hover:!bg-gradient-to-r hover:!from-red-50/50 hover:!via-white hover:!to-white hover:shadow-xs sm:items-center"
>
<div className="flex min-w-0 flex-1 items-start gap-3.5 pr-4 sm:items-center">
<div className="min-w-0 flex-1 space-y-1.5">
<div className="flex flex-wrap items-center gap-2">
<span className="inline-flex items-center gap-1.5 rounded-full border !border-red-200 bg-red-50/90 px-2 py-0.5 text-[11px] font-semibold text-red-700">
<span className="relative flex size-1.5">
{status.tone === "active" && (
<span className="absolute inline-flex size-full animate-ping rounded-full bg-red-400 opacity-75" />
)}
<span className="relative inline-flex size-1.5 rounded-full bg-red-600" />
</span>
<Users2 className="size-3 text-red-600" />
会商
</span>
<h3 className="truncate text-sm font-semibold text-neutral-900 transition-colors group-hover:text-red-700">
{renderHighlightedTitle(desensitize(draft.title || "未命名会商"), searchQuery)}
</h3>
</div>
<div className="flex flex-wrap items-center gap-2 pt-0.5 text-[11px] text-neutral-400">
<span className={`inline-flex items-center gap-1 rounded border !border-red-200 px-1.5 py-0.5 text-[10px] font-medium ${statusClass}`}>
<span className={`size-1.5 rounded-full ${statusDotClass}`} />
{status.label}
</span>
<span className="inline-flex items-center rounded border border-neutral-200/60 bg-neutral-100 px-1.5 py-0.5 text-[10px] font-medium text-neutral-600">
{status.stepLabel}
</span>
<span className="inline-flex items-center gap-1 rounded border border-neutral-200/40 bg-neutral-50 px-1.5 py-0.5 text-[10px] text-neutral-500">
<Cpu className="size-2.5 text-neutral-400" />
{item.businessChainName}
</span>
<span className="inline-flex items-center gap-1 text-neutral-400">
<Clock className="size-3 text-neutral-400" />
{formatRelative(draft.updatedAt)}
</span>
</div>
</div>
</div>
<div className="flex shrink-0 items-center gap-3">
<div className="hidden items-center gap-1 opacity-0 transition-opacity group-hover:opacity-100 sm:flex">
<button
id={`delete-btn-${draft.id}`}
type="button"
onClick={(event) => {
event.stopPropagation();
onDelete();
}}
className="rounded-md p-1.5 text-neutral-400 transition-colors hover:bg-red-50 hover:text-red-600"
title="删除会商"
aria-label="删除会商"
>
<Trash2 className="size-3.5" />
</button>
<button
id={`open-preview-btn-${draft.id}`}
type="button"
onClick={(event) => {
event.stopPropagation();
onOpen();
}}
className="rounded-md p-1.5 text-neutral-500 transition-colors hover:bg-red-50 hover:text-red-700"
title="打开会商详情"
aria-label="打开会商详情"
>
<ExternalLink className="size-3.5" />
</button>
</div>
</div>
</div>
);
}
export function RoundtableHistoryTab({
query,
onTotalChange,
}: {
query: string;
onTotalChange?: (total: number) => void;
}) {
const navigate = useNavigate();
const { drafts, loading, refresh, rename, remove } = useRoundtableDrafts();
const [renameId, setRenameId] = useState<string | null>(null);
const [renameTitle, setRenameTitle] = useState("");
const [shareId, setShareId] = useState<string | null>(null);
const { drafts, loading, refresh, remove } = useRoundtableDrafts();
const [records, setRecords] = useState<Record<string, DraftRecord>>({});
const [searchQuery, setSearchQuery] = useState("");
const [selectedTag, setSelectedTag] = useState<string | null>(null);
const [sortType, setSortType] = useState<SortType>("updated_desc");
useEffect(() => {
void refresh();
}, [refresh]);
const keyword = query.trim().toLowerCase();
const filtered = keyword
? drafts.filter((draft) =>
(draft.title || "未命名草稿").toLowerCase().includes(keyword),
)
: drafts;
useEffect(() => {
let cancelled = false;
if (drafts.length === 0) {
setRecords({});
return () => {
cancelled = true;
};
}
void Promise.all(
drafts.map(async (draft) => {
try {
return await getDraft(draft.id);
} catch {
return null;
}
}),
).then((fullDrafts) => {
if (cancelled) return;
setRecords(
Object.fromEntries(
fullDrafts
.filter((record): record is DraftRecord => record !== null)
.map((record) => [record.id, record]),
),
);
});
return () => {
cancelled = true;
};
}, [drafts]);
const allItems = useMemo<RoundtableItem[]>(
() =>
drafts.map((draft) => {
const record = records[draft.id];
return {
draft,
record,
status: progressOf(draft),
businessChainName: businessChainNameOf(draft, record),
messageCount: interactionCount(record),
model: modelOf(record),
tags: tagsOf(draft, record),
};
}),
[drafts, records],
);
const availableTags = useMemo(
() => [...new Set(allItems.flatMap((item) => item.tags))],
[allItems],
);
const filteredItems = useMemo(() => {
const keyword = searchQuery.trim().toLowerCase();
return allItems
.filter((item) => {
if (selectedTag && !item.tags.includes(selectedTag)) return false;
return !keyword || (item.draft.title || "未命名会商").toLowerCase().includes(keyword);
})
.sort((left, right) => {
if (sortType === "messages_desc") return right.messageCount - left.messageCount;
const delta = timestamp(right.draft.updatedAt) - timestamp(left.draft.updatedAt);
return sortType === "updated_desc" ? delta : -delta;
});
}, [allItems, searchQuery, selectedTag, sortType]);
useEffect(() => {
onTotalChange?.(filtered.length);
}, [filtered.length, onTotalChange]);
onTotalChange?.(filteredItems.length);
}, [filteredItems.length, onTotalChange]);
const openDraft = (draft: DraftMeta) => {
sessionStorage.setItem("roundtable:pending-draft", draft.id);
navigate(ROUNDTABLE_PATH);
};
const handleRename = async (id: string, title: string) => {
try {
await rename(id, title);
toast.success("已重命名");
} catch {
toast.error("重命名失败,请重试");
} finally {
setRenameId(null);
}
};
const handleDelete = async (draft: DraftMeta) => {
const deleteDraft = async (draft: DraftMeta) => {
if (!window.confirm("确定删除这条会商记录吗?此操作不可恢复。")) return;
try {
await remove(draft.id);
@@ -119,90 +332,135 @@ export function RoundtableHistoryTab({
}
};
if (loading && drafts.length === 0) {
return (
<div className="text-muted-foreground py-16 text-center text-sm">
加载中…
</div>
);
}
if (filtered.length === 0) {
return (
<div className="text-muted-foreground py-16 text-center text-sm">
{keyword ? "没有匹配的会商记录" : "暂无会商记录"}
</div>
);
}
return (
<>
<div>
{filtered.map((draft) => {
const progress = progressOf(draft);
return (
<HistoryRow
key={draft.id}
badge={<HistoryTypeBadge icon={Bot} label="会商" />}
title={desensitize(draft.title || "未命名草稿")}
meta={
<>
<HistoryStatusPill
label={progress.label}
className={progress.className}
/>
<span className="text-muted-foreground/60">·</span>
<span className="max-w-48 truncate" title={progress.stepLabel}>
{progress.stepLabel}
<div className="w-full space-y-4">
<div className="w-full space-y-3.5">
<div className="flex flex-col items-stretch justify-between gap-3 md:flex-row md:items-center">
<div className="flex items-center gap-1.5 overflow-x-auto pb-1 md:pb-0">
<button
id="filter-all-btn"
type="button"
onClick={() => setSelectedTag(null)}
className={`inline-flex items-center gap-1.5 rounded-lg px-3.5 py-1.5 text-xs font-medium transition-all ${
!selectedTag
? "border !border-red-600 bg-red-600 font-semibold text-white shadow-xs"
: "border border-neutral-200 bg-white text-neutral-600 hover:border-red-200 hover:bg-red-50/60 hover:text-red-700"
}`}
>
<span>全部</span>
<span className={`rounded-full px-1.5 py-0.2 font-mono text-[11px] ${!selectedTag ? "bg-red-700 font-semibold text-white" : "bg-neutral-100 text-neutral-500"}`}>
{allItems.length}
</span>
<span className="text-muted-foreground/60">·</span>
<span className="max-w-40 truncate" title={draft.chainTitle || "自由模式"}>
{draft.chainTitle || "自由模式"}
</span>
<span className="text-muted-foreground/60">·</span>
<span>{formatRelative(draft.updatedAt)}</span>
</>
}
onOpen={() => openDraft(draft)}
</button>
{selectedTag && (
<div className="inline-flex items-center gap-1 rounded-lg bg-red-600 px-2.5 py-1.5 text-xs font-medium text-white shadow-xs">
<Tag className="size-3 text-red-100" />
<span>{selectedTag}</span>
<button
type="button"
onClick={() => setSelectedTag(null)}
className="ml-1 hover:text-red-200"
title="清除标签筛选"
aria-label="清除标签筛选"
>
<HistoryRowIconButton
title="重命名"
onClick={() => {
setRenameId(draft.id);
setRenameTitle(draft.title || "");
}}
>
<RenameIcon />
</HistoryRowIconButton>
<HistoryRowIconButton title="分享" onClick={() => setShareId(draft.id)}>
<Share2 className="text-muted-foreground size-4" />
</HistoryRowIconButton>
<HistoryRowIconButton
title="删除"
onClick={() => void handleDelete(draft)}
>
<DeleteIcon />
</HistoryRowIconButton>
</HistoryRow>
);
})}
<X className="size-3" />
</button>
</div>
)}
</div>
<HistoryRenameDialog
open={renameId !== null}
initialTitle={renameTitle}
dialogTitle="重命名会商"
onCancel={() => setRenameId(null)}
onConfirm={(title) => renameId && void handleRename(renameId, title)}
<div className="relative flex-1 md:max-w-md">
<Search className="pointer-events-none absolute left-3.5 top-1/2 size-4 -translate-y-1/2 text-neutral-400" />
<input
id="conversation-search-input"
type="search"
value={searchQuery}
onChange={(event) => setSearchQuery(event.target.value)}
placeholder="搜索会商记录…"
className="w-full rounded-xl border border-neutral-200 bg-neutral-50/80 py-2 pl-9 pr-16 text-sm text-neutral-900 transition-all placeholder:text-neutral-400 hover:bg-white focus:border-red-500 focus:bg-white focus:outline-hidden focus:ring-2 focus:ring-red-500/20"
/>
{searchQuery ? (
<button
id="clear-search-query-btn"
type="button"
onClick={() => setSearchQuery("")}
className="absolute right-3 top-1/2 -translate-y-1/2 p-0.5 text-neutral-400 hover:text-red-600"
aria-label="清空搜索"
>
<X className="size-3.5" />
</button>
) : (
<div className="pointer-events-none absolute right-3 top-1/2 hidden -translate-y-1/2 items-center gap-0.5 rounded bg-neutral-200/60 px-1.5 py-0.5 font-mono text-[10px] text-neutral-400 sm:flex">
⌘K
</div>
)}
</div>
</div>
<ShareRoundtableDraftDialog
draftId={shareId}
open={shareId !== null}
onOpenChange={(o) => {
if (!o) setShareId(null);
}}
<div className="flex flex-wrap items-center justify-between gap-2 border-t border-red-50 pt-1 text-xs text-neutral-500">
<div className="flex items-center gap-3 overflow-x-auto">
<span className="whitespace-nowrap font-medium text-neutral-500">
共 <strong className="font-bold text-red-600">{filteredItems.length}</strong> 条会商记录
{searchQuery && <span className="ml-1 text-neutral-400">(匹配 “{searchQuery}”)</span>}
</span>
{availableTags.length > 0 && (
<div className="hidden items-center gap-1.5 text-xs lg:flex">
<span className="text-neutral-400">快捷标签:</span>
{availableTags.slice(0, 4).map((tag) => (
<button
key={tag}
type="button"
onClick={() => setSelectedTag(selectedTag === tag ? null : tag)}
className={`rounded-md px-2 py-0.5 transition-colors ${
selectedTag === tag
? "bg-red-600 font-medium text-white shadow-2xs"
: "border !border-red-200 bg-red-50 text-red-700 hover:bg-red-100 hover:text-red-800"
}`}
>
#{tag}
</button>
))}
</div>
)}
</div>
<div className="flex items-center gap-2">
<div className="flex items-center gap-1 rounded-lg border border-neutral-200 bg-white px-2 py-1 shadow-2xs">
<ArrowUpDown className="size-3 text-neutral-400" />
<select
id="sort-select-dropdown"
value={sortType}
onChange={(event) => setSortType(event.target.value as SortType)}
className="cursor-pointer bg-transparent text-xs text-neutral-700 focus:outline-hidden"
>
<option value="updated_desc">最近活跃优先</option>
<option value="updated_asc">最早创建优先</option>
<option value="messages_desc">消息数量最多</option>
</select>
</div>
</div>
</div>
</div>
{loading && allItems.length === 0 ? (
<div className="py-16 text-center text-sm text-muted-foreground">加载中…</div>
) : filteredItems.length === 0 ? (
<div className="py-16 text-center text-sm text-muted-foreground">
{searchQuery || selectedTag ? "没有匹配的会商记录" : "暂无会商记录"}
</div>
) : (
<div className="space-y-2">
{filteredItems.map((item) => (
<RoundtableHistoryRow
key={item.draft.id}
item={item}
searchQuery={searchQuery}
onDelete={() => void deleteDraft(item.draft)}
onOpen={() => openDraft(item.draft)}
/>
</>
))}
</div>
)}
</div>
);
}
@@ -43,7 +43,7 @@ export function HistoryTypeBadge({
label: string;
}) {
return (
<span className="inline-flex shrink-0 items-center gap-1 rounded-full border border-red-200/70 bg-red-50 px-2 py-0.5 text-[11px] font-semibold text-red-700">
<span className="inline-flex shrink-0 items-center gap-1 rounded-full border !border-red-200 bg-red-50 px-2 py-0.5 text-[11px] font-semibold text-red-700">
<Icon className="size-3 text-red-600" />
{label}
</span>
@@ -68,7 +68,7 @@ export function HistoryRow({
children?: ReactNode;
}) {
return (
<div className="group/chat-row relative mb-3 flex min-w-0 rounded-[10px] border border-neutral-200/80 bg-white transition-[border-color,box-shadow,background-color] duration-150 last:mb-0 hover:border-red-200 hover:bg-red-50/20 hover:shadow-xs">
<div className="group/chat-row relative mb-3 flex min-w-0 rounded-[10px] border border-neutral-200/80 bg-white transition-[border-color,box-shadow,background-color] duration-150 last:mb-0 hover:!border-red-300 hover:!bg-gradient-to-r hover:!from-red-50/50 hover:!via-white hover:!to-white hover:shadow-xs">
<button
type="button"
onClick={onOpen}
@@ -199,6 +199,39 @@ The backend is split into two layers with a strict dependency direction:
**Dependency rule**: App imports deerflow, but deerflow never imports app. This boundary is enforced by `tests/test_harness_boundary.py` which runs in CI.
### Workflow Studio
Coze Studio is canvas-only; ChatDev is reference-only. Plan + status per phase: `docs/WORKFLOW_STUDIO_BACKEND_DEV_ZH.md`; decisions: `docs/adr/0001-workflow-studio-runtime.md`; contracts: `docs/WORKFLOW_SCHEMA_V1_ZH.md` + `docs/WORKFLOW_SSE_V1_ZH.md` + `docs/WORKFLOW_OPENAPI_DRAFT_V1.yaml`. Master switch + all limits: `config.yaml` → `workflows` (`deerflow.config.workflow_config.WorkflowConfig`).
**Execution kernel is a custom DAG scheduler, not a compiled LangGraph.** LangGraph stays the kernel *inside* an `agent`/`skill` node. The reason is single-source-of-truth recovery: a re-leased or resumed run replays completed node outputs from `workflow_node_runs` plus the `workflow_run_events` log, so a second (checkpoint) copy of workflow-level state would have no defensible tie-breaker. See the ADR's phase-2 amendment.
**Harness** (`packages/harness/deerflow/workflows/`, never imports `app.*`):
- `schemas` / `events` / `errors` / `ports` / `validator` / `samples` — frozen v1 contracts.
- `expressions.py` — `{{ inputs.x }}` / `{{ nodes.n.data.y }}` templates and conditions resolved by path walking + a small comparator set. **No `eval`.** Roots are restricted to `inputs` / `nodes` / `run` / `loop` / `env`.
- `runtime/engine.py` — the scheduler. Every edge is `pending` → `active` (source chose this port) or `pruned`; a node is ready when its non-back incoming edges are all resolved and at least one is `active`, and is skipped when all are `pruned` (skips propagate). That one rule covers fan-out, condition branching, and merge joins with no topological pre-pass. The `loop` node's `continue` back edge is the only permitted cycle; re-entry clears the body's results and invalidates its node runs.
- `runtime/{sink,sql_runner,code_runner,context}.py`, `nodes/*` (13 executors), `security/{http_policy,sql_policy,secret_box}.py`.
**Persistence**: `workflow_definitions` / `workflow_versions` (migration `20260828_01`), `workflow_runs` / `workflow_node_runs` / `workflow_run_events` / `workflow_run_artifacts` (`20260828_02`), `workflow_data_sources` (`20260828_03`). Stores under `deerflow.persistence.workflow_{runs,events,data_sources}` with SQL + in-memory implementations.
**App layer**: `workflow_dispatcher.py` (lease scan → atomic claim; an expired lease is how a dead worker's run is reclaimed, so every worker can run the loop), `workflow_executor.py` (one lease's lifecycle: replay → engine under a run timeout with heartbeat + cancel watcher → exactly one terminal transition; also runs subworkflows inline, depth ≤ 3), `workflow_retention.py` (hourly sweep of `workflows.retention.event_retention_days` / `run_retention_days`), `workflow_live_hub.py` (in-process SSE fan-out), `workflow_agent_runner.py` (agent/skill nodes → lead agent).
**Gateway routes** (register static prefixes before `workflows.router`'s `/{workflow_id}` catch-alls): `/api/workflows` CRUD/draft/validate/publish/copy, `/api/workflows/node-types` + `/resources/*`, `/api/workflows/{id}/runs` + `/api/workflows/runs/{run_id}{,/events,/stream,/cancel,/resume,/retry,/artifacts}`, `/api/workflows/data-sources` (+ `/{id}/introspect`, `/{id}/validate-query`), `/api/workflows/embed/{tickets,verify}`, `/api/workflow_api/*` Coze compat.
**Guarantees worth not breaking**: events are persisted *then* published, and the DB is authoritative for `seq` ordering (live frames at or below the replay cursor are dropped and re-read on the next poll, so a reconnect sees no gap; `Last-Event-ID` wins over `?after=`); a run reaching a terminal status always gets a terminal event, synthesised if necessary, or SSE clients hang; `resumeToken` appears only in the `run.awaiting_input` event, never in the run view; `POST /retry` on a `failed` run copies completed node outputs onto a new run marked `retry_of_run_id` so the engine resumes from the failed node; data-source secrets are write-only (encrypted via `WORKFLOW_SECRET_KEY`, decrypted only inside the executor, reads expose `masked_target`); the `http` node denies private/loopback/link-local targets by default (SSRF), `sql_read` allows one read-only statement with bound parameters, and the `code` node is **off by default** because its runner is a hardened subprocess rather than a container.
HTTP interface resources persist a non-secret base_url plus an allowed_methods allowlist (migration
20260829_01); request headers stay only in the encrypted secret blob. The resource catalog may return
the address and methods to the owning user, but must never return headers or any decrypted secret. The
executor receives HTTP metadata only through its data-source resolver.
The Workflow Studio agent/skill resource catalog is a read-only display projection over the existing
agent and skill ownership stores. It returns stable `agentId` / `skillId`, an agent's cached skill names,
and a `createdBy` display label. Resolve human labels from the immutable owner id at request time and
fall back to that id if lookup fails; built-in records must return `系统内置`. Do not create a parallel
creator table or accept creator data from the browser.
Tests: `tests/test_workflow_schemas.py`, `tests/test_workflow_definitions.py`, `tests/test_workflow_engine.py`, `tests/test_workflow_runs.py`.
**Import conventions**:
```python
# Harness internal
@@ -168,6 +168,21 @@ FastAPI application providing REST endpoints for frontend integration:
The separately deployed TaskCOP Vue app uses `POST /api/v1/auth/login/username` directly (its `username`, `userId`, or `yUserId` URL parameter identifies the user), so this compatibility flow does not depend on the retired Consumer login service. It calls the Gateway address configured as `b1ConsumerUrl` directly; it does not use a Vite `/api` reverse proxy. An empty TaskCOP store remains empty; task data is introduced through normal creation or the JSON import API. The Gateway CORS middleware always merges local Vite origins `http://localhost|127.0.0.1:{5173,5174,3000,8080}`, including when `GATEWAY_CORS_ORIGINS` is set for another frontend. This keeps a Vite server that falls back from 5173 to 5174 (or the reverse) working after a backend restart. For a deployment, set `GATEWAY_CORS_ORIGINS` to the exact additional frontend origins that are allowed to call the Gateway.
### Workflow Studio interface resources
The Workflow Studio data-source API also registers reusable HTTP interface resources. Create them through
POST /api/workflows/data-sources with kind set to http; provide a non-secret baseUrl, an allowedMethods
allowlist, and optional request headers. The service encrypts the headers immediately and never returns
them through list, detail, resource-catalog, run-event, or canvas APIs. The canvas resource catalog
exposes only the name, description, HTTP address, and allowed methods. HTTP execution continues to
enforce the workflow target security policy, including the default SSRF restrictions.
The same catalog's agent and skill entries are display projections, not a second ownership store:
`/api/workflows/resources/agents` returns `agentId`, cached available `skills`, and `createdBy`; the
skills endpoint returns `skillId`, direct-call capability, and `createdBy`. Creator names are resolved
from the immutable owner id at read time (falling back to the id if an account was removed), while
built-in resources explicitly return `系统内置`.
### Deep Research modes
Deep Research is a separate Gateway feature rather than part of AI Writing. Its
@@ -27,10 +27,10 @@ from app.gateway.csrf_middleware import CSRFMiddleware
from app.gateway.deps import langgraph_runtime
from app.gateway.public_knowledge_cors import PublicKnowledgeCORSMiddleware
from app.gateway.routers import (
ai_writing,
admin_active_runs,
admin_users,
agents,
ai_writing,
artifacts,
assistants_compat,
auth,
@@ -39,11 +39,12 @@ from app.gateway.routers import (
channels,
checkpoint_migration,
compact_menu_overrides,
dashboard_sessions,
deep_research,
external_llmwiki,
feedback,
fixed_questions,
dashboard_sessions,
deep_research,
flow_extract,
html_page_favorites,
intent,
knowledge,
@@ -63,19 +64,19 @@ from app.gateway.routers import (
open_chat,
parallel_agent_tasks,
parallel_agents,
position_roles,
position_roundtable,
positions,
position_roles,
public_config,
public_embed,
public_knowledge_vector_search,
public_leaderboard,
public_scheduled_tasks,
recommend,
flow_extract,
recommended_questions,
roundtable_chains,
report_structures,
roundtable_artifact_submissions,
roundtable_chains,
roundtable_diagnostics,
roundtable_draft_shares,
roundtable_drafts,
@@ -91,16 +92,22 @@ from app.gateway.routers import (
system_settings,
tags,
task_buttons,
taskcop_tasks,
report_structures,
task_reports,
taskcop_tasks,
thread_runs,
tool_metrics,
thread_shares,
threads,
tool_metrics,
uploads,
user_preferences,
user_prompts,
workflow_data_sources,
workflow_embed,
workflow_resources,
workflow_runs,
workflow_studio,
workflows,
workflows_coze_compat,
writing,
)
from deerflow.config import app_config as deerflow_app_config
@@ -342,10 +349,10 @@ async def _run_db_migrations() -> None:
deadline = asyncio.get_running_loop().time() + timeout_s
while result_queue.empty():
if asyncio.get_running_loop().time() >= deadline:
raise asyncio.TimeoutError
raise TimeoutError
await asyncio.sleep(0.2)
result = result_queue.get_nowait()
except asyncio.TimeoutError:
except TimeoutError:
logger.error(
"Database migration did not finish within %.0fs; continuing startup on create_all schema. "
"Likely cause: MySQL metadata-lock wait or a slow DDL. Set DEER_FLOW_SKIP_ALEMBIC=1 "
@@ -1591,6 +1598,18 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Light-app registry (?????) at /api/light-apps
app.include_router(light_apps.router)
# Workflow Studio: definitions / runs / SSE / data sources / embed / Coze compat.
# Static prefixes (node-types, resources, runs, data-sources, embed) must be
# registered before the /{workflow_id} catch-alls in workflows.router.
app.include_router(workflow_resources.router)
app.include_router(workflow_data_sources.router)
app.include_router(workflow_embed.router)
app.include_router(workflow_runs.router)
app.include_router(workflow_studio.router)
app.include_router(workflows.router)
app.include_router(workflows_coze_compat.router)
app.include_router(workflows_coze_compat.chrome_router)
# Sidebar-menu overrides (菜单管理) at /api/menu-overrides
app.include_router(menu_overrides.router)
@@ -66,11 +66,30 @@ def should_check_csrf(request: Request) -> bool:
# 双提交 Cookie 既无 token 可比、也无会话可被伪造,豁免之。
if path.startswith("/api/open/"):
return False
if is_coze_studio_compat_path(path):
return False
if is_weknora_embed_proxy_request(request):
return False
return True
# Coze playground 自己的 axios 不会带 X-CSRF-Token;独立编排页经
# localhost:8080 同源代理访问这些接口。Cookie 会话仍由 AuthMiddleware 校验。
_COZE_STUDIO_PREFIXES: tuple[str, ...] = (
"/api/passport/",
"/api/playground_api/",
"/api/workflow_api/",
"/api/space/",
)
def is_coze_studio_compat_path(path: str) -> bool:
stripped = path.rstrip("/")
if stripped == "/api/bot/get_type_list":
return True
return any(path.startswith(prefix) for prefix in _COZE_STUDIO_PREFIXES)
def is_weknora_embed_proxy_request(request: Request) -> bool:
path = app_relative_path(request)
if not is_weknora_embed_proxy_path(path):
@@ -285,6 +285,18 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.light_app_store = make_light_app_store(sf)
from deerflow.persistence.workflows import make_workflow_store
app.state.workflow_store = make_workflow_store(sf)
from deerflow.persistence.workflow_data_sources import make_workflow_data_source_store
from deerflow.persistence.workflow_events import make_workflow_event_store
from deerflow.persistence.workflow_runs import make_workflow_run_store
app.state.workflow_run_store = make_workflow_run_store(sf)
app.state.workflow_event_store = make_workflow_event_store(sf, run_store=app.state.workflow_run_store)
app.state.workflow_data_source_store = make_workflow_data_source_store(sf)
from deerflow.persistence.menu_overrides import make_menu_override_store
app.state.menu_override_store = make_menu_override_store(sf)
@@ -555,6 +567,47 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.deep_research_executor = None
app.state.deep_research_dispatcher = None
# ── Workflow Studio runtime: live hub + executor + lease dispatcher ──
# Every worker runs a dispatcher; the atomic claim decides ownership and
# an expired lease is how a crashed worker's run gets picked back up.
workflow_cfg = getattr(config, "workflows", None)
if workflow_cfg is not None and workflow_cfg.enabled:
from app.gateway.workflow_dispatcher import WorkflowRunDispatcher
from app.gateway.workflow_executor import WorkflowRunExecutor
from app.gateway.workflow_live_hub import WorkflowLiveHub
app.state.workflow_live_hub = WorkflowLiveHub()
app.state.workflow_executor = WorkflowRunExecutor(
app,
run_store=app.state.workflow_run_store,
event_store=app.state.workflow_event_store,
workflow_store=app.state.workflow_store,
data_source_store=app.state.workflow_data_source_store,
live_publisher=app.state.workflow_live_hub.publish,
config=workflow_cfg,
)
app.state.workflow_dispatcher = WorkflowRunDispatcher(
app.state.workflow_run_store,
app.state.workflow_executor,
event_store=app.state.workflow_event_store,
live_publisher=app.state.workflow_live_hub.publish,
config=workflow_cfg,
)
app.state.workflow_dispatcher.start()
from app.gateway.workflow_retention import WorkflowRetentionCleaner
app.state.workflow_retention = WorkflowRetentionCleaner(
app.state.workflow_run_store,
app.state.workflow_event_store,
config=workflow_cfg,
)
app.state.workflow_retention.start()
else:
app.state.workflow_live_hub = None
app.state.workflow_executor = None
app.state.workflow_dispatcher = None
app.state.workflow_retention = None
# Run event store (has its own factory with config-driven backend selection)
run_events_config = getattr(config, "run_events", None)
app.state.run_event_store = make_run_event_store(run_events_config)
@@ -583,6 +636,15 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
dr_dispatcher = getattr(app.state, "deep_research_dispatcher", None)
if dr_dispatcher is not None:
await dr_dispatcher.stop()
wf_dispatcher = getattr(app.state, "workflow_dispatcher", None)
if wf_dispatcher is not None:
await wf_dispatcher.stop()
wf_retention = getattr(app.state, "workflow_retention", None)
if wf_retention is not None:
await wf_retention.stop()
from deerflow.workflows.runtime.sql_runner import dispose_engines
await dispose_engines()
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
if queue is not None:
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
@@ -0,0 +1,287 @@
"""Workflow data sources and HTTP credentials.
Write-only secrets: the DSN (or the credential header blob) is accepted on
create/update, encrypted immediately, and never returned. Reads expose only a
masked target so an operator can tell two sources apart.
Shared sources (``ownerId`` null) are admin-only; a regular user may manage
their own.
"""
from __future__ import annotations
import json
from typing import Any
from urllib.parse import urlsplit
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.deps import get_current_user, get_optional_user_from_request
from app.gateway.workflow_audit import audit_workflow
from deerflow.config.app_config import get_app_config
from deerflow.workflows.security.secret_box import SecretKeyMissingError, secrets_available
from deerflow.workflows.security.sql_policy import assert_read_only, assert_tables_allowed
router = APIRouter(prefix="/api/workflows/data-sources", tags=["workflow-data-sources"])
_HTTP_METHODS = frozenset({"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"})
class DataSourceCreate(BaseModel):
name: str = Field(min_length=1, max_length=191)
description: str = ""
kind: str = Field(default="sql", pattern="^(sql|http)$")
dsn: str | None = None
headers: dict[str, str] | None = None
base_url: str = Field(default="", max_length=2048, alias="baseUrl")
allowed_methods: list[str] = Field(default_factory=list, alias="allowedMethods")
max_rows: int = Field(default=1000, ge=1, le=100_000, alias="maxRows")
allowed_tables: list[str] = Field(default_factory=list, alias="allowedTables")
enabled: bool = True
shared: bool = False
model_config = {"populate_by_name": True}
class QueryBody(BaseModel):
statement: str = Field(min_length=1)
class DataSourceUpdate(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=191)
description: str | None = None
dsn: str | None = None
headers: dict[str, str] | None = None
base_url: str | None = Field(default=None, max_length=2048, alias="baseUrl")
allowed_methods: list[str] | None = Field(default=None, alias="allowedMethods")
max_rows: int | None = Field(default=None, ge=1, le=100_000, alias="maxRows")
allowed_tables: list[str] | None = Field(default=None, alias="allowedTables")
enabled: bool | None = None
model_config = {"populate_by_name": True}
def _require_enabled() -> None:
if not get_app_config().workflows.enabled:
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
def _store(request: Request):
store = getattr(request.app.state, "workflow_data_source_store", None)
if store is None:
raise HTTPException(status_code=503, detail="Workflow data source store not available")
return store
async def _is_admin(request: Request) -> bool:
user = await get_optional_user_from_request(request)
return user is not None and getattr(user, "system_role", None) == "admin"
def _normalise_http_metadata(
base_url: str,
allowed_methods: list[str],
) -> tuple[str, list[str]]:
target = base_url.strip()
try:
parsed = urlsplit(target)
except ValueError as exc:
raise HTTPException(status_code=400, detail="HTTP 接口地址格式无效") from exc
if (
parsed.scheme not in {"http", "https"}
or not parsed.netloc
or parsed.username is not None
or parsed.password is not None
or parsed.query
or parsed.fragment
):
raise HTTPException(
status_code=400,
detail="HTTP 接口地址必须是无凭据、无查询参数的 http(s) 地址",
)
normalised_methods: list[str] = []
for raw_method in allowed_methods:
method = str(raw_method).strip().upper()
if method not in _HTTP_METHODS:
raise HTTPException(status_code=400, detail=f"不支持的 HTTP 方法:{method}")
if method not in normalised_methods:
normalised_methods.append(method)
if not normalised_methods:
raise HTTPException(status_code=400, detail="至少选择一种 HTTP 方法")
return target, normalised_methods
def _normalise_headers(headers: dict[str, str] | None) -> dict[str, str]:
normalised: dict[str, str] = {}
for raw_name, raw_value in (headers or {}).items():
name = str(raw_name).strip()
value = str(raw_value)
if not name or any(char in name + value for char in "\r\n"):
raise HTTPException(status_code=400, detail="HTTP 请求头格式无效")
normalised[name] = value
return normalised
def _secret_from(kind: str, dsn: str | None, headers: dict[str, str] | None) -> str:
if kind == "http":
return json.dumps({"headers": _normalise_headers(headers)}, ensure_ascii=False)
if not dsn:
raise HTTPException(status_code=400, detail="SQL 数据源需要提供 dsn")
if "://" not in dsn:
raise HTTPException(status_code=400, detail="dsn 格式无效")
return dsn
@router.get("")
async def list_data_sources(request: Request) -> dict[str, Any]:
_require_enabled()
user_id = await get_current_user(request)
rows = await _store(request).list_sources(owner_id=user_id)
return {"dataSources": rows, "secretsEncryptionAvailable": secrets_available()}
@router.post("")
async def create_data_source(request: Request, body: DataSourceCreate) -> dict[str, Any]:
_require_enabled()
user_id = await get_current_user(request)
is_admin = await _is_admin(request)
if body.shared and not is_admin:
raise HTTPException(status_code=403, detail="仅管理员可创建共享数据源")
secret = _secret_from(body.kind, body.dsn, body.headers)
base_url = ""
allowed_methods: list[str] = []
if body.kind == "http":
base_url, allowed_methods = _normalise_http_metadata(
body.base_url,
body.allowed_methods,
)
try:
created = await _store(request).create_source(
{
"name": body.name,
"description": body.description,
"kind": body.kind,
"dsn": secret,
"base_url": base_url,
"allowed_methods": allowed_methods,
"max_rows": body.max_rows,
"allowed_tables": body.allowed_tables,
"enabled": body.enabled,
"owner_id": None if body.shared else user_id,
"created_by": user_id,
}
)
except SecretKeyMissingError as exc:
raise HTTPException(
status_code=503,
detail={
"code": "WORKFLOW_RESOURCE_MISSING",
"message": "服务端未配置 WORKFLOW_SECRET_KEY,无法安全保存连接信息",
},
) from exc
audit_workflow("data_source.create", user_id=user_id, dataSourceId=created["id"], kind=body.kind, shared=body.shared)
return created
async def _owned(request: Request, source_id: str) -> dict[str, Any]:
row = await _store(request).get_source(source_id)
if row is None:
raise HTTPException(status_code=404, detail="数据源不存在")
user_id = await get_current_user(request)
if row.get("owner_id") == user_id:
return row
if await _is_admin(request):
return row
raise HTTPException(status_code=403, detail="无权操作该数据源")
async def _visible(request: Request, source_id: str) -> dict[str, Any]:
row = await _store(request).get_source(source_id)
if row is None:
raise HTTPException(status_code=404, detail="数据源不存在")
user_id = await get_current_user(request)
if row.get("owner_id") in (user_id, None):
return row
if await _is_admin(request):
return row
raise HTTPException(status_code=403, detail="无权查看该数据源")
@router.get("/{source_id}")
async def get_data_source(request: Request, source_id: str) -> dict[str, Any]:
_require_enabled()
return await _visible(request, source_id)
@router.post("/{source_id}/introspect")
async def introspect_data_source(request: Request, source_id: str) -> dict[str, Any]:
"""Return the configured table allowlist only — never sample rows."""
_require_enabled()
row = await _visible(request, source_id)
tables = list(row.get("allowed_tables") or [])
return {
"dataSourceId": row["id"],
"kind": row.get("kind"),
"tables": [{"name": name} for name in tables],
"schemas": [],
}
@router.post("/{source_id}/validate-query")
async def validate_data_source_query(request: Request, source_id: str, body: QueryBody) -> dict[str, Any]:
"""Validate SQL against the read-only policy without executing it."""
_require_enabled()
row = await _visible(request, source_id)
try:
statement = assert_read_only(body.statement)
assert_tables_allowed(statement, list(row.get("allowed_tables") or []))
except Exception as exc: # noqa: BLE001
from deerflow.workflows.errors import WorkflowError
if isinstance(exc, WorkflowError):
raise HTTPException(status_code=400, detail=exc.to_body().model_dump(by_alias=True)) from exc
raise
audit_workflow("data_source.validate_query", user_id=await get_current_user(request), dataSourceId=source_id)
return {"ok": True, "statement": statement}
@router.put("/{source_id}")
async def update_data_source(request: Request, source_id: str, body: DataSourceUpdate) -> dict[str, Any]:
_require_enabled()
row = await _owned(request, source_id)
payload = body.model_dump(exclude_unset=True, by_alias=False)
kind = str(row.get("kind") or "sql")
if body.dsn is not None or body.headers is not None:
payload["dsn"] = _secret_from(kind, body.dsn, body.headers)
payload.pop("headers", None)
if kind == "http":
base_url, allowed_methods = _normalise_http_metadata(
body.base_url if body.base_url is not None else str(row.get("base_url") or ""),
body.allowed_methods
if body.allowed_methods is not None
else list(row.get("allowed_methods") or []),
)
payload["base_url"] = base_url
payload["allowed_methods"] = allowed_methods
else:
payload.pop("base_url", None)
payload.pop("allowed_methods", None)
try:
updated = await _store(request).update_source(source_id, payload)
except SecretKeyMissingError as exc:
raise HTTPException(status_code=503, detail="服务端未配置 WORKFLOW_SECRET_KEY") from exc
if updated is None:
raise HTTPException(status_code=404, detail="数据源不存在")
audit_workflow("data_source.update", user_id=await get_current_user(request), dataSourceId=source_id)
return updated
@router.delete("/{source_id}")
async def delete_data_source(request: Request, source_id: str) -> dict[str, Any]:
_require_enabled()
await _owned(request, source_id)
deleted = await _store(request).delete_source(source_id)
audit_workflow("data_source.delete", user_id=await get_current_user(request), dataSourceId=source_id)
return {"deleted": deleted}
@@ -0,0 +1,186 @@
"""iframe embedding support: origin allowlist + short-lived signed tickets.
Scope, stated plainly: a ticket proves *"this workflow may be embedded by this
origin, for this user, right now"*. It is **not** an API credential — embedded
pages still authenticate their API calls with the normal session, exactly like
the rest of the app's iframe surfaces. What the ticket adds is an explicit
origin decision that can be audited, plus the ``frame-ancestors`` value the host
should send.
Tickets are stateless HS256 JWTs (so any worker can verify them) with a 60s
default TTL. Single-use is enforced best-effort per process via a replay cache;
the short TTL is the real protection.
"""
from __future__ import annotations
import hashlib
import logging
import os
import time
from typing import Any
from urllib.parse import urlparse
import jwt
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.deps import get_current_user
from deerflow.config.app_config import get_app_config
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/workflows/embed", tags=["workflow-embed"])
_ALGORITHM = "HS256"
_AUDIENCE = "workflow-embed"
_replay_cache: dict[str, float] = {}
_REPLAY_CACHE_MAX = 4096
class TicketRequest(BaseModel):
workflow_id: str = Field(alias="workflowId")
origin: str
model_config = {"populate_by_name": True}
class VerifyRequest(BaseModel):
ticket: str
origin: str | None = None
def _signing_key() -> str:
for name in ("WORKFLOW_SECRET_KEY", "DEERFLOW_SECRET_KEY", "JWT_SECRET_KEY"):
value = os.environ.get(name)
if value and value.strip():
return hashlib.sha256(value.strip().encode("utf-8")).hexdigest()
raise HTTPException(
status_code=503,
detail={
"code": "WORKFLOW_EMBED_TICKET_INVALID",
"message": "服务端未配置签名密钥,无法签发嵌入票据",
},
)
def _normalize_origin(raw: str) -> str:
parsed = urlparse(raw.strip())
if not parsed.scheme or not parsed.netloc:
raise HTTPException(
status_code=400,
detail={"code": "WORKFLOW_EMBED_ORIGIN_DENIED", "message": "origin 格式无效"},
)
return f"{parsed.scheme.lower()}://{parsed.netloc.lower()}"
def _assert_origin_allowed(origin: str) -> str:
cfg = get_app_config().workflows.embed
normalized = _normalize_origin(origin)
allowed = {_normalize_origin(o) for o in cfg.allowed_origins if o.strip()}
if not allowed:
raise HTTPException(
status_code=403,
detail={
"code": "WORKFLOW_EMBED_ORIGIN_DENIED",
"message": "未配置任何允许的嵌入来源(workflows.embed.allowed_origins)",
},
)
if normalized not in allowed:
logger.warning("workflow embed origin denied: %s", normalized)
raise HTTPException(
status_code=403,
detail={"code": "WORKFLOW_EMBED_ORIGIN_DENIED", "message": "该来源不允许嵌入"},
)
return normalized
def _prune_replay_cache(now: float) -> None:
for key, expiry in list(_replay_cache.items()):
if expiry <= now:
_replay_cache.pop(key, None)
if len(_replay_cache) > _REPLAY_CACHE_MAX:
_replay_cache.clear()
@router.post("/tickets")
async def issue_ticket(request: Request, body: TicketRequest) -> dict[str, Any]:
cfg = get_app_config().workflows
if not cfg.enabled:
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
user_id = await get_current_user(request)
origin = _assert_origin_allowed(body.origin)
definitions = getattr(request.app.state, "workflow_store", None)
if definitions is None:
raise HTTPException(status_code=503, detail="Workflow store not available")
definition = await definitions.get_definition(body.workflow_id, include_draft=False)
if definition is None:
raise HTTPException(status_code=404, detail="工作流不存在")
if definition.get("owner_id") != user_id:
raise HTTPException(status_code=403, detail={"code": "WORKFLOW_FORBIDDEN", "message": "无权嵌入该工作流"})
now = int(time.time())
ttl = cfg.embed.ticket_ttl_seconds
payload = {
"aud": _AUDIENCE,
"sub": user_id,
"wf": body.workflow_id,
"org": origin,
"iat": now,
"exp": now + ttl,
"jti": hashlib.sha256(f"{user_id}{body.workflow_id}{origin}{now}".encode()).hexdigest()[:32],
}
ticket = jwt.encode(payload, _signing_key(), algorithm=_ALGORITHM)
logger.info(
"workflow embed ticket issued workflow=%s user=%s origin=%s ttl=%ss",
body.workflow_id,
user_id,
origin,
ttl,
)
return {
"ticket": ticket,
"expiresIn": ttl,
"origin": origin,
"frameAncestors": sorted({_normalize_origin(o) for o in cfg.embed.allowed_origins if o.strip()}),
}
@router.post("/verify")
async def verify_ticket(body: VerifyRequest) -> dict[str, Any]:
cfg = get_app_config().workflows
if not cfg.enabled:
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
try:
payload = jwt.decode(body.ticket, _signing_key(), algorithms=[_ALGORITHM], audience=_AUDIENCE)
except jwt.PyJWTError as exc:
raise HTTPException(
status_code=401,
detail={"code": "WORKFLOW_EMBED_TICKET_INVALID", "message": "票据无效或已过期"},
) from exc
if body.origin and _normalize_origin(body.origin) != payload.get("org"):
raise HTTPException(
status_code=403,
detail={"code": "WORKFLOW_EMBED_ORIGIN_DENIED", "message": "票据与当前来源不匹配"},
)
now = time.time()
_prune_replay_cache(now)
jti = str(payload.get("jti") or "")
if jti and jti in _replay_cache:
raise HTTPException(
status_code=401,
detail={"code": "WORKFLOW_EMBED_TICKET_INVALID", "message": "票据已被使用"},
)
if jti:
_replay_cache[jti] = float(payload.get("exp") or now + 60)
return {
"ok": True,
"workflowId": payload.get("wf"),
"userId": payload.get("sub"),
"origin": payload.get("org"),
"expiresAt": payload.get("exp"),
}
@@ -0,0 +1,236 @@
"""Workflow node-type and resource catalog APIs (phase 1)."""
from __future__ import annotations
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from app.gateway.deps import get_current_user, get_local_provider
from deerflow.config.app_config import get_app_config
from deerflow.persistence.workflows import WorkflowStore
from deerflow.workflows.schemas import ALL_NODE_TYPES
router = APIRouter(prefix="/api/workflows", tags=["workflow-resources"])
_NODE_META: dict[str, dict[str, str]] = {
"start": {"label": "开始", "category": "control"},
"output": {"label": "输出", "category": "control"},
"agent": {"label": "智能体", "category": "ai"},
"skill": {"label": "技能", "category": "ai"},
"http": {"label": "HTTP", "category": "data"},
"sql_read": {"label": "SQL 只读", "category": "data"},
"code": {"label": "代码", "category": "data"},
"transform": {"label": "模板/转换", "category": "data"},
"condition": {"label": "条件", "category": "control"},
"merge": {"label": "合并", "category": "control"},
"human_input": {"label": "人工介入", "category": "control"},
"subworkflow": {"label": "子工作流", "category": "control"},
"loop": {"label": "循环", "category": "control"},
}
def _require_enabled() -> None:
if not get_app_config().workflows.enabled:
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
def _get_store(request: Request) -> WorkflowStore:
store = getattr(request.app.state, "workflow_store", None)
if store is None:
raise HTTPException(status_code=503, detail="Workflow store not available")
return store
async def _resolve_creator_names(user_ids: set[str]) -> dict[str, str]:
"""Resolve resource-owner ids to short display names, best-effort.
The workflow catalog is a small display projection over the agent and
skill stores. Those stores intentionally retain immutable owner ids for
access control; this resolver exposes a friendly creator name without
changing that ownership model. If an account has since disappeared, the
stable id remains a useful audit fallback.
"""
if not user_ids:
return {}
try:
provider = get_local_provider()
except Exception: # noqa: BLE001
return {user_id: user_id for user_id in user_ids}
names: dict[str, str] = {}
for user_id in user_ids:
try:
user = await provider.get_user(user_id)
except Exception: # noqa: BLE001
user = None
email = getattr(user, "email", None) if user is not None else None
if isinstance(email, str) and email:
names[user_id] = email.split("@", 1)[0] or user_id
else:
names[user_id] = user_id
return names
def _creator_label(owner_id: object, creator_names: dict[str, str]) -> str:
"""Return the catalog label for a user-owned or built-in resource."""
if owner_id is None or not str(owner_id).strip():
return "系统内置"
owner_key = str(owner_id)
return creator_names.get(owner_key, owner_key)
@router.get("/node-types")
async def list_node_types(request: Request) -> dict[str, Any]:
_require_enabled()
await get_current_user(request)
items = []
for node_type in ALL_NODE_TYPES:
meta = _NODE_META.get(node_type, {"label": node_type, "category": "other"})
items.append({"type": node_type, **meta})
return {"nodeTypes": items}
@router.get("/resources/agents")
async def list_agent_resources(request: Request) -> dict[str, Any]:
_require_enabled()
user_id = await get_current_user(request)
agent_store = getattr(request.app.state, "agent_store", None)
agents: list[dict[str, Any]] = []
if agent_store is not None:
try:
if hasattr(agent_store, "list_visible"):
rows = await agent_store.list_visible(user_id)
elif hasattr(agent_store, "list_all"):
rows = await agent_store.list_all()
else:
rows = []
creator_names = await _resolve_creator_names({str(row["user_id"]) for row in rows or [] if row.get("user_id") is not None})
agent_ids = [str(row.get("id") or row.get("agent_id") or "") for row in rows or []]
extras_by_agent: dict[str, dict[str, Any]] = {}
if hasattr(agent_store, "get_extras_for"):
extras_by_agent = await agent_store.get_extras_for([agent_id for agent_id in agent_ids if agent_id])
keyword = request.query_params.get("keyword", "").strip().casefold()
for row in rows or []:
agent_id = str(row.get("id") or row.get("agent_id") or "")
name = str(row.get("name") or agent_id)
description = str(row.get("description") or "")
if keyword and keyword not in f"{agent_id} {name} {description}".casefold():
continue
cached_skills = (extras_by_agent.get(agent_id) or {}).get("skills") or []
agents.append(
{
"agentId": agent_id,
"name": name,
"description": description,
"skills": [str(skill) for skill in cached_skills],
"createdBy": _creator_label(row.get("user_id"), creator_names),
}
)
except Exception: # noqa: BLE001
agents = []
return {"agents": agents}
@router.get("/resources/skills")
async def list_skill_resources(request: Request) -> dict[str, Any]:
_require_enabled()
user_id = await get_current_user(request)
skills: list[dict[str, Any]] = []
try:
from deerflow.skills import get_or_new_skill_storage
skill_store = getattr(request.app.state, "skill_store", None)
ownership_by_name: dict[str, dict[str, Any]] = {}
if skill_store is not None and hasattr(skill_store, "list_visible"):
records = await skill_store.list_visible(user_id)
ownership_by_name = {str(record["name"]): record for record in records if record.get("name")}
creator_names = await _resolve_creator_names({str(record["owner_user_id"]) for record in ownership_by_name.values() if record.get("owner_user_id") is not None})
storage = get_or_new_skill_storage()
for skill in storage.load_skills(enabled_only=True) or []:
name = getattr(skill, "name", None)
if not name:
continue
record = ownership_by_name.get(str(name), {})
skills.append(
{
"skillId": name,
"name": name,
"description": getattr(skill, "description", "") or "",
# Workflow direct-skill execution remains opt-in. Older
# Skill definitions do not advertise this capability and
# are therefore correctly shown as agent-only resources.
"callable": bool(getattr(skill, "callable", False)),
"createdBy": _creator_label(record.get("owner_user_id"), creator_names),
}
)
except Exception: # noqa: BLE001
skills = []
return {"skills": skills}
@router.get("/resources/data-sources")
async def list_data_source_resources(request: Request) -> dict[str, Any]:
_require_enabled()
user_id = await get_current_user(request)
store = getattr(request.app.state, "workflow_data_source_store", None)
if store is None:
return {"dataSources": []}
rows = await store.list_sources(owner_id=user_id)
return {
"dataSources": [
{
"id": row["id"],
"dataSourceId": row["id"],
"name": row["name"],
"description": row.get("description") or "",
"kind": row.get("kind"),
"engine": row.get("driver") or "",
"baseUrl": row.get("base_url") or "",
"maskedTarget": row.get("masked_target"),
"enabled": row.get("enabled"),
"readOnly": row.get("kind") == "sql",
"allowedMethods": row.get("allowed_methods") or [],
"allowedTables": row.get("allowed_tables") or [],
}
for row in rows
if row.get("enabled", True)
]
}
@router.get("/resources/credentials")
async def list_credential_resources(request: Request) -> dict[str, Any]:
"""Read-only credential directory: ids and labels only.
HTTP credentials are ``http``-kind data sources; the encrypted header blob
is resolved exclusively by the run executor. This endpoint must never
return headers, tokens, DSNs or ciphertext — a 404-free catalog so the
canvas credential picker does not fail the whole resource panel.
"""
_require_enabled()
user_id = await get_current_user(request)
store = getattr(request.app.state, "workflow_data_source_store", None)
if store is None:
return {"credentials": []}
rows = await store.list_sources(owner_id=user_id, include_shared=True)
return {
"credentials": [
{
"credentialId": row["id"],
"name": row["name"],
"type": "http_headers",
"sourceKind": "http",
}
for row in rows
if str(row.get("kind") or "") == "http" and row.get("enabled", True)
]
}
@router.get("/resources/subworkflows")
async def list_subworkflow_resources(request: Request) -> dict[str, Any]:
_require_enabled()
await get_current_user(request)
store = _get_store(request)
return {"subworkflows": await store.list_published_summaries()}
@@ -0,0 +1,700 @@
"""Workflow run APIs: start / inspect / cancel / resume / retry / events / SSE stream."""
from __future__ import annotations
import asyncio
import json
import logging
from datetime import UTC, datetime
from typing import Any
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from app.gateway.deps import get_current_user, get_optional_user_from_request
from app.gateway.workflow_audit import audit_workflow
from deerflow.config.app_config import get_app_config
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
from deerflow.workflows.runtime.sink import envelope_from_row
from deerflow.workflows.schema_validation import schema_issues
from deerflow.workflows.schemas import WorkflowGraph
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api/workflows", tags=["workflow-runs"])
_TERMINAL_STATUSES = ("completed", "failed", "cancelled")
_TERMINAL_EVENTS = ("run.completed", "run.failed", "run.cancelled")
_REPLAY_PAGE = 200
_DB_POLL_SECONDS = 0.8
_LIVE_WAIT_SECONDS = 0.25
_HEARTBEAT_SECONDS = 15.0
_STREAM_DEADLINE_SECONDS = 60 * 60
class StartRunBody(BaseModel):
version_id: str | None = Field(default=None, alias="versionId")
inputs: dict[str, Any] = Field(default_factory=dict)
# 独立 Coze 工作流页早期使用 input/target;保留兼容入口,内部仍只使用 inputs。
input: dict[str, Any] | None = None
target: str | None = None
idempotency_key: str | None = Field(default=None, alias="idempotencyKey", max_length=128)
model_config = {"populate_by_name": True}
def resolved_inputs(self) -> dict[str, Any]:
return self.inputs if self.inputs or self.input is None else self.input
class ResumeRunBody(BaseModel):
resume_token: str = Field(alias="resumeToken")
node_id: str | None = Field(default=None, alias="nodeId")
action: str = "submit"
values: dict[str, Any] = Field(default_factory=dict)
# 兼容独立页的 ResumeRunRequest:payload/message。
payload: dict[str, Any] | None = None
message: str | None = None
model_config = {"populate_by_name": True}
def resolved_values(self) -> dict[str, Any]:
return self.values if self.values or self.payload is None else self.payload
# ── shared plumbing ─────────────────────────────────────────────────────
def _cfg():
return get_app_config().workflows
def _require_enabled() -> None:
if not _cfg().enabled:
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
def _runs(request: Request) -> WorkflowRunStore:
store = getattr(request.app.state, "workflow_run_store", None)
if store is None:
raise HTTPException(status_code=503, detail="Workflow run store not available")
return store
def _events(request: Request) -> WorkflowEventStore:
store = getattr(request.app.state, "workflow_event_store", None)
if store is None:
raise HTTPException(status_code=503, detail="Workflow event store not available")
return store
def _definitions(request: Request):
store = getattr(request.app.state, "workflow_store", None)
if store is None:
raise HTTPException(status_code=503, detail="Workflow store not available")
return store
def _nudge(request: Request) -> None:
dispatcher = getattr(request.app.state, "workflow_dispatcher", None)
if dispatcher is not None:
dispatcher.nudge()
async def _is_admin(request: Request) -> bool:
user = await get_optional_user_from_request(request)
return user is not None and getattr(user, "system_role", None) == "admin"
async def _authorized_run(request: Request, run_id: str) -> dict[str, Any]:
run = await _runs(request).get_run(run_id)
if run is None:
raise HTTPException(status_code=404, detail={"code": "WORKFLOW_RUN_NOT_FOUND", "message": "运行不存在"})
user_id = await get_current_user(request)
if run.get("owner_id") != user_id and not await _is_admin(request):
raise HTTPException(status_code=403, detail={"code": "WORKFLOW_FORBIDDEN", "message": "无权访问该运行"})
return run
def _public_run(run: dict[str, Any]) -> dict[str, Any]:
"""Run view for HTTP clients: no lease internals, no resume token."""
return {
"id": run["id"],
"runId": run["id"],
"workflowId": run["workflow_id"],
"versionId": run["workflow_version_id"],
"status": run["status"],
"ownerId": run["owner_id"],
"input": run.get("input") or {},
"output": run.get("output"),
"error": run.get("error"),
"pendingInput": run.get("pending_input"),
"attempt": run.get("attempt"),
"retryOfRunId": run.get("retry_of_run_id"),
"startedAt": run.get("started_at"),
"finishedAt": run.get("finished_at"),
"createdAt": run.get("created_at"),
"updatedAt": run.get("updated_at"),
# The run row's counter holds the last seq handed out by the event log
# (the event store bumps it in the insert transaction); 0 = no events.
"lastSeq": int(run.get("next_event_seq") or 0),
}
# ── start / list / get / cancel / resume ────────────────────────────────
@router.post("/{workflow_id}/runs")
async def start_run(request: Request, workflow_id: str, body: StartRunBody) -> dict[str, Any]:
_require_enabled()
user_id = await get_current_user(request)
if str(body.target or "").strip().lower() == "draft":
# Never silently fall back to the published version when the client
# thinks it is exercising the draft — draft snapshots need the explicit
# test-run API (not built yet).
raise HTTPException(
status_code=422,
detail={
"code": "WORKFLOW_DRAFT_RUN_UNSUPPORTED",
"message": "暂不支持运行草稿,请先发布或指定 versionId",
},
)
definitions = _definitions(request)
definition = await definitions.get_definition(workflow_id, include_draft=False)
if definition is None:
raise HTTPException(status_code=404, detail="工作流不存在")
if definition.get("owner_id") != user_id and not await _is_admin(request):
raise HTTPException(status_code=403, detail={"code": "WORKFLOW_FORBIDDEN", "message": "无权运行该工作流"})
version_id = body.version_id
if not version_id:
versions = await definitions.list_versions(workflow_id)
if not versions:
raise HTTPException(
status_code=400,
detail={"code": "WORKFLOW_VERSION_NOT_FOUND", "message": "工作流尚未发布任何版本"},
)
version_id = versions[0]["id"]
version = await definitions.get_version(version_id)
if version is None or version.get("workflow_id") != workflow_id:
raise HTTPException(status_code=404, detail={"code": "WORKFLOW_VERSION_NOT_FOUND", "message": "版本不存在"})
graph = WorkflowGraph.model_validate(version.get("graph") or {})
inputs = body.resolved_inputs()
missing = [key for key in (graph.input_schema.get("required") or []) if inputs.get(str(key)) in (None, "")]
if missing:
raise HTTPException(
status_code=400,
detail={
"code": "WORKFLOW_INPUT_INVALID",
"message": f"缺少必填输入:{'、'.join(str(m) for m in missing)}",
"details": {"missing": missing},
},
)
# Full JSON Schema pass so type/enum/nested mistakes fail here with a
# structured 400 instead of surfacing mid-run as a node error.
schema_errors = schema_issues(inputs, graph.input_schema or {})
if schema_errors:
raise HTTPException(
status_code=400,
detail={
"code": "WORKFLOW_INPUT_INVALID",
"message": "输入不符合 schema:" + ";".join(e["message"] for e in schema_errors[:3]),
"details": {"errors": schema_errors},
},
)
runs = _runs(request)
active = [r for r in await runs.list_runs(owner_id=user_id, limit=200) if r["status"] in ("queued", "running")]
if len(active) >= _cfg().max_concurrent_runs_per_user:
raise HTTPException(
status_code=429,
detail={
"code": "WORKFLOW_LIMIT_EXCEEDED",
"message": f"同时运行的工作流已达上限({_cfg().max_concurrent_runs_per_user})",
},
)
# Scope the idempotency key per workflow so two workflows can share a client key.
key = f"{workflow_id}:{body.idempotency_key}" if body.idempotency_key else None
run, created = await runs.create_run(
{
"workflow_id": workflow_id,
"workflow_version_id": version_id,
"owner_id": user_id,
"input": inputs,
"idempotency_key": key,
"context": {"depth": 0},
}
)
if created:
await _events(request).append(
run_id=run["id"],
workflow_id=workflow_id,
version_id=version_id,
event_type="run.created",
payload={"inputs": inputs},
)
audit_workflow("run.start", user_id=user_id, workflowId=workflow_id, runId=run["id"], versionId=version_id)
_nudge(request)
return {**_public_run(run), "created": created}
@router.get("/{workflow_id}/runs")
async def list_workflow_runs(
request: Request,
workflow_id: str,
status: str | None = None,
limit: int = Query(default=50, ge=1, le=200),
offset: int = Query(default=0, ge=0),
) -> dict[str, Any]:
_require_enabled()
user_id = await get_current_user(request)
owner = None if await _is_admin(request) else user_id
rows = await _runs(request).list_runs(workflow_id=workflow_id, owner_id=owner, status=status, limit=limit, offset=offset)
return {"runs": [_public_run(r) for r in rows], "hasMore": len(rows) >= limit}
@router.get("/runs/{run_id}")
async def get_run(request: Request, run_id: str) -> dict[str, Any]:
_require_enabled()
run = await _authorized_run(request, run_id)
node_runs = await _runs(request).list_node_runs(run_id, current_only=True)
return {
**_public_run(run),
"nodeRuns": [
{
"nodeId": r["node_id"],
"nodeType": r["node_type"],
"status": r["status"],
"attempt": r["attempt"],
"iteration": r["iteration"],
"durationMs": r["duration_ms"],
"error": r["error"],
"startedAt": r["started_at"],
"finishedAt": r["finished_at"],
}
for r in node_runs
],
}
@router.get("/runs/{run_id}/artifacts")
async def list_run_artifacts(request: Request, run_id: str) -> dict[str, Any]:
_require_enabled()
await _authorized_run(request, run_id)
rows = await _runs(request).list_artifacts(run_id)
return {
"artifacts": [
{
"artifactId": r["id"],
"nodeId": r["node_id"],
"name": r["name"],
"mimeType": r["mime_type"],
"sizeBytes": r["size_bytes"],
"preview": r["preview"],
}
for r in rows
]
}
# ── artifact content / download ─────────────────────────────────────────
_PREVIEW_MAX_BYTES = 256 * 1024
_TEXT_MIME_PREFIXES = ("text/", "application/json", "application/javascript", "application/x-yaml", "application/xml", "application/csv")
def _is_textual(mime_type: str) -> bool:
return (mime_type or "").lower().startswith(_TEXT_MIME_PREFIXES)
def _read_prefix(path: str, limit: int) -> tuple[bytes, bool]:
"""Read up to ``limit + 1`` bytes; the extra byte detects truncation."""
with open(path, "rb") as handle:
data = handle.read(limit + 1)
return data[:limit], len(data) > limit
def _file_exists(path: str) -> bool:
import os
return bool(path) and os.path.isfile(path)
async def _authorized_artifact(request: Request, run_id: str, artifact_id: str) -> dict[str, Any]:
"""Run-level authorization first, then artifact-must-belong-to-run.
The file path is resolved exclusively from the registered artifact row —
clients address artifacts by id and can never pass a sandbox path in.
"""
await _authorized_run(request, run_id)
artifact = await _runs(request).get_artifact(run_id, artifact_id)
if artifact is None:
raise HTTPException(
status_code=404,
detail={"code": "WORKFLOW_ARTIFACT_NOT_FOUND", "message": "制品不存在"},
)
return artifact
@router.get("/runs/{run_id}/artifacts/{artifact_id}")
async def get_run_artifact_content(request: Request, run_id: str, artifact_id: str) -> dict[str, Any]:
"""Size-bounded text preview for file cards; binaries stay download-only."""
from starlette.concurrency import run_in_threadpool
from deerflow.workflows.events import artifact_kind
_require_enabled()
artifact = await _authorized_artifact(request, run_id, artifact_id)
mime = str(artifact.get("mime_type") or "application/octet-stream")
kind = artifact_kind(mime)
base = {
"artifactId": artifact["id"],
"name": artifact["name"],
"kind": kind,
"mimeType": mime,
"sizeBytes": artifact.get("size_bytes"),
}
path = str(artifact.get("path") or "")
file_ok = bool(path) and await run_in_threadpool(_file_exists, path)
if file_ok:
raw, truncated = (b"", False)
if _is_textual(mime):
raw, truncated = await run_in_threadpool(_read_prefix, path, _PREVIEW_MAX_BYTES)
return {
**base,
"text": raw.decode("utf-8", errors="replace"),
"truncated": truncated,
"source": "file",
}
return {
**base,
"binary": True,
"downloadUrl": f"/api/workflows/runs/{run_id}/artifacts/{artifact_id}/download",
"source": "file",
}
preview = artifact.get("preview")
if preview:
return {**base, "text": str(preview), "truncated": True, "source": "preview"}
raise HTTPException(
status_code=404,
detail={"code": "WORKFLOW_ARTIFACT_CONTENT_UNAVAILABLE", "message": "制品内容不可用(文件已过期)"},
)
@router.get("/runs/{run_id}/artifacts/{artifact_id}/download")
async def download_run_artifact(request: Request, run_id: str, artifact_id: str):
from urllib.parse import quote
from starlette.concurrency import run_in_threadpool
from starlette.responses import FileResponse, Response
_require_enabled()
artifact = await _authorized_artifact(request, run_id, artifact_id)
mime = str(artifact.get("mime_type") or "application/octet-stream")
name = str(artifact.get("name") or artifact_id)
disposition = f"attachment; filename*=UTF-8''{quote(name)}"
path = str(artifact.get("path") or "")
if path and await run_in_threadpool(_file_exists, path):
return FileResponse(path, media_type=mime, headers={"Content-Disposition": disposition})
preview = artifact.get("preview")
if preview:
return Response(
content=str(preview),
media_type=mime if _is_textual(mime) else "text/plain",
headers={"Content-Disposition": disposition},
)
raise HTTPException(
status_code=404,
detail={"code": "WORKFLOW_ARTIFACT_CONTENT_UNAVAILABLE", "message": "制品文件已过期,无法下载"},
)
@router.post("/runs/{run_id}/cancel")
async def cancel_run(request: Request, run_id: str) -> dict[str, Any]:
_require_enabled()
run = await _authorized_run(request, run_id)
if run["status"] in _TERMINAL_STATUSES:
return {**_public_run(run), "cancelled": False}
updated = await _runs(request).request_cancel(run_id)
if updated is None:
raise HTTPException(
status_code=409,
detail={"code": "WORKFLOW_RUN_CONFLICT", "message": "当前状态无法取消"},
)
await _events(request).append(
run_id=run_id,
workflow_id=run["workflow_id"],
version_id=run["workflow_version_id"],
event_type="run.cancel_requested",
payload={},
)
_nudge(request)
audit_workflow("run.cancel", user_id=await get_current_user(request), runId=run_id, workflowId=run["workflow_id"])
return {**_public_run(updated), "cancelled": True}
@router.post("/runs/{run_id}/resume")
async def resume_run(request: Request, run_id: str, body: ResumeRunBody) -> dict[str, Any]:
_require_enabled()
run = await _authorized_run(request, run_id)
if run["status"] != "awaiting_input":
raise HTTPException(
status_code=409,
detail={"code": "WORKFLOW_RUN_NOT_RESUMABLE", "message": "运行当前不在等待人工输入"},
)
pending = run.get("pending_input") or {}
node_id = body.node_id or str(pending.get("nodeId") or "")
if not node_id:
raise HTTPException(
status_code=400,
detail={"code": "WORKFLOW_HUMAN_INPUT_INVALID", "message": "缺少 nodeId"},
)
runs = _runs(request)
# Single conditional update: token validation + payload write + token
# consumption + status flip all in one statement, so two concurrent resume
# requests with the same token cannot interleave — exactly one succeeds and
# the loser gets 409 without ever touching the run.
resumed = await runs.resume_with_payload(
run_id,
body.resume_token,
{
"nodeId": node_id,
"action": body.action,
"values": body.resolved_values(),
**({"message": body.message} if body.message else {}),
},
context=dict(run.get("context") or {}),
)
if resumed is None:
raise HTTPException(
status_code=409,
detail={"code": "WORKFLOW_RESUME_TOKEN_INVALID", "message": "恢复令牌无效或已被使用"},
)
await _events(request).append(
run_id=run_id,
workflow_id=run["workflow_id"],
version_id=run["workflow_version_id"],
event_type="run.resumed",
payload={"nodeId": node_id, "action": body.action},
node_id=node_id,
)
_nudge(request)
audit_workflow("run.resume", user_id=await get_current_user(request), runId=run_id, nodeId=node_id)
return _public_run(resumed)
@router.post("/runs/{run_id}/retry", status_code=201)
async def retry_run(request: Request, run_id: str) -> dict[str, Any]:
"""Start a new run from a failed one.
Completed node outputs are copied onto the new run so the engine resumes
from the first unfinished node. If there is nothing to replay, the new run
is a full replay of the same version and inputs, still marked
``retry_of_run_id``.
"""
_require_enabled()
run = await _authorized_run(request, run_id)
if run["status"] != "failed":
raise HTTPException(
status_code=409,
detail={"code": "WORKFLOW_RUN_NOT_RETRYABLE", "message": "只有失败的运行可以重试"},
)
user_id = await get_current_user(request)
runs = _runs(request)
active = [r for r in await runs.list_runs(owner_id=user_id, limit=200) if r["status"] in ("queued", "running")]
if len(active) >= _cfg().max_concurrent_runs_per_user:
raise HTTPException(
status_code=429,
detail={
"code": "WORKFLOW_LIMIT_EXCEEDED",
"message": f"同时运行的工作流已达上限({_cfg().max_concurrent_runs_per_user})",
},
)
context = dict(run.get("context") or {})
context.pop("resume_payload", None)
context["retry_of_run_id"] = run_id
new_run, _created = await runs.create_run(
{
"workflow_id": run["workflow_id"],
"workflow_version_id": run["workflow_version_id"],
"owner_id": user_id,
"input": run.get("input") or {},
"retry_of_run_id": run_id,
"context": context,
}
)
for row in await runs.list_node_runs(run_id, current_only=True):
if row.get("status") != "completed":
continue
await runs.record_node_run(
{
"run_id": new_run["id"],
"node_id": row["node_id"],
"node_type": row.get("node_type") or "unknown",
"status": "completed",
"attempt": 1,
"iteration": int(row.get("iteration") or 0),
"output": row.get("output"),
"duration_ms": row.get("duration_ms"),
"is_current": True,
}
)
await _events(request).append(
run_id=new_run["id"],
workflow_id=new_run["workflow_id"],
version_id=new_run["workflow_version_id"],
event_type="run.created",
payload={"retryOfRunId": run_id, "inputs": new_run.get("input") or {}},
)
_nudge(request)
audit_workflow("run.retry", user_id=user_id, runId=new_run["id"], retryOfRunId=run_id)
return {**_public_run(new_run), "created": True}
# ── events / SSE ────────────────────────────────────────────────────────
@router.get("/runs/{run_id}/events")
async def list_run_events(
request: Request,
run_id: str,
after: int = Query(default=0, ge=0),
after_seq: int | None = Query(default=None, alias="after_seq", ge=0),
limit: int = Query(default=_REPLAY_PAGE, ge=1, le=1000),
) -> dict[str, Any]:
_require_enabled()
await _authorized_run(request, run_id)
cursor = after_seq if after_seq is not None else after
rows = await _events(request).list_after(run_id, after_seq=cursor, limit=limit)
return {"events": [envelope_from_row(r) for r in rows], "lastSeq": rows[-1]["seq"] if rows else cursor}
def _frame(payload: dict[str, Any]) -> str:
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), default=str)
return f"id: {payload.get('seq', 0)}\nevent: {payload.get('event', 'message')}\ndata: {body}\n\n"
def _replay_cursor(request: Request, after: int) -> int:
"""``Last-Event-ID`` wins over ``?after=``; invalid headers fall back."""
raw = request.headers.get("last-event-id")
if raw:
try:
return max(0, int(str(raw).strip()))
except (TypeError, ValueError):
pass
return after
@router.get("/runs/{run_id}/stream")
async def stream_run(
request: Request,
run_id: str,
after: int = Query(default=0, ge=0),
after_seq: int | None = Query(default=None, alias="after_seq", ge=0),
) -> StreamingResponse:
"""Replay from ``after`` then tail live.
The database is authoritative for ordering: live frames whose ``seq`` we have
not yet replayed are held back and picked up by the next DB poll, so a
reconnecting client never sees a gap or an out-of-order ``seq``.
``Last-Event-ID`` takes priority over ``after``.
"""
_require_enabled()
run = await _authorized_run(request, run_id)
events = _events(request)
runs = _runs(request)
hub = getattr(request.app.state, "workflow_live_hub", None)
cursor_start = _replay_cursor(request, after_seq if after_seq is not None else after)
async def generator():
cursor = cursor_start
live_queue = hub.subscribe(run_id) if hub is not None else None
loop = asyncio.get_running_loop()
deadline = loop.time() + _STREAM_DEADLINE_SECONDS
next_poll = 0.0
last_beat = loop.time()
try:
while loop.time() < deadline:
if await request.is_disconnected():
return
if loop.time() >= next_poll:
while True:
rows = await events.list_after(run_id, after_seq=cursor, limit=_REPLAY_PAGE)
if not rows:
break
for row in rows:
yield _frame(envelope_from_row(row))
cursor = max(cursor, int(row["seq"]))
last_beat = loop.time()
if row["event_type"] in _TERMINAL_EVENTS:
return
if len(rows) < _REPLAY_PAGE:
break
current = await runs.get_run(run_id)
if current is None:
return
if current["status"] in _TERMINAL_STATUSES:
# Terminal state with no terminal event (e.g. a crash
# before the event landed): synthesise a closing frame
# so the client is never left hanging.
yield _frame(
{
"schemaVersion": "1.0",
"runId": run_id,
"workflowId": current["workflow_id"],
"versionId": current["workflow_version_id"],
"seq": cursor,
"event": f"run.{current['status']}",
"timestamp": datetime.now(UTC).isoformat(),
"data": {
"status": current["status"],
"output": current.get("output"),
"error": current.get("error"),
"synthesised": True,
},
}
)
return
next_poll = loop.time() + _DB_POLL_SECONDS
if live_queue is not None:
try:
event = await asyncio.wait_for(live_queue.get(), timeout=_LIVE_WAIT_SECONDS)
except TimeoutError:
event = None
if event is not None and int(event.get("seq") or 0) > cursor:
yield _frame(event)
cursor = int(event["seq"])
last_beat = loop.time()
if event.get("event") in _TERMINAL_EVENTS:
return
else:
await asyncio.sleep(_LIVE_WAIT_SECONDS)
if loop.time() - last_beat >= _HEARTBEAT_SECONDS:
yield ": heartbeat\n\n"
last_beat = loop.time()
finally:
if live_queue is not None and hub is not None:
hub.unsubscribe(run_id, live_queue)
return StreamingResponse(
generator(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache, no-transform",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
"X-Workflow-Run-Status": run["status"],
},
)
@@ -0,0 +1,351 @@
"""Studio facade: camelCase DTOs for the standalone Coze canvas page.
Four endpoints in front of the standard workflow store:
GET /api/workflows/{id}/studio-document
PUT /api/workflows/{id}/studio-draft
POST /api/workflows/{id}/studio-validate
POST /api/workflows/{id}/studio-publish
Design contract (docs/WORKFLOW_STUDIO_COZE_FRONTEND_ADAPTATION_ZH.md §3-§4):
- One workflow, two documents. ``canvasSchemaJson`` is an opaque Coze canvas
saved for editing; ``executionGraph`` is the WorkflowGraph v1.0 the runtime
executes. Both land in the SAME CAS update, so they can never disagree on
revision.
- The facade only maps DTO naming and delegates to the standard store /
validator / publish service — it never forks versioning logic.
- Publishing a workflow whose draft has no valid execution graph is a 400, not
a silent empty-graph publish.
"""
from __future__ import annotations
import json
from hashlib import sha256
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.deps import get_current_user
from app.gateway.workflow_audit import audit_workflow
from app.gateway.workflow_resource_checks import build_resource_catalog, make_resource_checker
from deerflow.config.app_config import get_app_config
from deerflow.persistence.workflows import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
)
from deerflow.workflows.schemas import WorkflowGraph
from deerflow.workflows.validator import validate_workflow_graph
router = APIRouter(prefix="/api/workflows", tags=["workflow-studio"])
# Canvas document guards: it is stored and replayed as an opaque document, so
# the only checks are size / JSON shape / depth / obvious credential keys.
_CANVAS_MAX_CHARS = 2_000_000
_CANVAS_MAX_DEPTH = 64
_CANVAS_SECRET_KEYS = frozenset(
{
"authorization",
"proxy-authorization",
"cookie",
"x-api-key",
"apikey",
"api_key",
"secret",
"client_secret",
"password",
"dsn",
"access_token",
"refresh_token",
"resumetoken",
"resume_token",
}
)
class StudioDraftBody(BaseModel):
expected_revision: int = Field(alias="expectedRevision", ge=0)
# Canvas may arrive as a JSON string or an embedded object; both are
# canonicalised to compact JSON text before storage.
canvas_schema_json: Any = Field(default=None, alias="canvasSchemaJson")
execution_graph: dict[str, Any] = Field(alias="executionGraph")
model_config = {"populate_by_name": True}
class StudioValidateBody(BaseModel):
execution_graph: dict[str, Any] | None = Field(default=None, alias="executionGraph")
model_config = {"populate_by_name": True}
class StudioPublishBody(BaseModel):
expected_revision: int | None = Field(default=None, alias="expectedRevision", ge=0)
change_note: str = Field(default="", alias="changeNote", max_length=2000)
model_config = {"populate_by_name": True}
def _require_enabled() -> None:
if not get_app_config().workflows.enabled:
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
def _get_store(request: Request) -> WorkflowStore:
store = getattr(request.app.state, "workflow_store", None)
if store is None:
raise HTTPException(status_code=503, detail="Workflow store not available")
return store
async def _load_owned(request: Request, workflow_id: str) -> dict[str, Any]:
"""Owner or admin only — the studio edits the draft, so read == write."""
from app.gateway.routers.workflows import _load_owned as _standard_load_owned
return await _standard_load_owned(request, workflow_id, write=True)
def _limits() -> dict[str, int]:
cfg = get_app_config().workflows
return {
"system_max_steps": cfg.max_steps,
"system_max_loop_iterations": cfg.max_loop_iterations,
"system_max_parallelism": cfg.max_parallelism,
}
def _parse_canvas(raw: Any) -> str:
"""Validate + canonicalise the canvas document into compact JSON text."""
if raw is None or raw == "":
return "{}"
if isinstance(raw, str):
text = raw
try:
document = json.loads(text)
except json.JSONDecodeError as exc:
raise HTTPException(
status_code=400,
detail={"code": "WORKFLOW_CANVAS_INVALID", "message": "画布 JSON 无法解析"},
) from exc
elif isinstance(raw, dict):
document = raw
else:
raise HTTPException(
status_code=400,
detail={"code": "WORKFLOW_CANVAS_INVALID", "message": "画布必须是 JSON 对象或 JSON 字符串"},
)
text = json.dumps(document, ensure_ascii=False, separators=(",", ":"))
if len(text) > _CANVAS_MAX_CHARS:
raise HTTPException(
status_code=400,
detail={"code": "WORKFLOW_CANVAS_INVALID", "message": f"画布文档超过 {_CANVAS_MAX_CHARS} 字符上限"},
)
offending = _find_secret_key(document, 0)
if offending:
raise HTTPException(
status_code=400,
detail={
"code": "WORKFLOW_CANVAS_INVALID",
"message": f"画布中出现凭据类字段 {offending},请改用凭据引用",
"field": offending,
},
)
return text
def _find_secret_key(node: Any, depth: int) -> str | None:
if depth > _CANVAS_MAX_DEPTH:
return "<max-depth-exceeded>"
if isinstance(node, dict):
for key, value in node.items():
name = str(key).strip().lower()
if name in _CANVAS_SECRET_KEYS and value not in (None, "", {}, []):
return str(key)
found = _find_secret_key(value, depth + 1)
if found:
return found
elif isinstance(node, list):
for item in node:
found = _find_secret_key(item, depth + 1)
if found:
return found
return None
def _studio_issues(issues: list[Any]) -> list[dict[str, Any]]:
out: list[dict[str, Any]] = []
for issue in issues:
body = issue.model_dump(by_alias=True)
details = body.get("details") or {}
out.append(
{
"code": body.get("code"),
"nodeId": body.get("nodeId"),
"field": details.get("field") if isinstance(details, dict) else None,
"message": body.get("message"),
}
)
return out
async def _validate_with_resources(request: Request, workflow_id: str, raw_graph: dict[str, Any], user_id: str) -> list[Any]:
catalog = await build_resource_catalog(request, user_id=user_id)
checker = make_resource_checker(catalog, config=get_app_config().workflows, current_workflow_id=workflow_id)
return validate_workflow_graph(raw_graph, resource_checker=checker, **_limits())
@router.get("/{workflow_id}/studio-document")
async def get_studio_document(request: Request, workflow_id: str) -> dict[str, Any]:
_require_enabled()
definition = await _load_owned(request, workflow_id)
store = _get_store(request)
graph = definition.get("draft_graph") or {}
versions = await store.list_versions(workflow_id)
return {
"workflowId": definition["id"],
"name": definition.get("name"),
"description": definition.get("description") or "",
"status": definition.get("status"),
"draftRevision": int(definition.get("draft_revision") or 0),
"publishedVersionId": versions[0]["id"] if versions else None,
# Canvas stays an opaque JSON document (string); never null.
"canvasSchemaJson": definition.get("draft_canvas_schema") or "{}",
"executionGraph": graph,
"inputSchema": graph.get("inputSchema") or graph.get("input_schema") or {"type": "object", "properties": {}},
"outputSchema": graph.get("outputSchema") or graph.get("output_schema") or {"type": "object", "properties": {}},
"updatedAt": definition.get("updated_at"),
}
@router.put("/{workflow_id}/studio-draft")
async def put_studio_draft(request: Request, workflow_id: str, body: StudioDraftBody) -> dict[str, Any]:
_require_enabled()
await _load_owned(request, workflow_id)
store = _get_store(request)
user_id = await get_current_user(request)
try:
WorkflowGraph.model_validate(body.execution_graph)
except Exception as exc: # noqa: BLE001 - surface pydantic as 400
raise HTTPException(
status_code=400,
detail={"code": "WORKFLOW_SCHEMA_INVALID", "message": f"执行图不符合 WorkflowGraph v1.0: {exc}"},
) from exc
canvas_text = _parse_canvas(body.canvas_schema_json)
try:
saved = await store.save_studio_draft(
workflow_id,
expected_revision=body.expected_revision,
graph=body.execution_graph,
canvas_schema=canvas_text,
updated_by=user_id,
)
except WorkflowDraftConflictError as exc:
raise HTTPException(
status_code=409,
detail={
"code": "WORKFLOW_DRAFT_CONFLICT",
"message": "草稿版本冲突,请刷新后重试或另存副本",
"currentRevision": exc.current_revision,
},
) from exc
except WorkflowNotFoundError as exc:
raise HTTPException(status_code=404, detail="工作流不存在") from exc
audit_workflow("workflow.studio_draft", user_id=user_id, workflowId=workflow_id, revision=saved.get("draft_revision"))
return {"revision": int(saved.get("draft_revision") or 0), "updatedAt": saved.get("updated_at")}
@router.post("/{workflow_id}/studio-validate")
async def studio_validate(request: Request, workflow_id: str, body: StudioValidateBody | None = None) -> dict[str, Any]:
_require_enabled()
definition = await _load_owned(request, workflow_id)
user_id = await get_current_user(request)
graph = (body.execution_graph if body and body.execution_graph is not None else definition.get("draft_graph")) or {}
try:
issues = await _validate_with_resources(request, workflow_id, graph, user_id)
except Exception as exc: # noqa: BLE001 - unparsable graph is a validation result, not a 500
raise HTTPException(
status_code=400,
detail={"code": "WORKFLOW_SCHEMA_INVALID", "message": f"图结构无效: {exc}"},
) from exc
valid = not issues
graph_hash = None
if valid:
try:
graph_hash = WorkflowGraph.model_validate(graph).graph_hash()
except Exception: # noqa: BLE001
valid = False
return {"valid": valid, "issues": _studio_issues(issues), "graphHash": graph_hash}
@router.post("/{workflow_id}/studio-publish")
async def studio_publish(request: Request, workflow_id: str, body: StudioPublishBody | None = None) -> dict[str, Any]:
_require_enabled()
definition = await _load_owned(request, workflow_id)
store = _get_store(request)
user_id = await get_current_user(request)
body = body or StudioPublishBody()
graph_raw = definition.get("draft_graph") or {}
if not graph_raw or not (graph_raw.get("nodes") or []):
raise HTTPException(
status_code=400,
detail={
"code": "WORKFLOW_SCHEMA_INVALID",
"message": "草稿没有可发布的执行图(只有画布文档不能发布)",
},
)
try:
issues = await _validate_with_resources(request, workflow_id, graph_raw, user_id)
except Exception as exc: # noqa: BLE001
raise HTTPException(
status_code=400,
detail={"code": "WORKFLOW_SCHEMA_INVALID", "message": f"图结构无效: {exc}"},
) from exc
if issues:
raise HTTPException(
status_code=400,
detail={
"code": "WORKFLOW_SCHEMA_INVALID",
"message": "发布校验未通过",
"issues": _studio_issues(issues),
},
)
if body.expected_revision is not None and body.expected_revision != definition.get("draft_revision"):
raise HTTPException(
status_code=409,
detail={
"code": "WORKFLOW_DRAFT_CONFLICT",
"message": "发布前草稿版本已变化",
"currentRevision": definition.get("draft_revision"),
},
)
graph = WorkflowGraph.model_validate(graph_raw)
canvas_text = str(definition.get("draft_canvas_schema") or "{}")
canvas_hash = sha256(canvas_text.encode("utf-8")).hexdigest()
version = await store.publish_version(
workflow_id,
graph=graph.model_dump(by_alias=True),
graph_hash=graph.graph_hash(),
published_by=user_id,
change_note=body.change_note,
canvas_schema_hash=canvas_hash,
)
audit_workflow(
"workflow.studio_publish",
user_id=user_id,
workflowId=workflow_id,
versionId=version.get("id"),
graphHash=graph.graph_hash(),
)
return {
"versionId": version.get("id"),
"versionNumber": version.get("version_number"),
"publishedAt": version.get("published_at"),
}
@@ -0,0 +1,313 @@
"""Workflow definition / draft / publish API (phase 1 — no execution yet)."""
from __future__ import annotations
from hashlib import sha256
from typing import Any
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field
from app.gateway.deps import get_current_user, get_optional_user_from_request
from app.gateway.workflow_audit import audit_workflow
from app.gateway.workflow_resource_checks import build_resource_catalog, make_resource_checker
from deerflow.config.app_config import get_app_config
from deerflow.persistence.workflows import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
)
from deerflow.workflows.schemas import WorkflowGraph
from deerflow.workflows.validator import validate_workflow_graph
router = APIRouter(prefix="/api/workflows", tags=["workflows"])
class WorkflowCreateRequest(BaseModel):
name: str = Field(default="未命名工作流", min_length=1, max_length=255)
description: str = ""
graph: dict[str, Any] | None = None
class WorkflowPatchRequest(BaseModel):
name: str | None = Field(default=None, min_length=1, max_length=255)
description: str | None = None
class DraftPutRequest(BaseModel):
expected_revision: int = Field(alias="expectedRevision", ge=0)
graph: dict[str, Any]
model_config = {"populate_by_name": True}
class PublishRequest(BaseModel):
change_note: str = Field(default="", alias="changeNote")
expected_revision: int | None = Field(default=None, alias="expectedRevision")
model_config = {"populate_by_name": True}
class ValidateRequest(BaseModel):
graph: dict[str, Any] | None = None
def _get_store(request: Request) -> WorkflowStore:
store = getattr(request.app.state, "workflow_store", None)
if store is None:
raise HTTPException(status_code=503, detail="Workflow store not available")
return store
def _require_enabled() -> None:
cfg = get_app_config().workflows
if not cfg.enabled:
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
async def _current_user_id(request: Request) -> str:
return await get_current_user(request)
def _can_access(definition: dict[str, Any], user_id: str, *, write: bool = False) -> bool:
# Phase 1: owner-only. Shared/org ACLs land later.
owner = definition.get("owner_id")
if owner == user_id:
return True
# Auth disabled environments often use "default" / anonymous — still owner-scoped.
if write:
return False
return False
async def _load_owned(request: Request, workflow_id: str, *, write: bool = False) -> dict[str, Any]:
store = _get_store(request)
user_id = await _current_user_id(request)
definition = await store.get_definition(workflow_id, include_draft=True)
if definition is None:
raise HTTPException(status_code=404, detail="工作流不存在")
if not _can_access(definition, user_id, write=write):
# Soft admin bypass for list/read when system_role=admin
user = await get_optional_user_from_request(request)
if user is not None and getattr(user, "system_role", None) == "admin" and not write:
return definition
if user is not None and getattr(user, "system_role", None) == "admin" and write:
return definition
raise HTTPException(status_code=403, detail="无权访问该工作流")
return definition
def _limits() -> dict[str, int]:
cfg = get_app_config().workflows
return {
"system_max_steps": cfg.max_steps,
"system_max_loop_iterations": cfg.max_loop_iterations,
"system_max_parallelism": cfg.max_parallelism,
}
@router.get("")
async def list_workflows(
request: Request,
include_archived: bool = Query(False, alias="includeArchived"),
status: str | None = None,
) -> dict[str, Any]:
_require_enabled()
store = _get_store(request)
user_id = await _current_user_id(request)
user = await get_optional_user_from_request(request)
is_admin = user is not None and getattr(user, "system_role", None) == "admin"
rows = await store.list_definitions(
owner_id=None if is_admin else user_id,
status=status,
include_archived=include_archived,
)
if is_admin:
# Still hide other users' drafts from non-explicit admin listing? Phase 1: admin sees all.
pass
else:
rows = [r for r in rows if r.get("owner_id") == user_id]
return {"workflows": rows}
@router.post("")
async def create_workflow(request: Request, body: WorkflowCreateRequest) -> dict[str, Any]:
_require_enabled()
store = _get_store(request)
user_id = await _current_user_id(request)
created = await store.create_definition(
{
"name": body.name,
"description": body.description,
"owner_id": user_id,
"draft_graph": body.graph,
}
)
audit_workflow("workflow.create", user_id=user_id, workflowId=created["id"])
return created
@router.get("/{workflow_id}")
async def get_workflow(request: Request, workflow_id: str) -> dict[str, Any]:
_require_enabled()
return await _load_owned(request, workflow_id, write=False)
@router.patch("/{workflow_id}")
async def patch_workflow(request: Request, workflow_id: str, body: WorkflowPatchRequest) -> dict[str, Any]:
_require_enabled()
await _load_owned(request, workflow_id, write=True)
store = _get_store(request)
updated = await store.update_definition(
workflow_id,
body.model_dump(exclude_unset=True),
)
if updated is None:
raise HTTPException(status_code=404, detail="工作流不存在")
return updated
@router.delete("/{workflow_id}")
async def archive_workflow(request: Request, workflow_id: str) -> dict[str, Any]:
_require_enabled()
await _load_owned(request, workflow_id, write=True)
store = _get_store(request)
archived = await store.archive_definition(workflow_id)
if archived is None:
raise HTTPException(status_code=404, detail="工作流不存在")
audit_workflow("workflow.archive", user_id=await _current_user_id(request), workflowId=workflow_id)
return archived
@router.post("/{workflow_id}/copy")
async def copy_workflow(request: Request, workflow_id: str) -> dict[str, Any]:
_require_enabled()
definition = await _load_owned(request, workflow_id, write=False)
store = _get_store(request)
user_id = await _current_user_id(request)
created = await store.create_definition(
{
"name": f"{definition.get('name') or '未命名工作流'} 副本",
"description": definition.get("description") or "",
"owner_id": user_id,
"draft_graph": definition.get("draft_graph"),
}
)
audit_workflow("workflow.copy", user_id=user_id, workflowId=created["id"], sourceWorkflowId=workflow_id)
return created
@router.put("/{workflow_id}/draft")
async def put_draft(request: Request, workflow_id: str, body: DraftPutRequest) -> dict[str, Any]:
_require_enabled()
await _load_owned(request, workflow_id, write=True)
store = _get_store(request)
user_id = await _current_user_id(request)
try:
# Light structural parse so obviously broken drafts fail early; full publish
# checks run on /validate and /publish.
WorkflowGraph.model_validate(body.graph)
except Exception as exc: # noqa: BLE001 — surface pydantic as 400
raise HTTPException(status_code=400, detail=f"图结构无效: {exc}") from exc
try:
return await store.save_draft(
workflow_id,
expected_revision=body.expected_revision,
graph=body.graph,
updated_by=user_id,
)
except WorkflowDraftConflictError as exc:
raise HTTPException(
status_code=409,
detail={
"code": "WORKFLOW_DRAFT_CONFLICT",
"message": "草稿版本冲突,请刷新后重试或另存副本",
"currentRevision": exc.current_revision,
},
) from exc
except WorkflowNotFoundError as exc:
raise HTTPException(status_code=404, detail="工作流不存在") from exc
@router.post("/{workflow_id}/validate")
async def validate_workflow(request: Request, workflow_id: str, body: ValidateRequest | None = None) -> dict[str, Any]:
_require_enabled()
definition = await _load_owned(request, workflow_id, write=False)
graph = (body.graph if body and body.graph is not None else definition.get("draft_graph")) or {}
issues = validate_workflow_graph(graph, **_limits())
return {
"ok": len(issues) == 0,
"issues": [i.model_dump(by_alias=True) for i in issues],
"graphHash": WorkflowGraph.model_validate(graph).graph_hash() if not issues else None,
}
@router.post("/{workflow_id}/publish")
async def publish_workflow(request: Request, workflow_id: str, body: PublishRequest | None = None) -> dict[str, Any]:
_require_enabled()
definition = await _load_owned(request, workflow_id, write=True)
body = body or PublishRequest()
if body.expected_revision is not None and body.expected_revision != definition.get("draft_revision"):
raise HTTPException(
status_code=409,
detail={
"code": "WORKFLOW_DRAFT_CONFLICT",
"message": "发布前草稿版本已变化",
"currentRevision": definition.get("draft_revision"),
},
)
graph_raw = definition.get("draft_graph") or {}
# Same resource/permission checks as the studio facade (§5.3): the standard
# publish path must not accept graphs referencing invisible agents or
# forbidden resources.
catalog = await build_resource_catalog(request, user_id=await _current_user_id(request))
checker = make_resource_checker(catalog, config=get_app_config().workflows, current_workflow_id=workflow_id)
issues = validate_workflow_graph(graph_raw, resource_checker=checker, **_limits())
if issues:
raise HTTPException(
status_code=400,
detail={
"code": "WORKFLOW_SCHEMA_INVALID",
"message": "发布校验未通过",
"issues": [i.model_dump(by_alias=True) for i in issues],
},
)
graph = WorkflowGraph.model_validate(graph_raw)
store = _get_store(request)
user_id = await _current_user_id(request)
canvas_text = str(definition.get("draft_canvas_schema") or "{}")
version = await store.publish_version(
workflow_id,
graph=graph.model_dump(by_alias=True),
graph_hash=graph.graph_hash(),
published_by=user_id,
change_note=body.change_note,
canvas_schema_hash=sha256(canvas_text.encode("utf-8")).hexdigest(),
)
audit_workflow(
"workflow.publish",
user_id=user_id,
workflowId=workflow_id,
versionId=version.get("id"),
graphHash=graph.graph_hash(),
)
return version
@router.get("/{workflow_id}/versions")
async def list_versions(request: Request, workflow_id: str) -> dict[str, Any]:
_require_enabled()
await _load_owned(request, workflow_id, write=False)
store = _get_store(request)
return {"versions": await store.list_versions(workflow_id)}
@router.get("/{workflow_id}/versions/{version_id}")
async def get_version(request: Request, workflow_id: str, version_id: str) -> dict[str, Any]:
_require_enabled()
await _load_owned(request, workflow_id, write=False)
store = _get_store(request)
version = await store.get_version(version_id)
if version is None or version.get("workflow_id") != workflow_id:
raise HTTPException(status_code=404, detail="版本不存在")
return version
@@ -0,0 +1,368 @@
"""Minimal Coze Studio workflow_api compatibility layer (phase 1).
Internal services always use DeerFlow draft DTOs. This module only translates
request/response shapes for the standalone canvas page: ``/canvas`` replies
with the Coze playground envelope and ``/save`` persists the Coze canvas into
``save_studio_draft`` without letting it anywhere near ``WorkflowGraph``
validation (the canvas is an opaque editing document; the executable graph is
compiled client-side and saved via the studio facade).
"""
from __future__ import annotations
import json
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from pydantic import BaseModel, Field
from app.gateway.deps import get_current_user, get_optional_user_from_request
from app.gateway.routers.workflow_studio import _parse_canvas
from deerflow.config.app_config import get_app_config
from deerflow.persistence.workflows import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
)
router = APIRouter(prefix="/api/workflow_api", tags=["workflow-coze-compat"])
# Playground 启动还会打这些 Coze 壳接口;缺了它们画布和节点资源都不会挂载。
chrome_router = APIRouter(tags=["workflow-coze-compat"])
_DEFAULT_SPACE: dict[str, Any] = {
"id": "1",
"space_id": 1,
"name": "默认空间",
"description": "",
# SpaceType.Personal = 1;Playground 找不到个人空间会去调 space/save。
"space_type": 1,
"space_mode": 0,
"icon_uri": "",
"icon_url": "",
"owner_id": "1",
"owner_user_id": "1",
"role_type": 1,
"has_permission": True,
}
# Coze playground node_template_list payload. The standalone canvas maps
# DeerFlow node kinds onto these StandardNodeType ids (see the frontend's
# node-catalog.ts); entries keep the Coze field names (node_type / icon_url /
# color) so the playground's template list renders without adaptation.
_COZE_NODE_TEMPLATES: list[dict[str, Any]] = [
{"node_type": "1", "name": "开始", "icon_url": "", "color": "#5C62FF", "desc": "工作流入口,定义输入参数"},
{"node_type": "2", "name": "结束", "icon_url": "", "color": "#5C62FF", "desc": "工作流出口,汇总输出"},
{"node_type": "3", "name": "大模型", "icon_url": "", "color": "#5C62FF", "desc": "调用智能体处理任务", "support_batch": 1},
{"node_type": "5", "name": "代码", "icon_url": "", "color": "#00B2B2", "desc": "在沙箱中执行代码", "support_batch": 0},
{"node_type": "8", "name": "选择器", "icon_url": "", "color": "#00B2B2", "desc": "按条件分支执行", "support_batch": 0},
{"node_type": "9", "name": "工作流", "icon_url": "", "color": "#00B83E", "desc": "调用已发布子工作流", "support_batch": 1},
{"node_type": "13", "name": "输出", "icon_url": "", "color": "#5C62FF", "desc": "定义输出字段", "support_batch": 0},
{"node_type": "15", "name": "文本处理", "icon_url": "", "color": "#3071F2", "desc": "模板/文本转换", "support_batch": 0},
{"node_type": "18", "name": "问答", "icon_url": "", "color": "#3071F2", "desc": "人工介入确认", "support_batch": 0},
{"node_type": "21", "name": "循环", "icon_url": "", "color": "#00B2B2", "desc": "循环执行子图", "support_batch": 0},
{"node_type": "32", "name": "变量聚合", "icon_url": "", "color": "#00B2B2", "desc": "合并多分支变量", "support_batch": 0},
{"node_type": "43", "name": "查询数据", "icon_url": "", "color": "#F2B600", "desc": "SQL 只读查询", "support_batch": 0},
{"node_type": "45", "name": "HTTP 请求", "icon_url": "", "color": "#3071F2", "desc": "调用外部接口", "support_batch": 0},
]
_COZE_NODE_CATEGORIES: list[dict[str, Any]] = [
{"name": "", "node_type_list": ["3", "9"]},
{"name": "业务逻辑", "node_type_list": ["5", "8", "21", "32"]},
{"name": "输入&输出", "node_type_list": ["1", "2", "13"]},
{"name": "组件", "node_type_list": ["15", "18", "45"]},
{"name": "数据库", "node_type_list": ["43"]},
]
# Mirrors the standalone page's built-in empty canvas (start → end), used when
# a workflow has never been opened in the Coze editor.
_DEFAULT_CANVAS_SCHEMA = json.dumps(
{
"nodes": [
{
"id": "100001",
"type": "1",
"meta": {"position": {"x": 0, "y": 0}},
"data": {"outputs": [{"type": "string", "name": "", "required": True}]},
},
{
"id": "900001",
"type": "2",
"meta": {"position": {"x": 1000, "y": 0}},
},
],
"edges": [{"sourceNodeID": "100001", "targetNodeID": "900001"}],
},
ensure_ascii=False,
)
class CozeCreateRequest(BaseModel):
name: str = Field(default="未命名工作流")
description: str = ""
graph: dict[str, Any] | None = None
class CozeCanvasRequest(BaseModel):
workflow_id: str | None = Field(default=None, alias="workflowId")
workflow_id_snake: str | None = Field(default=None, alias="workflow_id")
model_config = {"populate_by_name": True}
def resolved_id(self) -> str | None:
return self.workflow_id or self.workflow_id_snake
class CozeSaveRequest(BaseModel):
workflow_id: str | None = Field(default=None, alias="workflowId")
workflow_id_snake: str | None = Field(default=None, alias="workflow_id")
expected_revision: int | None = Field(default=None, alias="expectedRevision")
expected_revision_snake: int | None = Field(default=None, alias="expected_revision")
# Coze posts the canvas as a JSON string under ``schema``.
schema_: str | dict[str, Any] | None = Field(default=None, alias="schema")
graph: str | dict[str, Any] | None = None
data: dict[str, Any] | None = None
model_config = {"populate_by_name": True}
def resolved_id(self) -> str | None:
return self.workflow_id or self.workflow_id_snake
def resolved_revision(self) -> int | None:
if self.expected_revision is not None:
return self.expected_revision
return self.expected_revision_snake
def resolved_canvas(self) -> str | dict[str, Any] | None:
if self.schema_ is not None:
return self.schema_
if isinstance(self.graph, (str, dict)):
return self.graph
if self.data is not None:
for key in ("schema", "graph"):
value = self.data.get(key)
if isinstance(value, (str, dict)):
return value
return self.data
return None
def _require_enabled() -> None:
if not get_app_config().workflows.enabled:
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
def _get_store(request: Request) -> WorkflowStore:
store = getattr(request.app.state, "workflow_store", None)
if store is None:
raise HTTPException(status_code=503, detail="Workflow store not available")
return store
def _canvas_schema_text(definition: dict[str, Any]) -> str:
raw = definition.get("draft_canvas_schema")
if isinstance(raw, str) and raw.strip() not in ("", "{}", "null"):
return raw
if isinstance(raw, dict) and raw:
return json.dumps(raw, ensure_ascii=False)
return _DEFAULT_CANVAS_SCHEMA
def _to_coze_canvas(definition: dict[str, Any], *, is_published: bool = False) -> dict[str, Any]:
owner = str(definition.get("owner_id") or "1")
return {
"code": 0,
"msg": "success",
"data": {
"workflow": {
"workflow_id": definition["id"],
"space_id": "1",
"name": definition.get("name"),
"desc": definition.get("description") or "",
"schema_json": _canvas_schema_text(definition),
"status": 1,
"collaborator_mode": "Open",
"creator": {"user_id": owner, "nickname": owner},
"is_published": is_published,
},
"vcs_data": {"can_edit": True},
"db_data": {"status": 1},
"operation_info": {},
"is_bind_agent": False,
"workflow_version": "1",
},
}
async def _load_owned_definition(request: Request, store: WorkflowStore, workflow_id: str, user_id: str) -> dict[str, Any]:
definition = await store.get_definition(workflow_id, include_draft=True)
if definition is None:
raise HTTPException(status_code=404, detail="工作流不存在")
if definition.get("owner_id") == user_id:
return definition
user = await get_optional_user_from_request(request)
if user is not None and getattr(user, "system_role", None) == "admin":
return definition
raise HTTPException(status_code=403, detail="无权访问该工作流")
def _coze_ok(data: dict[str, Any]) -> dict[str, Any]:
return {"code": 0, "msg": "success", "data": data}
def _account_payload(user: Any, user_id: str) -> dict[str, Any]:
email = str(getattr(user, "email", "") or "")
nickname = email.split("@", 1)[0] if email else user_id
return {
"user_id_str": user_id,
"user_id": user_id,
"nickname": nickname,
"name": nickname,
"email": email,
"avatar_url": "",
"login_status": "logined",
}
async def _require_studio_user(request: Request) -> tuple[str, Any]:
_require_enabled()
user_id = await get_current_user(request)
if not user_id:
raise HTTPException(status_code=401, detail="未登录")
user = await get_optional_user_from_request(request)
return user_id, user
def _space_list_payload() -> dict[str, Any]:
return {
"has_personal_space": True,
"bot_space_list": [_DEFAULT_SPACE],
"team_space_num": 0,
"max_team_space_num": 10,
"recently_used_space_list": [],
}
@chrome_router.api_route("/api/passport/account/info/v2/", methods=["GET", "POST"])
async def coze_account_info(request: Request) -> dict[str, Any]:
user_id, user = await _require_studio_user(request)
return _coze_ok(_account_payload(user, user_id))
@chrome_router.api_route("/api/space/list", methods=["GET", "POST"])
async def coze_space_list(request: Request) -> dict[str, Any]:
await _require_studio_user(request)
payload = _space_list_payload()
# developer_api.SpaceList 把列表放在顶层;playground v2 读 data。
return {"code": 0, "msg": "success", "data": payload, **payload}
@chrome_router.api_route(
"/api/playground_api/space/list",
methods=["GET", "POST", "PUT", "PATCH"],
)
@chrome_router.api_route(
"/api/playground_api/space/list/",
methods=["GET", "POST", "PUT", "PATCH"],
)
async def coze_playground_space_list(request: Request) -> dict[str, Any]:
await _require_studio_user(request)
return _coze_ok(_space_list_payload())
@chrome_router.api_route(
"/api/playground_api/space/save",
methods=["GET", "POST"],
)
async def coze_playground_space_save(request: Request) -> dict[str, Any]:
await _require_studio_user(request)
return _coze_ok({"id": _DEFAULT_SPACE["id"], "check_not_pass": False})
@chrome_router.api_route("/api/bot/get_type_list", methods=["GET", "POST"])
async def coze_model_type_list(request: Request) -> dict[str, Any]:
await _require_studio_user(request)
return _coze_ok({"model_list": []})
@router.post("/create")
async def coze_create(request: Request, body: CozeCreateRequest) -> dict[str, Any]:
_require_enabled()
store = _get_store(request)
user_id = await get_current_user(request)
created = await store.create_definition(
{
"name": body.name,
"description": body.description,
"owner_id": user_id,
}
)
return {"code": 0, "msg": "success", "data": {"workflow_id": created["id"]}}
@router.post("/canvas")
async def coze_canvas(request: Request, body: CozeCanvasRequest) -> dict[str, Any]:
_require_enabled()
workflow_id = body.resolved_id()
if not workflow_id:
raise HTTPException(status_code=400, detail="workflow_id required")
store = _get_store(request)
user_id = await get_current_user(request)
definition = await _load_owned_definition(request, store, workflow_id, user_id)
versions = await store.list_versions(workflow_id)
return _to_coze_canvas(definition, is_published=bool(versions))
@router.post("/save")
async def coze_save(request: Request, body: CozeSaveRequest) -> dict[str, Any]:
_require_enabled()
workflow_id = body.resolved_id()
if not workflow_id:
raise HTTPException(status_code=400, detail="workflow_id required")
canvas_raw = body.resolved_canvas()
if canvas_raw is None or canvas_raw == "" or canvas_raw == {}:
raise HTTPException(status_code=400, detail="schema required")
canvas_text = _parse_canvas(canvas_raw)
store = _get_store(request)
user_id = await get_current_user(request)
definition = await _load_owned_definition(request, store, workflow_id, user_id)
expected = body.resolved_revision()
if expected is None:
expected = int(definition.get("draft_revision") or 0)
try:
await store.save_studio_draft(
workflow_id,
expected_revision=expected,
# The execution graph is owned by the studio facade; the Coze save
# only persists the canvas, so the current graph passes through.
graph=definition.get("draft_graph") or {},
canvas_schema=canvas_text,
updated_by=user_id,
)
except WorkflowDraftConflictError as exc:
raise HTTPException(
status_code=409,
detail={
"code": "WORKFLOW_DRAFT_CONFLICT",
"currentRevision": exc.current_revision,
},
) from exc
except WorkflowNotFoundError as exc:
raise HTTPException(status_code=404, detail="工作流不存在") from exc
return {"code": 0, "msg": "success", "data": {}}
@router.post("/node_template_list")
async def coze_node_template_list(request: Request) -> dict[str, Any]:
_require_enabled()
await get_current_user(request)
# Coze playground envelope: data.template_list / cate_list (§ workflow-playground-context).
return {
"code": 0,
"msg": "success",
"data": {
"template_list": _COZE_NODE_TEMPLATES,
"cate_list": _COZE_NODE_CATEGORIES,
"plugin_api_list": [],
"plugin_category_list": [],
},
}
@@ -0,0 +1,209 @@
"""Runs a DeerFlow agent for one workflow ``agent`` / ``skill`` node.
This is the single place where the workflow runtime touches LangGraph. We build
the lead agent with :func:`make_lead_agent` (so the node inherits the same
middleware chain, SOUL, skills and model resolution as a normal chat turn) and
stream ``stream_mode=["values", "messages"]`` so the node can forward token
deltas into the workflow event log.
Each node gets its own thread id (``wf-{run_id}-{node_id}``) unless the node
asks for ``shared_run_thread``, which keeps the whole run in one conversation.
"""
from __future__ import annotations
import logging
from collections.abc import Awaitable, Callable
from typing import Any
from langchain_core.messages import AIMessage, ToolMessage
from deerflow.workflows.errors import WorkflowError
logger = logging.getLogger(__name__)
def _extract_text(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict) and block.get("type") == "text":
parts.append(str(block.get("text") or ""))
return "".join(parts)
return ""
def _thread_id(run_id: str, node_id: str, thread_mode: str) -> str:
if thread_mode == "shared_run_thread":
return f"wf-{run_id}"
return f"wf-{run_id}-{node_id}"
class WorkflowAgentRunner:
def __init__(self, app: Any) -> None:
self._app = app
async def run_agent(
self,
*,
run_id: str,
node_id: str,
owner_id: str,
agent_id: str,
prompt: str,
skill_names: list[str] | None = None,
thread_mode: str = "isolated_per_node",
cancel: Any = None,
on_delta: Callable[[str], Awaitable[None]] | None = None,
on_tool: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
model_name: str | None = None,
) -> dict[str, Any]:
from deerflow.agents.lead_agent.agent import make_lead_agent
thread_id = _thread_id(run_id, node_id, thread_mode)
configurable: dict[str, Any] = {
"thread_id": thread_id,
"agent_id": agent_id,
"agent_name": agent_id,
"user_id": owner_id,
"model_name": model_name or "",
# Workflow nodes are non-interactive: no plan mode, no clarification.
"is_plan_mode": False,
"workflow_run_id": run_id,
"workflow_node_id": node_id,
}
if skill_names:
configurable["skill_names"] = list(skill_names)
config = {"configurable": configurable, "recursion_limit": 60}
try:
agent = make_lead_agent(config)
except Exception as exc: # noqa: BLE001
logger.warning("failed to build agent %s for workflow node %s", agent_id, node_id, exc_info=True)
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
"引用的智能体不存在或配置无效",
node_id=node_id,
details={"agentId": agent_id},
) from exc
checkpointer = getattr(self._app.state, "checkpointer", None)
if checkpointer is not None:
agent.checkpointer = checkpointer
store = getattr(self._app.state, "store", None)
if store is not None:
agent.store = store
seen_tool_calls: set[str] = set()
finished_tool_calls: set[str] = set()
tool_names: dict[str, str] = {}
final_state: dict[str, Any] | None = None
streamed: dict[str, list[str]] = {}
last_message_id = ""
async for mode, chunk in agent.astream(
{"messages": [{"role": "user", "content": prompt}]},
config=config,
stream_mode=["values", "messages"],
):
if cancel is not None and getattr(cancel, "cancelled", False):
raise WorkflowError("WORKFLOW_CANCELLED", "运行已被取消", node_id=node_id)
if mode == "values":
if isinstance(chunk, dict):
final_state = chunk
continue
if mode != "messages":
continue
message = chunk[0] if isinstance(chunk, tuple) and chunk else chunk
if isinstance(message, ToolMessage):
# Tool result for an earlier call: close its step in the event log.
call_id = str(getattr(message, "tool_call_id", "") or "")
if call_id and call_id not in finished_tool_calls:
finished_tool_calls.add(call_id)
if on_tool is not None:
content = getattr(message, "content", "")
if not isinstance(content, str):
content = str(content)
await on_tool(
{
"phase": "finished",
"id": call_id,
"name": tool_names.get(call_id) or getattr(message, "name", "") or "",
"ok": getattr(message, "status", None) != "error",
"summary": content[:200],
}
)
continue
if not isinstance(message, AIMessage):
continue
text = _extract_text(getattr(message, "content", ""))
if text:
message_id = str(getattr(message, "id", "") or "")
streamed.setdefault(message_id, []).append(text)
last_message_id = message_id
if on_delta is not None:
await on_delta(text)
for call in getattr(message, "tool_calls", None) or []:
call_id = str(call.get("id") or "")
if not call_id or call_id in seen_tool_calls:
continue
seen_tool_calls.add(call_id)
tool_names[call_id] = str(call.get("name") or "")
if on_tool is not None:
await on_tool(
{
"phase": "started",
"id": call_id,
"name": tool_names[call_id],
"args": call.get("args"),
}
)
final_text = "".join(streamed.get(last_message_id, ())) or self._final_text(final_state)
if not final_text.strip():
raise WorkflowError(
"WORKFLOW_AGENT_FAILED",
"智能体没有返回任何内容",
retryable=True,
node_id=node_id,
)
return {
"text": final_text,
"thread_id": thread_id,
"messages": [],
"artifacts": self._artifacts(final_state),
"usage": {},
}
def _final_text(self, state: dict[str, Any] | None) -> str:
for message in reversed((state or {}).get("messages") or []):
if isinstance(message, AIMessage):
text = _extract_text(getattr(message, "content", "")).strip()
if text:
return text
return ""
def _artifacts(self, state: dict[str, Any] | None) -> list[dict[str, Any]]:
raw = (state or {}).get("artifacts")
if not isinstance(raw, list):
return []
out: list[dict[str, Any]] = []
for item in raw:
if isinstance(item, dict) and item.get("name"):
out.append(
{
"name": str(item.get("name")),
"path": str(item.get("path") or ""),
"mime_type": str(item.get("mime_type") or item.get("mimeType") or ""),
"size_bytes": item.get("size_bytes") or item.get("sizeBytes"),
"preview": item.get("preview"),
}
)
return out
__all__ = ["WorkflowAgentRunner"]
@@ -0,0 +1,22 @@
"""Structured audit lines for Workflow Studio mutating APIs.
These are operator logs, not a separate audit table: every line is a single
JSON object with a stable ``action`` plus the ids needed to reconstruct who
did what. Secrets never go in.
"""
from __future__ import annotations
import json
import logging
from typing import Any
logger = logging.getLogger("deerflow.workflows.audit")
def audit_workflow(action: str, *, user_id: str, **fields: Any) -> None:
payload = {"action": action, "userId": user_id, **fields}
logger.info("workflow.audit %s", json.dumps(payload, ensure_ascii=False, default=str))
__all__ = ["audit_workflow"]
@@ -0,0 +1,151 @@
"""Lease-based dispatcher for workflow runs.
A single background loop scans for claimable runs (``queued``, or ``running``
with an expired lease) and hands each successfully claimed run to the executor.
Because the claim is an atomic conditional UPDATE, several workers can run this
loop concurrently and a run that dies with its worker is picked up by whoever
notices the expired lease first — that is the restart-recovery story.
``nudge()`` is called by the start/resume routes so a fresh run does not wait
for the next poll tick.
"""
from __future__ import annotations
import asyncio
import logging
import os
import socket
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from app.gateway.workflow_executor import LeaseParams, WorkflowRunExecutor
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
from deerflow.workflows.runtime import PersistingWorkflowEventSink
logger = logging.getLogger(__name__)
_POLL_INTERVAL_SECONDS = 2.0
_CLAIM_BATCH = 5
def _worker_id() -> str:
return f"{socket.gethostname()[:24]}-{os.getpid()}-{uuid.uuid4().hex[:6]}"
class WorkflowRunDispatcher:
def __init__(
self,
store: WorkflowRunStore,
executor: WorkflowRunExecutor,
*,
event_store: WorkflowEventStore | None = None,
live_publisher: Any = None,
config: WorkflowConfig | None = None,
) -> None:
self._store = store
self._executor = executor
self._events = event_store
self._publish = live_publisher
self._config = config or WorkflowConfig()
self._worker_id = _worker_id()
self._wake = asyncio.Event()
self._task: asyncio.Task | None = None
self._stopping = False
@property
def worker_id(self) -> str:
return self._worker_id
def start(self) -> None:
if self._task is not None and not self._task.done():
return
self._stopping = False
self._task = asyncio.create_task(self._loop(), name="workflow-run-dispatcher")
logger.info("workflow dispatcher started (worker=%s)", self._worker_id)
async def stop(self) -> None:
self._stopping = True
self._wake.set()
if self._task is not None:
self._task.cancel()
try:
await self._task
except (asyncio.CancelledError, Exception): # noqa: BLE001
pass
self._task = None
await self._executor.drain()
def nudge(self) -> None:
self._wake.set()
async def _loop(self) -> None:
await self._safe_dispatch()
while not self._stopping:
try:
await asyncio.wait_for(self._wake.wait(), timeout=_POLL_INTERVAL_SECONDS)
except TimeoutError:
pass
self._wake.clear()
if self._stopping:
return
await self._safe_dispatch()
async def _safe_dispatch(self) -> None:
try:
await self.dispatch_once()
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001 - the loop must survive a bad round
logger.exception("workflow dispatch round failed")
async def dispatch_once(self) -> int:
now = datetime.now(UTC)
await self._reap_cancel_requested()
candidates = await self._store.list_claimable(now=now, limit=_CLAIM_BATCH)
if not candidates:
return 0
ttl = max(30, self._config.lease_ttl_seconds)
started = 0
for run_id in candidates:
row = await self._store.claim_run(
run_id,
lease_owner=self._worker_id,
lease_until=now + timedelta(seconds=ttl),
)
if row is None:
continue
if self._executor.start(LeaseParams(run_id=run_id, lease_owner=self._worker_id, attempt=int(row.get("attempt") or 1))):
started += 1
return started
async def _reap_cancel_requested(self) -> None:
"""Finalize cancels whose worker is gone (no live lease to notice them)."""
rows = await self._store.list_runs(status="cancel_requested", limit=20)
now = datetime.now(UTC)
for row in rows:
lease_until = row.get("lease_until")
if lease_until:
try:
if datetime.fromisoformat(str(lease_until)) > now:
continue # its worker is alive and will handle the cancel
except ValueError:
pass
finalized = await self._store.finalize_cancel(row["id"])
if finalized is not None and self._events is not None:
# Nobody owns this run, so the dispatcher owes the stream its
# terminal frame — otherwise open SSE connections hang.
sink = PersistingWorkflowEventSink(
self._events,
run_id=row["id"],
workflow_id=row["workflow_id"],
version_id=row["workflow_version_id"],
publish=self._publish,
)
await sink.emit("run.cancelled", data={"reapedBy": self._worker_id})
__all__ = ["WorkflowRunDispatcher"]
@@ -0,0 +1,444 @@
"""Executes leased workflow runs and owns every terminal state transition.
Lifecycle for one lease:
1. resolve the published version → immutable graph snapshot,
2. replay completed node results from ``workflow_node_runs`` (so a re-lease or a
resume never re-runs finished work),
3. drive :class:`WorkflowEngine` under a run-level timeout with a heartbeat
renewing the lease and a watcher polling for ``cancel_requested``,
4. write exactly one terminal outcome: ``completed`` / ``failed`` /
``cancelled`` / ``awaiting_input``.
Every exit path emits a matching terminal event, because the SSE endpoint uses
those events to decide when to close a stream.
"""
from __future__ import annotations
import asyncio
import json
import logging
import secrets
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from typing import Any
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.nodes import build_default_registry
from deerflow.workflows.runtime import (
CancelToken,
PersistingWorkflowEventSink,
RunContext,
WorkflowEngine,
WorkflowPaused,
WorkflowRuntimeDeps,
)
from deerflow.workflows.schemas import WorkflowGraph
logger = logging.getLogger(__name__)
_HEARTBEAT_INTERVAL_SECONDS = 15
_CANCEL_POLL_SECONDS = 2.0
_MAX_SUBWORKFLOW_DEPTH = 3
@dataclass
class LeaseParams:
run_id: str
lease_owner: str
attempt: int = 1
class WorkflowRunExecutor:
def __init__(
self,
app: Any,
*,
run_store: WorkflowRunStore,
event_store: WorkflowEventStore,
workflow_store: Any,
data_source_store: Any = None,
live_publisher: Any = None,
config: WorkflowConfig | None = None,
) -> None:
self._app = app
self._runs = run_store
self._events = event_store
self._workflows = workflow_store
self._data_sources = data_source_store
self._publish = live_publisher
self._config = config or WorkflowConfig()
self._registry = build_default_registry(self._config)
self._tasks: dict[str, asyncio.Task] = {}
# ── public API ──────────────────────────────────────────────────────
def start(self, params: LeaseParams) -> bool:
"""Launch a background task for a leased run. Idempotent per run id."""
existing = self._tasks.get(params.run_id)
if existing is not None and not existing.done():
return False
task = asyncio.create_task(self._guarded(params), name=f"workflow-run-{params.run_id}")
self._tasks[params.run_id] = task
task.add_done_callback(lambda _t, rid=params.run_id: self._tasks.pop(rid, None))
return True
async def drain(self, timeout: float = 5.0) -> None:
tasks = [t for t in self._tasks.values() if not t.done()]
if not tasks:
return
for task in tasks:
task.cancel()
await asyncio.wait(tasks, timeout=timeout)
# ── run driver ──────────────────────────────────────────────────────
async def _guarded(self, params: LeaseParams) -> None:
try:
await self._execute_lease(params)
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001 - never let a run kill the dispatcher
logger.exception("workflow run %s crashed outside the engine", params.run_id)
await self._fail(
params,
WorkflowError("WORKFLOW_INTERNAL", "运行发生内部错误,请查看服务端日志"),
)
async def _execute_lease(self, params: LeaseParams) -> None:
run = await self._runs.get_run(params.run_id)
if run is None:
return
sink = self._sink(run)
graph = await self._load_graph(run)
if graph is None:
await self._fail(
params,
WorkflowError("WORKFLOW_VERSION_NOT_FOUND", "运行引用的工作流版本不存在"),
sink=sink,
)
return
cancel = CancelToken()
resume_payload = (run.get("context") or {}).get("resume_payload")
preloaded, loop_state = await self._replay_state(params.run_id)
ctx = RunContext(
run_id=run["id"],
workflow_id=run["workflow_id"],
version_id=run["workflow_version_id"],
owner_id=run["owner_id"],
graph=graph,
inputs=run.get("input") or {},
deps=self._deps(depth=int((run.get("context") or {}).get("depth") or 0)),
cancel=cancel,
emit=sink.emit,
resume_payload=resume_payload if isinstance(resume_payload, dict) else None,
)
await sink.emit(
"run.started" if not preloaded else "run.resumed",
data={"attempt": params.attempt, "replayedNodes": sorted(preloaded)},
)
engine = WorkflowEngine(self._registry, run_store=self._runs)
main = asyncio.create_task(
asyncio.wait_for(
engine.run(ctx, preloaded=preloaded, loop_state=loop_state, attempt=params.attempt),
timeout=min(graph.settings.run_timeout_seconds, self._config.run_timeout_seconds),
)
)
heartbeat = asyncio.create_task(self._heartbeat(params, main))
watcher = asyncio.create_task(self._watch_cancel(params, cancel, main))
try:
result = await main
except WorkflowPaused as paused:
await self._pause(params, paused, ctx, sink)
return
except TimeoutError:
await self._fail(
params,
WorkflowError(
"WORKFLOW_TIMEOUT",
f"运行超时({graph.settings.run_timeout_seconds}s)",
details={"timeoutSeconds": graph.settings.run_timeout_seconds},
),
sink=sink,
)
return
except WorkflowError as exc:
if exc.code == "WORKFLOW_CANCELLED" or cancel.cancelled:
await self._cancel(params, sink)
else:
await self._fail(params, exc, sink=sink)
return
except asyncio.CancelledError:
# Lease lost or process shutting down: leave the run claimable again.
await self._runs.update_run(params.run_id, lease_owner=params.lease_owner, status="queued")
raise
finally:
heartbeat.cancel()
watcher.cancel()
for task in (heartbeat, watcher):
try:
await task
except (asyncio.CancelledError, Exception): # noqa: BLE001
pass
await self._complete(params, result, sink)
# ── terminal transitions ────────────────────────────────────────────
async def _complete(self, params: LeaseParams, result: Any, sink: Any) -> None:
await self._runs.update_run(
params.run_id,
lease_owner=params.lease_owner,
status="completed",
output_json=result.output,
context_json={"loop_state": result.loop_state, "steps": result.steps},
finished_at=datetime.now(UTC),
)
await sink.emit("run.completed", data={"output": result.output, "steps": result.steps})
async def _fail(self, params: LeaseParams, error: WorkflowError, *, sink: Any = None) -> None:
run = await self._runs.get_run(params.run_id)
if run is None:
return
body = error.to_body().model_dump(by_alias=True)
retry = error.retryable and int(run.get("attempt") or 1) < int(run.get("max_attempts") or 1)
sink = sink or self._sink(run)
if retry:
await self._runs.update_run(
params.run_id,
lease_owner=params.lease_owner,
status="queued",
error_json=body,
)
await sink.emit("run.queued", data={"retryAfterError": body, "attempt": run.get("attempt")})
return
await self._runs.update_run(
params.run_id,
lease_owner=params.lease_owner,
status="failed",
error_json=body,
finished_at=datetime.now(UTC),
)
await sink.emit("run.failed", data={"error": body})
async def _cancel(self, params: LeaseParams, sink: Any) -> None:
await self._runs.request_cancel(params.run_id)
finalized = await self._runs.finalize_cancel(params.run_id)
if finalized is None:
await self._runs.update_run(params.run_id, lease_owner=params.lease_owner, status="cancelled")
await sink.emit("run.cancelled", data={})
async def _pause(self, params: LeaseParams, paused: WorkflowPaused, ctx: RunContext, sink: Any) -> None:
token = secrets.token_urlsafe(24)
# One conditional update writes status + token + pending descriptor +
# run context together; a separate follow-up context write could clobber
# a resume that lands in between.
updated = await self._runs.set_awaiting_input(
params.run_id,
lease_owner=params.lease_owner,
pending_input=paused.pending_input,
resume_token=token,
context={"loop_state": dict(ctx.loop_state), "resume_payload": None},
)
if updated is None:
# Lost the lease (cancel or steal) — do not fabricate a pause.
logger.info("workflow run %s could not enter awaiting_input", params.run_id)
return
await sink.emit(
"run.awaiting_input",
data={
# Flattened wire contract: the full descriptor keeps living in
# the run row (pending_input), the event exposes exactly what a
# client needs to render the intervention card.
"resumeToken": token,
"prompt": paused.pending_input.get("prompt") or "",
"formSchema": paused.pending_input.get("formSchema") or {},
"actions": paused.pending_input.get("actions") or ["submit"],
},
node_id=paused.node_id,
)
# ── helpers ─────────────────────────────────────────────────────────
def _sink(self, run: dict[str, Any]) -> PersistingWorkflowEventSink:
return PersistingWorkflowEventSink(
self._events,
run_id=run["id"],
workflow_id=run["workflow_id"],
version_id=run["workflow_version_id"],
publish=self._publish,
)
async def _load_graph(self, run: dict[str, Any]) -> WorkflowGraph | None:
version = await self._workflows.get_version(run["workflow_version_id"])
if version is None or version.get("workflow_id") != run["workflow_id"]:
return None
try:
return WorkflowGraph.model_validate(version.get("graph") or {})
except Exception: # noqa: BLE001
logger.warning("workflow version %s has an invalid graph", run["workflow_version_id"], exc_info=True)
return None
async def _replay_state(self, run_id: str) -> tuple[dict[str, Any], dict[str, int]]:
run = await self._runs.get_run(run_id)
loop_state = {}
raw_loop = (run or {}).get("context", {}).get("loop_state")
if isinstance(raw_loop, dict):
loop_state = {str(k): int(v) for k, v in raw_loop.items() if isinstance(v, int)}
completed: dict[str, Any] = {}
for row in await self._runs.list_node_runs(run_id, current_only=True):
if row.get("status") != "completed":
continue
output = row.get("output")
if isinstance(output, dict):
completed[str(row["node_id"])] = output
return completed, loop_state
def _deps(self, *, depth: int = 0) -> WorkflowRuntimeDeps:
from app.gateway.workflow_agent_runner import WorkflowAgentRunner
agent_runner = WorkflowAgentRunner(self._app)
async def save_artifact(**kwargs: Any) -> dict[str, Any]:
return await self._runs.add_artifact(kwargs)
async def resolve_data_source(source_id: str) -> dict[str, Any] | None:
if self._data_sources is None:
return None
return await self._data_sources.resolve_dsn(source_id)
async def resolve_credential(ref: str) -> dict[str, Any] | None:
"""HTTP credentials are data-source rows of kind ``http`` whose
secret is a JSON blob of headers to inject."""
if self._data_sources is None:
return None
resolved = await self._data_sources.resolve_dsn(ref)
if not resolved:
return None
try:
payload = json.loads(resolved.get("dsn") or "{}")
except json.JSONDecodeError:
return None
headers = payload.get("headers") if isinstance(payload, dict) else None
return {"headers": headers} if isinstance(headers, dict) else None
async def run_subworkflow(**kwargs: Any) -> dict[str, Any]:
if depth + 1 > _MAX_SUBWORKFLOW_DEPTH:
raise WorkflowError(
"WORKFLOW_LIMIT_EXCEEDED",
f"子工作流嵌套超过 {_MAX_SUBWORKFLOW_DEPTH} 层",
details={"depth": depth + 1},
)
return await self._run_child(depth=depth + 1, **kwargs)
return WorkflowRuntimeDeps(
run_agent=agent_runner.run_agent,
save_artifact=save_artifact,
resolve_data_source=resolve_data_source,
resolve_credential=resolve_credential,
run_subworkflow=run_subworkflow,
)
async def _run_child(
self,
*,
depth: int,
parent_run_id: str,
node_id: str,
owner_id: str,
workflow_id: str,
version_id: str,
inputs: dict[str, Any],
cancel: CancelToken | None = None,
) -> dict[str, Any]:
"""Run a child workflow inline: same process, its own run row and events."""
child, _ = await self._runs.create_run(
{
"workflow_id": workflow_id,
"workflow_version_id": version_id,
"owner_id": owner_id,
"input": inputs,
"status": "running",
"context": {
"depth": depth,
"parent_run_id": parent_run_id,
"parent_node_id": node_id,
},
}
)
graph = await self._load_graph(child)
if graph is None:
raise WorkflowError("WORKFLOW_VERSION_NOT_FOUND", "子工作流版本不存在", node_id=node_id)
sink = self._sink(child)
ctx = RunContext(
run_id=child["id"],
workflow_id=workflow_id,
version_id=version_id,
owner_id=owner_id,
graph=graph,
inputs=inputs,
deps=self._deps(depth=depth),
cancel=cancel or CancelToken(),
emit=sink.emit,
)
await sink.emit("run.started", data={"parentRunId": parent_run_id, "depth": depth})
engine = WorkflowEngine(self._registry, run_store=self._runs)
try:
result = await engine.run(ctx)
except WorkflowPaused as exc:
await self._runs.update_run(child["id"], status="failed")
await sink.emit(
"run.failed",
data={
"error": {
"code": "WORKFLOW_SUBWORKFLOW_FAILED",
"message": "子工作流不支持人工节点",
}
},
)
raise WorkflowError(
"WORKFLOW_SUBWORKFLOW_FAILED",
"子工作流不支持人工输入节点",
node_id=node_id,
details={"childRunId": child["id"], "pausedAt": exc.node_id},
) from exc
except WorkflowError as exc:
await self._runs.update_run(child["id"], status="failed", error_json=exc.to_body().model_dump(by_alias=True))
await sink.emit("run.failed", data={"error": exc.to_body().model_dump(by_alias=True)})
raise
await self._runs.update_run(child["id"], status="completed", output_json=result.output)
await sink.emit("run.completed", data={"output": result.output})
return {"run_id": child["id"], "status": "completed", "output": result.output}
async def _heartbeat(self, params: LeaseParams, main: asyncio.Task) -> None:
ttl = max(30, self._config.lease_ttl_seconds)
while not main.done():
await asyncio.sleep(_HEARTBEAT_INTERVAL_SECONDS)
if main.done():
return
ok = await self._runs.renew_lease(
params.run_id,
lease_owner=params.lease_owner,
lease_until=datetime.now(UTC) + timedelta(seconds=ttl),
)
if not ok:
logger.warning("workflow run %s lost its lease; aborting", params.run_id)
main.cancel()
return
async def _watch_cancel(self, params: LeaseParams, cancel: CancelToken, main: asyncio.Task) -> None:
while not main.done():
await asyncio.sleep(_CANCEL_POLL_SECONDS)
run = await self._runs.get_run(params.run_id)
if run is None:
return
if run.get("status") in ("cancel_requested", "cancelled"):
cancel.cancel()
return
__all__ = ["LeaseParams", "WorkflowRunExecutor"]
@@ -0,0 +1,59 @@
"""In-process fan-out of workflow events to connected SSE clients.
The durable log in ``workflow_run_events`` is the source of truth; this hub only
shortens latency for the tail. Slow consumers lose the *oldest* queued frame
rather than blocking the executor, and the SSE endpoint reconciles any gap from
the database using the ``seq`` cursor.
"""
from __future__ import annotations
import asyncio
import logging
from collections import defaultdict
from collections.abc import Mapping
from typing import Any
logger = logging.getLogger(__name__)
class WorkflowLiveHub:
def __init__(self, *, queue_size: int = 512) -> None:
self._subscribers: dict[str, set[asyncio.Queue[dict[str, Any]]]] = defaultdict(set)
self._queue_size = queue_size
def subscribe(self, run_id: str) -> asyncio.Queue[dict[str, Any]]:
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=self._queue_size)
self._subscribers[run_id].add(queue)
return queue
def unsubscribe(self, run_id: str, queue: asyncio.Queue[dict[str, Any]]) -> None:
subscribers = self._subscribers.get(run_id)
if not subscribers:
return
subscribers.discard(queue)
if not subscribers:
self._subscribers.pop(run_id, None)
def subscriber_count(self, run_id: str) -> int:
return len(self._subscribers.get(run_id) or ())
async def publish(self, event: Mapping[str, Any]) -> None:
run_id = str(event.get("runId") or "")
subscribers = self._subscribers.get(run_id)
if not run_id or not subscribers:
return
payload = dict(event)
for queue in list(subscribers):
if queue.full():
try:
queue.get_nowait()
except asyncio.QueueEmpty: # pragma: no cover - race with a reader
pass
try:
queue.put_nowait(payload)
except asyncio.QueueFull: # pragma: no cover - race with a reader
logger.debug("workflow live queue still full for run %s", run_id)
__all__ = ["WorkflowLiveHub"]
@@ -0,0 +1,303 @@
"""App-layer publish-time resource checks for workflow graphs.
The harness validator stays store-agnostic via its ``resource_checker`` hook;
this module is where the Gateway binds that hook to the real agent / skill /
data-source / published-version stores. It pre-fetches the catalogs the caller
is allowed to see, then exposes a **sync** checker so it can be passed straight
into ``validate_workflow_graph``.
Never imports from ``packages/harness`` internals beyond the public workflow
API, and the harness never imports this module.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
from fastapi import Request
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.security.sql_policy import assert_read_only, assert_tables_allowed
# Header-ish keys that must never be configured as literal values on the canvas:
# credentials belong in an http data source referenced by ``credentialRef``.
_LITERAL_SECRET_KEYS = frozenset(
{
"authorization",
"proxy-authorization",
"cookie",
"x-api-key",
"apikey",
"api_key",
"secret",
"client_secret",
"password",
"access_token",
"refresh_token",
}
)
_HTTP_METHODS = frozenset({"GET", "POST", "PUT", "PATCH", "DELETE"})
@dataclass
class ResourceCatalog:
"""Everything the checker is allowed to reference, prefetched."""
agent_ids: set[str] = field(default_factory=set)
# agent_id -> cached skill whitelist from the agent store (empty = no
# restriction recorded).
agent_skills: dict[str, set[str]] = field(default_factory=dict)
enabled_skills: set[str] = field(default_factory=set)
# data source id -> public row (kind / enabled / allowed_* / max_rows).
data_sources: dict[str, dict[str, Any]] = field(default_factory=dict)
# workflow_id -> set of published version ids.
published_subworkflows: dict[str, set[str]] = field(default_factory=dict)
def _issue(message: str, field_path: str) -> dict[str, Any]:
return {"message": message, "field": field_path}
async def build_resource_catalog(request: Request, *, user_id: str) -> ResourceCatalog:
catalog = ResourceCatalog()
agent_store = getattr(request.app.state, "agent_store", None)
if agent_store is not None:
try:
if hasattr(agent_store, "list_visible"):
rows = await agent_store.list_visible(user_id)
elif hasattr(agent_store, "list_all"):
rows = await agent_store.list_all()
else:
rows = []
catalog.agent_ids = {
str(row.get("id") or row.get("agent_id") or "") for row in rows or [] if row.get("id") or row.get("agent_id")
}
if hasattr(agent_store, "get_extras_for"):
extras = await agent_store.get_extras_for([aid for aid in catalog.agent_ids if aid])
for agent_id, extra in (extras or {}).items():
skills = (extra or {}).get("skills") or []
if skills:
catalog.agent_skills[str(agent_id)] = {str(s) for s in skills}
except Exception: # noqa: BLE001 - publish checks degrade to strict-by-empty
catalog.agent_ids = set()
try:
from deerflow.skills import get_or_new_skill_storage
catalog.enabled_skills = {
str(getattr(skill, "name"))
for skill in get_or_new_skill_storage().load_skills(enabled_only=True) or []
if getattr(skill, "name", None)
}
except Exception: # noqa: BLE001
catalog.enabled_skills = set()
ds_store = getattr(request.app.state, "workflow_data_source_store", None)
if ds_store is not None:
try:
for row in await ds_store.list_sources(owner_id=user_id, include_shared=True) or []:
catalog.data_sources[str(row.get("id"))] = row
except Exception: # noqa: BLE001
catalog.data_sources = {}
wf_store = getattr(request.app.state, "workflow_store", None)
if wf_store is not None:
try:
for row in await wf_store.list_published_summaries() or []:
versions = catalog.published_subworkflows.setdefault(str(row.get("workflow_id")), set())
versions.add(str(row.get("version_id")))
except Exception: # noqa: BLE001
catalog.published_subworkflows = {}
return catalog
def _check_agent(config: dict[str, Any], catalog: ResourceCatalog) -> dict[str, Any] | None:
agent_id = str(config.get("agentId") or config.get("agent_id") or "")
if not agent_id:
return _issue("未选择智能体", "config.agentId")
if agent_id not in catalog.agent_ids:
return _issue("未选择可访问的智能体", "config.agentId")
if not str(config.get("promptTemplate") or config.get("prompt_template") or "").strip():
return _issue("智能体节点的提示词不能为空", "config.promptTemplate")
mode = str(config.get("responseMode") or config.get("response_mode") or "text")
schema = config.get("responseSchema") or config.get("response_schema")
if mode == "json":
if not isinstance(schema, dict) or not schema:
return _issue("JSON 输出模式必须配置 responseSchema 对象", "config.responseSchema")
if schema.get("type") not in (None, "object"):
return _issue("responseSchema 目前仅支持 object 类型", "config.responseSchema")
return None
def _check_skill(config: dict[str, Any], catalog: ResourceCatalog) -> dict[str, Any] | None:
mode = str(config.get("mode") or "agent_skill")
if mode != "agent_skill":
return _issue("callable_skill 模式暂未开放执行,请改用 agent_skill", "config.mode")
agent_id = str(config.get("agentId") or config.get("agent_id") or "")
if not agent_id:
return _issue("技能节点未选择智能体", "config.agentId")
if agent_id not in catalog.agent_ids:
return _issue("技能节点引用的智能体不可访问", "config.agentId")
skill_names = [str(s) for s in (config.get("skillNames") or config.get("skill_names") or [])]
if not skill_names:
return _issue("技能节点至少需要选择一个技能", "config.skillNames")
for name in skill_names:
if name not in catalog.enabled_skills:
return _issue(f"技能不存在或未启用:{name}", "config.skillNames")
whitelist = catalog.agent_skills.get(agent_id)
if whitelist:
missing = [name for name in skill_names if name not in whitelist]
if missing:
return _issue(f"技能不在智能体白名单内:{'、'.join(missing)}", "config.skillNames")
return None
def _check_sql(config: dict[str, Any], catalog: ResourceCatalog, cfg: WorkflowConfig) -> dict[str, Any] | None:
source_id = str(config.get("dataSourceId") or config.get("data_source_id") or "")
if not source_id:
return _issue("SQL 节点未选择数据源", "config.dataSourceId")
source = catalog.data_sources.get(source_id)
if source is None:
return _issue("数据源不存在或无权访问", "config.dataSourceId")
if not source.get("enabled", True):
return _issue("数据源已被禁用", "config.dataSourceId")
if str(source.get("kind") or "sql") != "sql":
return _issue("该资源不是 SQL 数据源", "config.dataSourceId")
statement = str(config.get("statement") or config.get("queryTemplate") or config.get("query_template") or "")
if not statement.strip():
return _issue("SQL 节点缺少查询语句", "config.statement")
try:
assert_read_only(statement)
assert_tables_allowed(statement, list(source.get("allowed_tables") or []))
except WorkflowError as exc:
return _issue(str(exc.message if hasattr(exc, "message") else exc), "config.statement")
parameters = config.get("parameters")
if parameters is not None and not isinstance(parameters, dict):
return _issue("SQL 参数必须是对象", "config.parameters")
max_rows = config.get("maxRows") or config.get("max_rows")
if max_rows is not None:
try:
rows = int(max_rows)
except (TypeError, ValueError):
return _issue("maxRows 必须是整数", "config.maxRows")
ceiling = min(int(cfg.sql.max_rows), int(source.get("max_rows") or cfg.sql.max_rows))
if rows < 1 or rows > ceiling:
return _issue(f"maxRows 超过上限 {ceiling}", "config.maxRows")
return None
def _check_http(config: dict[str, Any], catalog: ResourceCatalog) -> dict[str, Any] | None:
method = str(config.get("method") or "GET").upper()
if method not in _HTTP_METHODS:
return _issue(f"不支持的 HTTP 方法:{method}", "config.method")
headers = config.get("headers")
if headers is not None:
if not isinstance(headers, dict):
return _issue("请求头必须是字符串到字符串的映射", "config.headers")
for name in headers:
if str(name).strip().lower() in _LITERAL_SECRET_KEYS:
return _issue(f"请勿在节点上明文配置 {name},改用凭据引用", "config.headers")
url = str(config.get("url") or "").strip()
credential_ref = str(config.get("credentialRef") or config.get("credential_ref") or "").strip()
if not url and not credential_ref:
return _issue("HTTP 节点缺少 URL 或凭据引用", "config.url")
if credential_ref:
source = catalog.data_sources.get(credential_ref)
if source is None:
return _issue("引用的凭据(HTTP 数据源)不存在或无权访问", "config.credentialRef")
if not source.get("enabled", True):
return _issue("引用的凭据已被禁用", "config.credentialRef")
if str(source.get("kind") or "") != "http":
return _issue("引用的凭据不是 HTTP 类型数据源", "config.credentialRef")
allowed = {str(m).upper() for m in (source.get("allowed_methods") or [])}
if allowed and method not in allowed:
return _issue(f"方法 {method} 不在凭据资源允许的方法列表内", "config.method")
return None
def _check_code(config: dict[str, Any], cfg: WorkflowConfig) -> dict[str, Any] | None:
if not cfg.code.enabled:
return _issue("代码节点未启用(需管理员在 config.yaml 中开启 workflows.code.enabled)", "")
language = config.get("language")
if language is not None and str(language) != "python":
return _issue("代码节点目前仅支持 python", "config.language")
source = str(config.get("source") or "")
if not source.strip():
return _issue("代码节点没有源代码", "config.source")
timeout = config.get("timeoutSeconds") or config.get("timeout_seconds")
if timeout is not None:
try:
seconds = int(timeout)
except (TypeError, ValueError):
return _issue("timeoutSeconds 必须是整数", "config.timeoutSeconds")
if seconds < 1 or seconds > int(cfg.code.timeout_seconds):
return _issue(f"timeoutSeconds 超过上限 {cfg.code.timeout_seconds}", "config.timeoutSeconds")
return None
def _check_output(config: dict[str, Any]) -> dict[str, Any] | None:
mapping = config.get("mapping")
if mapping is None:
return None
if not isinstance(mapping, dict):
return _issue("输出映射必须是对象", "config.mapping")
for key, template in mapping.items():
if not isinstance(template, str) or not template.strip():
return _issue(f"输出字段 {key} 的映射模板为空", f"config.mapping.{key}")
return None
def _check_subworkflow(config: dict[str, Any], catalog: ResourceCatalog, *, current_workflow_id: str) -> dict[str, Any] | None:
workflow_id = str(config.get("workflowId") or config.get("workflow_id") or "")
if not workflow_id:
return _issue("子工作流节点缺少 workflowId", "config.workflowId")
version_id = str(config.get("versionId") or config.get("version_id") or "")
if not version_id:
return _issue("子工作流节点缺少 versionId", "config.versionId")
if workflow_id == current_workflow_id:
return _issue("子工作流不能引用自身", "config.workflowId")
versions = catalog.published_subworkflows.get(workflow_id)
if not versions:
return _issue("子工作流未发布或无权访问", "config.workflowId")
if version_id not in versions:
return _issue("子工作流版本不存在或已下线", "config.versionId")
return None
def make_resource_checker(
catalog: ResourceCatalog,
*,
config: WorkflowConfig,
current_workflow_id: str = "",
):
"""Build the sync ``resource_checker`` consumed by ``validate_workflow_graph``."""
def checker(node_type: str, node_config: dict[str, Any]):
try:
if node_type == "agent":
return _check_agent(node_config, catalog)
if node_type == "skill":
return _check_skill(node_config, catalog)
if node_type == "sql_read":
return _check_sql(node_config, catalog, config)
if node_type == "http":
return _check_http(node_config, catalog)
if node_type == "code":
return _check_code(node_config, config)
if node_type == "output":
return _check_output(node_config)
if node_type == "subworkflow":
return _check_subworkflow(node_config, catalog, current_workflow_id=current_workflow_id)
except Exception: # noqa: BLE001 - a broken check must fail closed
return _issue("资源配置校验失败,请检查节点配置", "")
return None
return checker
__all__ = ["ResourceCatalog", "build_resource_catalog", "make_resource_checker"]
@@ -0,0 +1,104 @@
"""Periodic retention sweep for workflow runs, events, and artifacts.
Consumes ``workflows.retention.event_retention_days`` /
``run_retention_days``. Events older than the event window are dropped first;
terminal runs older than the run window are deleted with their node-run and
artifact rows, and any leftover events for those run ids are cleaned up so the
event log cannot outlive the run it describes.
Artifact *files* stay on the existing sandbox/artifact lifecycle — this job
only removes the workflow catalog rows.
"""
from __future__ import annotations
import asyncio
import logging
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
logger = logging.getLogger(__name__)
_INTERVAL_SECONDS = 60 * 60
_MAX_BATCHES_PER_SWEEP = 8
_SQL_PURGE_BATCH = 500
class WorkflowRetentionCleaner:
def __init__(
self,
runs: WorkflowRunStore,
events: WorkflowEventStore,
*,
config: WorkflowConfig | None = None,
) -> None:
self._runs = runs
self._events = events
self._config = config or WorkflowConfig()
self._task: asyncio.Task | None = None
self._stopping = False
def start(self) -> None:
if self._task is not None and not self._task.done():
return
self._stopping = False
self._task = asyncio.create_task(self._loop(), name="workflow-retention")
logger.info(
"workflow retention cleaner started (events=%sd runs=%sd)",
self._config.retention.event_retention_days,
self._config.retention.run_retention_days,
)
async def stop(self) -> None:
self._stopping = True
if self._task is not None:
self._task.cancel()
try:
await self._task
except (asyncio.CancelledError, Exception): # noqa: BLE001
pass
self._task = None
async def sweep(self) -> dict[str, int]:
"""One pass. Safe to call from tests without starting the loop."""
event_days = max(1, self._config.retention.event_retention_days)
run_days = max(1, self._config.retention.run_retention_days)
events_deleted = await self._events.purge_before(older_than_days=event_days)
runs_deleted = 0
for _ in range(_MAX_BATCHES_PER_SWEEP):
ids = await self._runs.purge_terminal_before(older_than_days=run_days)
if not ids:
break
runs_deleted += len(ids)
events_deleted += await self._events.purge_for_runs(ids)
if len(ids) < _SQL_PURGE_BATCH:
break
if events_deleted or runs_deleted:
logger.info("workflow retention sweep: events=%s runs=%s", events_deleted, runs_deleted)
return {"events": events_deleted, "runs": runs_deleted}
async def _loop(self) -> None:
try:
await self.sweep()
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001 - the loop must survive a bad pass
logger.exception("workflow retention sweep failed")
while not self._stopping:
try:
await asyncio.sleep(_INTERVAL_SECONDS)
except asyncio.CancelledError:
raise
if self._stopping:
return
try:
await self.sweep()
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001
logger.exception("workflow retention sweep failed")
__all__ = ["WorkflowRetentionCleaner"]
@@ -647,3 +647,45 @@ task_deeplink:
taskDirection: "正向引导 + 风险预警"
taskName: "任务01:暑期文旅舆情综合研判"
taskContent: "围绕暑期文旅消费高峰,对重点城市、重点景区的公众情绪与舆情态势做综合研判,识别情绪拐点与潜在风险,提出正向引导与处置建议。"
# Workflow Studio (phase 1: definitions/publish; runtime later)
workflows:
enabled: true
max_steps: 500
max_loop_iterations: 20
max_parallelism: 16
run_timeout_seconds: 3600
node_timeout_seconds: 600
max_concurrent_runs_per_user: 5
lease_ttl_seconds: 60
embed:
allowed_origins: []
ticket_ttl_seconds: 60
# http 节点出网策略:默认拒绝内网/环回/云元数据,只放行公网。
# allowed_hosts 非空时变为白名单模式(精确域名或 .suffix)。
http:
enabled: true
allowed_hosts: []
blocked_hosts: []
allow_private_networks: false
allowed_ports: [80, 443, 8080, 8443]
max_response_bytes: 2097152
max_redirects: 0
timeout_seconds: 60
# sql_read 节点:仅单条 SELECT/WITH,参数绑定,强制 LIMIT。
sql:
enabled: true
max_rows: 1000
statement_timeout_seconds: 30
max_cell_chars: 4000
# code 节点默认关闭:本地执行器是加固子进程而非容器,需管理员显式开启。
code:
enabled: false
timeout_seconds: 30
max_source_chars: 20000
max_output_chars: 100000
memory_limit_mb: 512
allow_network: false
retention:
event_retention_days: 30
run_retention_days: 90
@@ -22,6 +22,16 @@ This directory contains detailed documentation for the DeerFlow backend.
| [plan_mode_usage.md](plan_mode_usage.md) | Plan mode with TodoList |
| [AUTO_TITLE_GENERATION.md](AUTO_TITLE_GENERATION.md) | Automatic title generation |
## Workflow Studio
| Document | Description |
|----------|-------------|
| [WORKFLOW_STUDIO_BACKEND_DEV_ZH.md](WORKFLOW_STUDIO_BACKEND_DEV_ZH.md) | 后端开发基线方案 + 各阶段完成状态 |
| [adr/0001-workflow-studio-runtime.md](adr/0001-workflow-studio-runtime.md) | ADR:Coze 只做画布 / ChatDev 只作参考 / 自建 DAG 调度器(LangGraph 仅作智能体内核) |
| [WORKFLOW_SCHEMA_V1_ZH.md](WORKFLOW_SCHEMA_V1_ZH.md) | 内部图 Schema v1.0 |
| [WORKFLOW_SSE_V1_ZH.md](WORKFLOW_SSE_V1_ZH.md) | SSE 事件协议 v1.0 |
| [WORKFLOW_OPENAPI_DRAFT_V1.yaml](WORKFLOW_OPENAPI_DRAFT_V1.yaml) | 接口 OpenAPI 草案 |
## Development
| Document | Description |
@@ -0,0 +1,740 @@
openapi: 3.0.3
info:
title: DeerFlow Workflow Studio API (Draft v1)
version: "1.0.0-draft"
description: |
Wire models: deerflow.workflows.schemas / events / errors.
Paths marked `x-implemented: false` are still sketches; everything else is
mounted today (app/gateway/routers/workflow*.py). Where the phase-0 sketch
and the implementation disagreed, this file follows the implementation.
servers:
- url: http://localhost:8001
description: Local Gateway
tags:
- name: workflows
- name: runs
- name: resources
- name: data-sources
- name: embed
- name: coze-compat
paths:
/api/workflows:
get:
tags: [workflows]
summary: List workflow definitions
responses:
"200":
description: OK
post:
tags: [workflows]
summary: Create workflow definition
responses:
"201":
description: Created
/api/workflows/{workflow_id}:
get:
tags: [workflows]
summary: Get workflow definition
parameters:
- $ref: "#/components/parameters/WorkflowId"
responses:
"200":
description: OK
patch:
tags: [workflows]
summary: Update workflow metadata
parameters:
- $ref: "#/components/parameters/WorkflowId"
responses:
"200":
description: OK
delete:
tags: [workflows]
summary: Soft-delete / archive
parameters:
- $ref: "#/components/parameters/WorkflowId"
responses:
"204":
description: Archived
/api/workflows/{workflow_id}/copy:
post:
tags: [workflows]
summary: Duplicate a workflow as a new draft owned by the caller
parameters:
- $ref: "#/components/parameters/WorkflowId"
responses:
"200":
description: Created copy
/api/workflows/{workflow_id}/draft:
put:
tags: [workflows]
summary: Save draft (optimistic lock)
parameters:
- $ref: "#/components/parameters/WorkflowId"
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [expectedRevision, graph]
properties:
expectedRevision:
type: integer
graph:
$ref: "#/components/schemas/WorkflowGraph"
responses:
"200":
description: Saved
"409":
description: Draft revision conflict
/api/workflows/{workflow_id}/validate:
post:
tags: [workflows]
summary: Validate draft without publishing
parameters:
- $ref: "#/components/parameters/WorkflowId"
responses:
"200":
description: Validation result
/api/workflows/{workflow_id}/publish:
post:
tags: [workflows]
summary: Publish immutable version
parameters:
- $ref: "#/components/parameters/WorkflowId"
responses:
"201":
description: Version created
/api/workflows/{workflow_id}/versions:
get:
tags: [workflows]
summary: List published versions
parameters:
- $ref: "#/components/parameters/WorkflowId"
responses:
"200":
description: OK
/api/workflows/{workflow_id}/versions/{version_id}:
get:
tags: [workflows]
summary: Get one published version snapshot
parameters:
- $ref: "#/components/parameters/WorkflowId"
- $ref: "#/components/parameters/VersionId"
responses:
"200":
description: OK
/api/workflows/node-types:
get:
tags: [resources]
summary: List supported node types
responses:
"200":
description: OK
/api/workflows/resources/agents:
get:
tags: [resources]
summary: Agents available to workflow nodes
responses:
"200":
description: OK
/api/workflows/resources/skills:
get:
tags: [resources]
summary: Skills available to workflow nodes
responses:
"200":
description: OK
/api/workflows/resources/data-sources:
get:
tags: [resources]
summary: Registered read-only data sources
responses:
"200":
description: OK
/api/workflows/resources/subworkflows:
get:
tags: [resources]
summary: Published workflows usable as subworkflows
responses:
"200":
description: OK
/api/workflows/data-sources:
get:
tags: [data-sources]
summary: List data sources visible to the caller (own + shared)
description: |
Never returns the DSN or credential headers — only `masked_target`.
responses:
"200":
description: OK
post:
tags: [data-sources]
summary: Register a data source (`shared: true` is admin-only)
description: |
`kind: sql` requires `dsn`; `kind: http` requires `headers`. The secret
is encrypted on write and only ever decrypted inside the run executor.
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/DataSourceCreate"
responses:
"200":
description: Created (masked view)
"503":
description: WORKFLOW_SECRET_KEY not configured
/api/workflows/data-sources/{id}:
get:
tags: [data-sources]
summary: Get one data source (masked view)
parameters:
- $ref: "#/components/parameters/DataSourceId"
responses:
"200":
description: OK
put:
tags: [data-sources]
summary: Update data source (owner or admin)
parameters:
- $ref: "#/components/parameters/DataSourceId"
responses:
"200":
description: OK
delete:
tags: [data-sources]
summary: Delete data source (owner or admin)
parameters:
- $ref: "#/components/parameters/DataSourceId"
responses:
"200":
description: '{ "deleted": true }'
/api/workflows/data-sources/{id}/introspect:
post:
tags: [data-sources]
summary: Introspect whitelisted schema/table/column metadata
description: Returns the configured table allowlist only. Never returns sample rows or secrets.
parameters:
- $ref: "#/components/parameters/DataSourceId"
responses:
"200":
description: '{ dataSourceId, kind, tables, schemas }'
/api/workflows/data-sources/{id}/validate-query:
post:
tags: [data-sources]
summary: Validate SQL against read-only policy (no execution)
parameters:
- $ref: "#/components/parameters/DataSourceId"
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [statement]
properties:
statement:
type: string
responses:
"200":
description: '{ ok: true, statement }'
"400":
description: WORKFLOW_SQL_POLICY_DENIED
/api/workflows/{workflow_id}/runs:
post:
tags: [runs]
summary: Start a run against a published version
parameters:
- $ref: "#/components/parameters/WorkflowId"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/StartRunRequest"
responses:
"200":
description: |
Run view plus `created`; `created: false` means the idempotencyKey
matched an existing run (scoped per workflow).
"400":
description: WORKFLOW_INPUT_INVALID / WORKFLOW_VERSION_NOT_FOUND
"429":
description: WORKFLOW_LIMIT_EXCEEDED (per-user concurrent runs)
get:
tags: [runs]
summary: List runs for a workflow
parameters:
- $ref: "#/components/parameters/WorkflowId"
responses:
"200":
description: OK
/api/workflows/runs/{run_id}:
get:
tags: [runs]
summary: Get run status projection plus current node runs
description: |
Never includes `resumeToken` — that is delivered on the event stream only.
parameters:
- $ref: "#/components/parameters/RunId"
responses:
"200":
description: OK
"403":
description: WORKFLOW_FORBIDDEN (not owner, not admin)
"404":
description: WORKFLOW_RUN_NOT_FOUND
/api/workflows/runs/{run_id}/events:
get:
tags: [runs]
summary: Page durable events
parameters:
- $ref: "#/components/parameters/RunId"
- name: after
in: query
description: Exclusive `seq` cursor. `after_seq` is accepted as an alias.
schema:
type: integer
default: 0
- name: after_seq
in: query
schema:
type: integer
- name: limit
in: query
schema:
type: integer
default: 200
maximum: 1000
responses:
"200":
description: '{ "events": [...], "lastSeq": 42 }'
/api/workflows/runs/{run_id}/stream:
get:
tags: [runs]
summary: SSE replay + live tail
description: |
Replays from `Last-Event-ID` (preferred) or `after` / `after_seq`, then
tails live. Closes on the first terminal event; if the run is already
terminal without one, a synthesised terminal frame is emitted so the
client never hangs.
parameters:
- $ref: "#/components/parameters/RunId"
- name: after
in: query
schema:
type: integer
default: 0
- name: after_seq
in: query
schema:
type: integer
- name: Last-Event-ID
in: header
schema:
type: string
responses:
"200":
description: text/event-stream
headers:
X-Workflow-Run-Status:
description: Run status at connection time.
schema:
type: string
content:
text/event-stream:
schema:
type: string
/api/workflows/runs/{run_id}/cancel:
post:
tags: [runs]
summary: Request cancellation (idempotent)
parameters:
- $ref: "#/components/parameters/RunId"
responses:
"200":
description: |
Run view plus `cancelled`; `false` means it was already terminal.
/api/workflows/runs/{run_id}/resume:
post:
tags: [runs]
summary: Resume after human_input
parameters:
- $ref: "#/components/parameters/RunId"
requestBody:
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/ResumeRunRequest"
responses:
"200":
description: Requeued
"409":
description: WORKFLOW_RUN_NOT_RESUMABLE / WORKFLOW_RESUME_TOKEN_INVALID
/api/workflows/runs/{run_id}/retry:
post:
tags: [runs]
summary: Retry a failed run from the first unfinished node
description: |
Creates a new queued run with `retryOfRunId` pointing at the failed
original. Completed `workflow_node_runs` are copied so the engine
resumes from the failed node; if nothing is replayable it is a full
re-run of the same version and inputs. Only `failed` runs are retryable.
parameters:
- $ref: "#/components/parameters/RunId"
responses:
"201":
description: '{ ..., retryOfRunId, created: true }'
"409":
description: WORKFLOW_RUN_NOT_RETRYABLE
"429":
description: WORKFLOW_LIMIT_EXCEEDED
/api/workflows/runs/{run_id}/artifacts:
get:
tags: [runs]
summary: List artifacts for a run
parameters:
- $ref: "#/components/parameters/RunId"
responses:
"200":
description: OK
/api/workflows/embed/tickets:
post:
tags: [embed]
summary: Issue a short-lived iframe ticket (authenticated parent)
description: |
`origin` must exactly match `workflows.embed.allowed_origins`. The
response also carries `frameAncestors` for the host's CSP.
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [workflowId, origin]
properties:
workflowId:
type: string
origin:
type: string
responses:
"200":
description: '{ ticket, expiresIn, origin, frameAncestors }'
"403":
description: WORKFLOW_EMBED_ORIGIN_DENIED / WORKFLOW_FORBIDDEN
"503":
description: No signing key configured
/api/workflows/embed/verify:
post:
tags: [embed]
summary: Verify a ticket (single-use, best effort per process)
description: |
A ticket proves "this origin may embed this workflow for this user right
now". It is NOT an API credential — embedded pages still authenticate
their API calls with the normal session.
requestBody:
required: true
content:
application/json:
schema:
type: object
required: [ticket]
properties:
ticket:
type: string
origin:
type: string
responses:
"200":
description: '{ ok, workflowId, userId, origin, expiresAt }'
"401":
description: WORKFLOW_EMBED_TICKET_INVALID (expired or replayed)
"403":
description: WORKFLOW_EMBED_ORIGIN_DENIED (origin mismatch)
/api/workflow_api/canvas:
post:
tags: [coze-compat]
summary: Coze canvas load adapter
responses:
"200":
description: Coze DTO
/api/workflow_api/save:
post:
tags: [coze-compat]
summary: Coze canvas save adapter
responses:
"200":
description: Saved
/api/workflow_api/create:
post:
tags: [coze-compat]
summary: Coze create workflow adapter
responses:
"200":
description: Created
/api/workflow_api/node_template_list:
post:
tags: [coze-compat]
summary: Coze node template list adapter
responses:
"200":
description: Templates
components:
parameters:
WorkflowId:
name: workflow_id
in: path
required: true
schema:
type: string
VersionId:
name: version_id
in: path
required: true
schema:
type: string
RunId:
name: run_id
in: path
required: true
schema:
type: string
DataSourceId:
name: id
in: path
required: true
schema:
type: string
schemas:
WorkflowGraph:
type: object
description: Internal standard graph (see WORKFLOW_SCHEMA_V1_ZH.md)
required: [schemaVersion, id, nodes, edges]
properties:
schemaVersion:
type: string
enum: ["1.0"]
id:
type: string
name:
type: string
description:
type: string
inputSchema:
type: object
outputSchema:
type: object
nodes:
type: array
items:
$ref: "#/components/schemas/WorkflowNode"
edges:
type: array
items:
$ref: "#/components/schemas/WorkflowEdge"
settings:
$ref: "#/components/schemas/WorkflowGraphSettings"
WorkflowNode:
type: object
required: [id, type]
properties:
id:
type: string
type:
type: string
enum:
- start
- output
- agent
- skill
- http
- sql_read
- code
- transform
- condition
- merge
- human_input
- subworkflow
- loop
name:
type: string
config:
type: object
WorkflowEdge:
type: object
required: [id, source, target]
properties:
id:
type: string
source:
type: string
target:
type: string
sourcePort:
type: string
nullable: true
WorkflowGraphSettings:
type: object
properties:
runTimeoutSeconds:
type: integer
default: 1800
nodeTimeoutSeconds:
type: integer
default: 300
maxSteps:
type: integer
default: 200
maxLoopIterations:
type: integer
default: 5
maxParallelism:
type: integer
default: 4
NodeResult:
type: object
properties:
data:
type: object
messages:
type: array
items:
type: object
artifacts:
type: array
items:
$ref: "#/components/schemas/ArtifactRef"
metadata:
type: object
warnings:
type: array
items:
type: string
ArtifactRef:
type: object
required: [artifactId, name]
properties:
artifactId:
type: string
name:
type: string
mimeType:
type: string
path:
type: string
sizeBytes:
type: integer
nullable: true
preview:
type: string
nullable: true
StartRunRequest:
type: object
properties:
versionId:
type: string
description: Omit to use the latest published version.
inputs:
type: object
idempotencyKey:
type: string
maxLength: 128
executionMode:
type: string
enum: [normal]
default: normal
ResumeRunRequest:
type: object
required: [resumeToken]
properties:
resumeToken:
type: string
nodeId:
type: string
description: Defaults to the paused node from `pendingInput`.
action:
type: string
default: submit
values:
type: object
DataSourceCreate:
type: object
required: [name]
properties:
name:
type: string
description:
type: string
kind:
type: string
enum: [sql, http]
default: sql
dsn:
type: string
description: Write-only; required for `kind: sql`.
headers:
type: object
additionalProperties:
type: string
description: Write-only; required for `kind: http`.
maxRows:
type: integer
default: 1000
allowedTables:
type: array
items:
type: string
enabled:
type: boolean
default: true
shared:
type: boolean
default: false
description: Admin-only. Shared sources have a null owner.
WorkflowEventEnvelope:
type: object
required: [schemaVersion, runId, workflowId, versionId, seq, event]
properties:
schemaVersion:
type: string
enum: ["1.0"]
runId:
type: string
workflowId:
type: string
versionId:
type: string
seq:
type: integer
event:
type: string
nodeId:
type: string
nullable: true
nodeRunId:
type: string
nullable: true
timestamp:
type: string
format: date-time
data:
type: object
WorkflowErrorBody:
type: object
required: [code, message]
properties:
code:
type: string
message:
type: string
retryable:
type: boolean
nodeId:
type: string
nullable: true
details:
type: object
@@ -0,0 +1,93 @@
# 工作流图 Schema v1.0(冻结)
> 阶段 0 交付。实现源:`packages/harness/deerflow/workflows/schemas.py`
> 样例:`packages/harness/deerflow/workflows/samples/`
> ADR:[`adr/0001-workflow-studio-runtime.md`](adr/0001-workflow-studio-runtime.md)
## 原则
- Coze 画布 DTO **进入 Gateway 后**转换成此内部结构;运行时不依赖 Coze 私有字段。
- Wire JSON 使用 **camelCase**;Python 模型使用 snake_case(`populate_by_name=True`)。
- `schemaVersion` 固定为 `"1.0"`;破坏性变更必须升版本并双写迁移说明。
## 顶层 `WorkflowGraph`
| 字段 | 类型 | 说明 |
|------|------|------|
| `schemaVersion` | `"1.0"` | 协议版本 |
| `id` | string | 工作流逻辑 id(定义侧稳定标识) |
| `name` / `description` | string | 展示用 |
| `inputSchema` / `outputSchema` | JSON Schema object | 开始/输出校验 |
| `nodes` | `WorkflowNode[]` | 节点列表,`id` 唯一 |
| `edges` | `WorkflowEdge[]` | 边列表,`id` 唯一;`source`/`target` 必须引用已有节点 |
| `settings` | object | 见下 |
### `settings`
| 字段 | 默认 | 说明 |
|------|------|------|
| `runTimeoutSeconds` | 1800 | 整 run 超时 |
| `nodeTimeoutSeconds` | 300 | 单节点默认超时 |
| `maxSteps` | 200 | 最大节点步数 |
| `maxLoopIterations` | 5 | 全局循环上限 |
| `maxParallelism` | 4 | 同层并行度 |
发布时仍受系统级配额覆盖(阶段 6)。
## 节点类型(封闭集合)
`start` · `output` · `agent` · `skill` · `http` · `sql_read` · `code` · `transform` · `condition` · `merge` · `human_input` · `subworkflow` · `loop`
每类最小样例见 `workflows/samples/nodes/<type>.json`。
## `NodeResult`(统一节点结果)
```json
{
"data": {},
"messages": [],
"artifacts": [
{
"artifactId": "artifact_01",
"name": "专题报告.md",
"mimeType": "text/markdown",
"path": "/mnt/data/专题报告.md"
}
],
"metadata": {},
"warnings": []
}
```
大文本 / 二进制 / 超大查询结果写入 artifact,事件与 `output_json` 只传引用与有界预览。
## 运行状态
`queued` → `running` → `awaiting_input` | `completed` | `failed` | `cancel_requested` → `cancelled`
终态:`completed` / `failed` / `cancelled`。
## 安全表达式(约定,求值器阶段 2+)
允许只读引用:
- `{{ inputs.<field> }}`
- `{{ nodes.<nodeId>.data... }}`
- `{{ run.id }}`
禁止:`eval`、任意属性反射、字符串拼接构造 SQL/HTTP。条件节点运算符白名单:相等、包含、数值比较、空值、布尔组合。
## `graph_hash`
对规范化 camelCase JSON(`model_dump_json(by_alias=True, exclude_none=True)`)做 SHA-256。发布版本写入 `workflow_versions.graph_hash`;编译缓存可按 hash 做有界缓存,缓存对象不得含用户密钥。
## 样例图
| 文件 | 用途 |
|------|------|
| `samples/report_generation.json` | 专题报告端到端验收基线 |
| `samples/start_output.json` | 最小 start→output(运行骨架) |
| `samples/nodes/*.json` | 各节点最小 config |
| `samples/event_envelope_example.json` | SSE envelope 契约样例 |
加载:`from deerflow.workflows.samples import load_sample_graph`。
@@ -0,0 +1,107 @@
# 工作流 SSE / 事件协议 v1.0(冻结)
> 实现源:`packages/harness/deerflow/workflows/events.py`
> 错误体:`packages/harness/deerflow/workflows/errors.py`
> 事件清单与开发方案对齐:`WORKFLOW_STUDIO_BACKEND_DEV_ZH.md` §9
## 不变式
1. **先持久化,后发布** — 用户可见事件先写入 `workflow_run_events`,提交成功后再 fan-out。
2. **`seq` 严格单调** — 唯一键 `(run_id, seq)`。
3. **可重放** — `Last-Event-ID` 优先于 `after_seq`,否则从 `0`。
4. **增量合并** — 模型 token 建议 30–100ms 或约 1KB 合并落库。
5. **终态停连** — `run.completed` / `run.failed` / `run.cancelled` 且库已补齐后关闭 SSE。
6. **不泄密** — 无凭据、无完整认证头、无 DB 连接串、无隐藏思维链。
## 端点(阶段 2 已实现)
```http
GET /api/workflows/runs/{run_id}/stream?after=128
Accept: text/event-stream
Last-Event-ID: 128
```
实现:`app/gateway/routers/workflow_runs.py`。
- **库是排序的权威来源**:连接先按 `Last-Event-ID`(优先)或 `after` 从 `workflow_run_events` 回放,再挂 `WorkflowLiveHub` 订阅实时帧;`seq` 不大于已回放游标的实时帧被丢弃,下一轮 DB 轮询(0.8s)会带上它。断线重连因此不会看到空洞或乱序。
- **终态必闭**:读到 `run.completed` / `run.failed` / `run.cancelled` 即关闭。若运行已是终态却没有终态事件(例如崩在写事件之前),补发一条 `synthesised: true` 的终态帧再关闭,客户端不会永久挂住。
- **响应头** `X-Workflow-Run-Status` 给出连接建立瞬间的状态。
- Heartbeat:SSE comment `: heartbeat`(**不写库**),15s 静默后发送。
- 同一份事件也可用 `GET /api/workflows/runs/{run_id}/events?after=&limit=` 分页拉取(无长连接场景)。
## Envelope
```json
{
"schemaVersion": "1.0",
"runId": "run_01",
"workflowId": "wf_01",
"versionId": "wv_03",
"seq": 42,
"event": "node.output.delta",
"nodeId": "agent_writer",
"nodeRunId": "nr_09",
"timestamp": "2026-08-26T10:00:00+08:00",
"data": {}
}
```
SSE 帧:
```text
id: 42
event: node.output.delta
data: { ... envelope ... }
```
Python:`WorkflowEventEnvelope.to_sse_frame()`。
## 事件类型(封闭集合)
| 事件 | `data` 要点 |
|------|-------------|
| `run.created` | input 摘要、version |
| `run.queued` | queue 信息 |
| `run.started` | startedAt |
| `node.queued` | node 信息 |
| `node.started` | attempt、输入摘要 |
| `node.progress` | phase、message、percent |
| `node.output.delta` | channel、delta |
| `node.tool.started` | toolCallId、name、input 摘要 |
| `node.tool.finished` | toolCallId、status、output 摘要 |
| `artifact.created` | artifact 元数据 |
| `node.completed` | output 摘要、duration |
| `node.failed` | error code/message/retryable |
| `run.awaiting_input` | formSchema、actions、resumeToken |
| `run.resumed` | action |
| `run.cancel_requested` | requestedBy |
| `run.cancelled` | finishedAt |
| `run.completed` | output、artifacts |
| `run.failed` | error、failedNodeId |
## 统一错误体
```json
{
"code": "WORKFLOW_SQL_POLICY_DENIED",
"message": "SQL 节点只允许单条只读查询",
"retryable": false,
"nodeId": "sql_1",
"details": { "rule": "single_read_statement" }
}
```
完整错误码列表:`ALL_WORKFLOW_ERROR_CODES`(`deerflow.workflows.errors`)。
## `resumeToken` 只走事件流
`run.awaiting_input` 的 `data.resumeToken` 是恢复运行的唯一凭证,**只在事件流里出现**——`GET /api/workflows/runs/{run_id}` 的运行视图不包含它。恢复时 `POST /api/workflows/runs/{run_id}/resume` 携带该 token,服务端以「消费即失效」的方式原子翻转状态(同一个 token 不能恢复两次)。
## 前后端契约测试
前后端共用 fixture:
- `workflows/samples/event_envelope_example.json`
- 本文件事件名表
实现测试:`tests/test_workflow_runs.py`(回放游标、终态关闭、暂停/恢复的事件序列)。
@@ -3,7 +3,16 @@
> 文档状态:开发基线方案
> 适用项目:`offline-backend-20260512/backend`
> 对接前端:Coze Studio 独立工作流页面 `frontend/apps/coze-studio/src/workflow-standalone/`
> 更新时间:2026-08-26
> 更新时间:2026-08-29
> **阶段 0 状态:已完成** — 协议代码 `packages/harness/deerflow/workflows/`;文档见 `WORKFLOW_SCHEMA_V1_ZH.md` / `WORKFLOW_SSE_V1_ZH.md` / `WORKFLOW_OPENAPI_DRAFT_V1.yaml` / `adr/0001-workflow-studio-runtime.md`;契约测试 `tests/test_workflow_schemas.py`
> **阶段 1 状态:已完成** — 定义/版本持久化、草稿乐观锁、validate/publish、节点目录、Coze canvas/save/create/node_template 最小适配;路由 `/api/workflows*` + `/api/workflow_api*`;测试 `tests/test_workflow_definitions.py`(尚不执行运行)
> **阶段 2 状态:已完成** — run/node_run/event/artifact 表 + 迁移 `20260828_02`;幂等启动、租约调度、重启恢复、持久事件日志、SSE 回放/追尾。**编排内核改为自建 DAG 调度器,不再编译成 LangGraph**(理由见 `adr/0001` 阶段 2 修订)
> **阶段 3 状态:已完成** — start/output/transform/condition/merge + 安全表达式引擎;http 节点 SSRF 默认拒内网;sql_read 只读策略 + 数据源登记(密钥加密、只写不读,迁移 `20260828_03`);code 节点加固子进程执行器(**默认关闭**)
> **阶段 4 状态:已完成** — agent / skill 节点走 DeerFlow lead agent,模型增量与工具步骤映射为 `node.output.delta` / `node.tool.*`
> **阶段 5 状态:已完成** — `human_input` 暂停 + 一次性 resume token、子工作流(深度上限 3)、显式循环与 `max_loop_iterations` / `max_steps` 上限
> **阶段 6 状态:已完成** — iframe 票据(origin 白名单 + 60s 一次性 + `frameAncestors`)、按 owner/admin 的运行与数据源鉴权、每用户并发运行上限、结构化审计日志、事件/运行保留期配置与后台清理作业(`WorkflowRetentionCleaner`,每小时扫一次)、失败运行 `POST /retry`(复制已完成节点从失败点续跑,新 run 标 `retry_of_run_id`)、SSE `Last-Event-ID` 优先于 `after`。制品落盘文件仍跟随现有 sandbox/artifact 生命周期,不做工作流专属文件回收。多实例租约竞争 + SSE 重连有单机并发测试(`test_two_dispatchers_only_one_claims_a_run` / `test_sse_reconnect_storm_is_ordered_and_complete`),不是外部分布式压测。
> **审计整改(2026-08-29)状态:功能健壮性项已完成** — 按 `WORKFLOW_STUDIO_BACKEND_REMEDIATION_ZH.md`「0. 整改决策记录」:①人工恢复原子化(`resume_with_payload` 单条件更新 + `set_awaiting_input` 一次写入 checkpoint);②草稿保存 CAS 条件更新(409 带 `currentRevision`)+ 发布版本号计数器 `workflow_definitions.next_version_number`;③事件 seq 计数器 `workflow_runs.next_event_seq`(同事务取号,冲突自愈重试),事件落库失败**丢弃不发布**(删除负序号幽灵事件);④输入/输出完整 JSON Schema 校验(`workflows/schema_validation.py`,路由 400 + 节点错误均带字段路径);⑤测试纳入 Git(`.gitignore` 白名单)+ 新增 SQL 并发回归 `tests/test_workflow_concurrency.py`。迁移 `20260828_04_workflow_counter_columns.py` 增加两个计数器列(存量库由启动列同步自动补齐)。P0-01/02(内网可信环境,不实施)、P1-04/05(安全隔离,暂缓)见整改文档决策记录。
> 运行时测试:`tests/test_workflow_engine.py`(引擎、表达式、节点、安全策略)+ `tests/test_workflow_runs.py`(HTTP 生命周期、SSE、暂停恢复、租约回收、数据源、票据)+ `tests/test_workflow_concurrency.py`(整改并发回归:内存 + 真实 SQLite)
## 1. 目标与结论
@@ -189,6 +198,8 @@ ChatDev 适合参考的源码路径见第 15 节。
画布只保存 `data_source_id`。前端永远拿不到数据库密码,运行事件也不能回传连接串或认证头。
HTTP 接口资源使用同一张表:kind=http 时,base_url 和 allowed_methods 是可展示的非密钥元数据;请求头仍加密放在 credential blob,列表、详情、资源目录和运行事件均不得回显。创建时至少提供一个允许方法,地址只接受不含账户密码、查询参数或片段的 http(s) URL。迁移:20260829_01_workflow_http_source_metadata.py。
### 5.4 iframe 短时票据
如 Coze 页面与 DeerFlow 不同域,新增短时、一次性的嵌入票据:
@@ -581,6 +592,7 @@ DELETE /api/workflows/{workflow_id} # 软删除/归档
PUT /api/workflows/{workflow_id}/draft # draft_revision 乐观锁
POST /api/workflows/{workflow_id}/validate
POST /api/workflows/{workflow_id}/publish
POST /api/workflows/{workflow_id}/copy
GET /api/workflows/{workflow_id}/versions
GET /api/workflows/{workflow_id}/versions/{version_id}
```
@@ -602,18 +614,19 @@ GET /api/workflows/resources/subworkflows
### 10.3 数据源
```text
GET /api/workflow-data-sources
POST /api/workflow-data-sources # 管理员或授权角色
GET /api/workflow-data-sources/{id}
PATCH /api/workflow-data-sources/{id}
DELETE /api/workflow-data-sources/{id}
POST /api/workflow-data-sources/{id}/test
POST /api/workflow-data-sources/{id}/introspect
POST /api/workflow-data-sources/{id}/validate-query
GET /api/workflows/data-sources
POST /api/workflows/data-sources
GET /api/workflows/data-sources/{id}
PUT /api/workflows/data-sources/{id}
DELETE /api/workflows/data-sources/{id}
POST /api/workflows/data-sources/{id}/introspect
POST /api/workflows/data-sources/{id}/validate-query
```
`introspect` 只返回白名单内 schema/table/column 元数据,不返回样例敏感数据。
HTTP 接口资源的创建体使用 kind=http、baseUrl、allowedMethods 和可选 headers;headers 是只写字段,服务端加密后不会出现在响应。资源目录 GET /api/workflows/resources/data-sources 返回前端所需的 dataSourceId、baseUrl、allowedMethods、description 等精简字段。
### 10.4 运行
```text
@@ -743,7 +756,7 @@ POST /api/workflow-embed/tickets/exchange
- SQL 只读:连接临时失败可有限重试;语法/权限错误不重试。
- HTTP:GET/HEAD 可按策略重试;POST 默认不重试,除非配置幂等键。
- 代码:资源不足或代码错误默认不重试。
- 工作流级 retry 建议从失败节点恢复;若 checkpoint 不可靠则创建新 run 并标记 `retry_of_run_id`。
- 工作流级 retry:`POST /api/workflows/runs/{run_id}/retry` 仅接受 `failed` 运行。新 run 复制已完成节点输出并从失败节点续跑;没有任何可回放 checkpoint 时则整图重跑,并标记 `retry_of_run_id`。
### 12.3 取消
@@ -1083,6 +1096,8 @@ tests/test_workflow_migrations.py
- 多 Gateway 实例、租约竞争、数据库短暂失败、SSE 大量重连压测。
- 事件保留、清理和 artifact 生命周期。
阶段 6 已落地:`WorkflowRetentionCleaner` 消费 `workflows.retention.*`;`POST /api/workflows/runs/{run_id}/retry`;SSE `Last-Event-ID`;单机租约竞争与 SSE 重连风暴测试。制品文件仍走 sandbox/artifact,不另做工作流专属文件回收。
### 阶段 7:试点与扩展(持续)
- 选择 1–2 个报告业务真实试点。
@@ -0,0 +1,674 @@
# Workflow Studio 后端审计整改方案
> 文档状态:已按 2026-08-29 决策部分实施(见「0. 整改决策记录」)
> 适用范围:`offline-backend-20260512/backend`
> 审核日期:2026-08-28(决策与实施:2026-08-29)
> 关联文档:[工作流编排与流式执行后端开发方案](./WORKFLOW_STUDIO_BACKEND_DEV_ZH.md)
> 适用对象:后端开发、测试、安全评审、运维发布
## 0. 整改决策记录(2026-08-29)
本轮整改以**功能健壮性**为唯一目标(用户决策:"咱们这次的整改主要是保证功能的健壮"),安全隔离类项目暂缓。逐项状态:
| 项目 | 状态 | 说明 |
|---|---|---|
| P1-01 人工介入恢复原子性 | ✅ 已实施 | `resume_with_payload` 单条件更新,并发恢复恰一个成功 |
| P1-02 草稿乐观锁 + 发布版本号 | ✅ 已实施 | 草稿 CAS 条件更新;`next_version_number` 计数器分配版本号 |
| P1-03 事件序号 + SSE 一致性 | ✅ 已实施 | `next_event_seq` 计数器;落库失败**丢弃不发布**(无幽灵事件) |
| P1-06 测试纳入版本控制 | ✅ 已实施 | `.gitignore` 白名单 + 新增 `tests/test_workflow_concurrency.py`(SQL 并发) |
| P2 §11.1 完整 JSON Schema 校验 | ✅ 已实施 | `schema_validation.py`(Draft202012,中文字段路径错误) |
| P0-01 数据源越权 / HTTP 凭证外泄 | ❌ 不实施 | **内网可信环境**,按 ID 共享数据源、HTTP 节点访问任意内网地址是既有使用方式,加 owner 校验 / host 白名单会直接破坏现网用法,风险由部署方接受 |
| P0-02 子工作流跨用户执行 | ❌ 不实施 | 同上:内网用户间按 ID 引用子工作流是预期用法 |
| P1-04 智能体/技能最小权限 | ⏸ 暂缓 | 安全隔离类,本轮不做(用户决策:"安全性的问题咱们暂时不用做") |
| P1-05 代码节点沙箱化 | ⏸ 暂缓 | 同上;`config.yaml` 中 `workflows.code.enabled` 保持默认 `false`,代码节点不开放 |
| P2 §11.2 iframe Redis nonce | ⏸ 暂缓 | 单实例部署,进程内 replay cache + 60s TTL 足够;多实例化时再引入 Redis |
已实施项的回归测试集中在 `tests/test_workflow_concurrency.py`(内存 + 真实 SQLite 双轨),外加 `tests/test_workflow_runs.py` 的路由级 schema 校验回归;全部 88 个 workflow 测试通过。
## 1. 结论与整改目标
当前 Workflow Studio 的定义、发布、运行、SSE、SQL/HTTP、智能体、技能、人工介入等功能骨架已经具备,且基础测试可运行。但在共享资源授权、凭证使用边界、并发一致性、事件可靠性和代码执行隔离方面存在上线阻断问题。
本次整改的目标不是继续扩展节点功能,而是让现有节点满足以下底线:
1. 一个用户不能通过资源 ID、流程 JSON 或子工作流引用使用别人的私有资源。
2. HTTP 凭证只能发送至管理员明确批准的目标域名,不能被流程编辑者导出到任意 URL。
3. 人工介入恢复、草稿保存、发布版本和事件序号在多请求、多实例条件下保持一致。
4. SSE 中显示的业务事件必须能够从数据库重放;不能出现只在内存短暂存在的“幽灵事件”。
5. 智能体和技能节点只能使用流程配置所授予的能力。
6. 代码节点只有在隔离沙箱中才能开启,不能在 Gateway 宿主机执行用户代码。
在 P0 项全部完成前,生产环境不得向普通用户开放:共享 SQL 数据源、共享 HTTP 凭证、子工作流跨用户引用和工作流编辑/执行权限。
## 2. 总体整改策略
### 2.1 建立运行时资源授权边界
当前 API 层可以识别当前用户,但执行器通常只持有 `source_id`、`workflow_id` 或 `agent_id`。整改后必须将“谁在执行”作为运行时一等参数,从创建运行记录开始贯穿到每一个节点。
新增领域对象,建议放在 `packages/harness/deerflow/workflows/`:
```python
from dataclasses import dataclass
@dataclass(frozen=True, slots=True)
class WorkflowExecutionPrincipal:
"""工作流运行时的可信执行身份,由 Gateway 创建,不接受前端传入。"""
user_id: str
is_admin: bool = False
tenant_id: str | None = None
```
运行创建时将发起人固化在 `workflow_runs.owner_id`,恢复、重试和子工作流也沿用原始运行的 `owner_id`,不得使用前端请求中的任意 `user_id`。
```python
principal = WorkflowExecutionPrincipal(
user_id=run.owner_id,
is_admin=await permission_service.is_admin(run.owner_id),
tenant_id=run.tenant_id,
)
```
所有需要使用资源的运行时入口必须接收 `principal`:
```python
data_source_store.resolve_for_execution(source_id, principal)
workflow_store.get_published_for_execution(workflow_id, version_id, principal)
agent_store.get_visible(agent_id, principal.user_id)
skill_registry.resolve_allowed(skill_id, principal, agent)
```
资源不存在、无权限、已禁用三种情况对普通用户应统一返回 `RESOURCE_NOT_AVAILABLE`,避免通过 UUID 枚举资源。
### 2.2 采用“发布校验 + 运行时校验”的双层防线
发布时校验是为了尽早提示编辑者;运行时校验是为了防止发布后授权被收回、资源被禁用、恶意请求绕过 UI 或历史版本继续引用资源。
| 阶段 | 应做的事 |
|---|---|
| 保存草稿 | 校验节点结构、字段格式,不因暂时无权限而破坏用户草稿 |
| 发布版本 | 校验每个外部资源对发布者是否可用,并将资源依赖写入版本元数据 |
| 创建运行 | 再次校验工作流版本和根工作流权限 |
| 节点执行 | 每次解析数据源、智能体、技能、子工作流时按 `principal` 校验 |
| 重试/恢复 | 使用原 run 的身份和已发布版本,不重新信任浏览器传参 |
## 3. P0-01:数据源越权与 HTTP 凭证外泄
> **决策(2026-08-29):不实施。** 本部署运行于内网可信环境,用户之间按数据源 ID 共享、HTTP 节点访问任意内网 URL 是既有且必需的使用方式;owner 校验、baseUrl+path 拆分、host 白名单会使现网流程全部失效。风险由部署方明确接受("这个改了我们就没法用了")。
### 3.1 问题说明
当前运行时在 `app/gateway/workflow_executor.py` 中按数据源 ID 直接解析 DSN/凭证;持久化层的 `resolve_dsn` 也没有传入 owner 或授权上下文。HTTP 节点可由流程 JSON 提供任意 URL,再把已解析的认证头附加到该 URL。
因此存在两类风险:
1. 知道或获得他人私有数据源 ID 的用户,可能在自己的流程中引用该数据源。
2. 可使用共享 HTTP 凭证的用户,可能把请求 URL 写为攻击者控制的公网域名,从而收到服务端附带的 API Key、Bearer Token 或 Basic Auth。
### 3.2 数据模型调整
在 `workflow_data_sources` 中增加显式共享与用途约束。字段命名可按现有 ORM 风格调整:
```sql
ALTER TABLE workflow_data_sources
ADD COLUMN visibility VARCHAR(16) NOT NULL DEFAULT 'private',
ADD COLUMN allowed_user_ids JSON NULL,
ADD COLUMN allowed_tenant_ids JSON NULL,
ADD COLUMN allowed_hosts JSON NULL,
ADD COLUMN allowed_methods JSON NULL;
```
字段语义:
| 字段 | 含义 |
|---|---|
| `visibility` | `private`、`shared`、`system` |
| `allowed_user_ids` | 共享给指定用户;第一期可先支持此项 |
| `allowed_tenant_ids` | 多租户部署时按租户共享;单租户可暂不启用 |
| `allowed_hosts` | HTTP 数据源允许请求的 hostname 白名单,必须填写 |
| `allowed_methods` | HTTP 数据源可执行的方法白名单,例如 `GET`、`POST` |
SQL 数据源的数据库账号仍必须由 DBA 创建为只读账号;应用层 SQL 解析和只读 SQL policy 是第二道防线,不能替代数据库权限。
### 3.3 Store 接口调整
将现有无身份的解析接口废弃:
```python
# 禁止继续在节点执行路径使用
resolve_dsn(source_id: str)
```
替换为:
```python
async def resolve_for_execution(
self,
source_id: str,
principal: WorkflowExecutionPrincipal,
expected_kind: Literal["sql", "http"],
) -> WorkflowDataSource:
...
```
实现规则:
1. 资源必须存在且 `enabled=True`。
2. `kind` 必须与节点类型匹配。
3. 允许 owner、被显式共享的用户、同租户授权用户或管理员访问。
4. 禁止仅凭“已发布工作流”绕过资源权限;每一次运行均需检查。
5. 无权限统一抛 `ResourceNotAvailableError`,日志中可记录真实原因和 source ID。
示例伪代码:
```python
def can_execute(source: WorkflowDataSource, principal: WorkflowExecutionPrincipal) -> bool:
if principal.is_admin:
return True
if source.owner_id == principal.user_id:
return True
if source.visibility == "shared" and principal.user_id in source.allowed_user_ids:
return True
if source.visibility == "system" and principal.tenant_id in source.allowed_tenant_ids:
return True
return False
```
需要同步修改:
- `packages/harness/deerflow/persistence/workflow_data_sources/store.py`
- `packages/harness/deerflow/persistence/workflow_data_sources/sql.py`
- 内存 store 实现与测试 fake
- `app/gateway/workflow_executor.py`
- 使用数据源的 SQL / HTTP 节点执行器
### 3.4 HTTP 节点配置改造
不要继续允许流程节点同时填写任意完整 URL 与 `credentialRef`。推荐将 HTTP 数据源定义为受控目标,节点只填写相对路径:
```json
{
"type": "http",
"dataSourceId": "crm-production-api",
"method": "GET",
"path": "/v1/customers",
"query": {
"page": "{{ inputs.page }}"
}
}
```
数据源的机密配置中保存:
```json
{
"baseUrl": "https://api.company.com",
"allowedHosts": ["api.company.com"],
"allowedMethods": ["GET", "POST"],
"auth": {
"type": "bearer",
"secret": "加密存储"
}
}
```
执行顺序必须固定:
1. 读取经授权的数据源。
2. 变量渲染 `path`、query、body。
3. 由 `baseUrl + path` 构造最终 URL,拒绝绝对 URL 和 `//host/path` 形式。
4. 校验最终 host、端口、DNS 解析结果、方法、请求体上限。
5. 最后才注入认证头。
6. 每次重定向重新执行 host/SSRF 校验。
7. 一旦发生跨 origin 重定向,移除 `Authorization`、Cookie、`X-API-Key` 等敏感头;建议默认拒绝跨 origin 重定向。
若业务确实需要任意 URL 调用,应提供另一种**无凭证 HTTP 节点**,并且其网络范围仍受全局 SSRF policy 约束。
### 3.5 验收测试
- 用户 A 创建私有 SQL 数据源;用户 B 使用其 ID 运行,返回 `RESOURCE_NOT_AVAILABLE`。
- 用户 A 发布引用私有数据源的流程后,用户 B 即使复制 JSON 也不能运行。
- 共享 HTTP 数据源只能请求 `allowed_hosts` 中的地址。
- URL 模板渲染为不在白名单中的主机时被拒绝,且认证头没有发送。
- 302/307 跨域跳转时凭证不会跟随;推荐断言请求整体被拒绝。
- SQL 节点使用只读账号执行 `INSERT`、`CALL`、多语句均被数据库和 SQL policy 双重拒绝。
## 4. P0-02:子工作流跨用户执行
> **决策(2026-08-29):不实施。** 内网可信环境下按 workflow/version ID 跨用户引用子工作流是预期用法,运行时沿用原 run 的 `owner_id` 已满足功能正确性;完整 ACL 见 P0-01 同批决策,一并接受风险。
### 4.1 工作流 ACL
在 workflow 主表增加与数据源一致的共享模型:
```sql
ALTER TABLE workflows
ADD COLUMN visibility VARCHAR(16) NOT NULL DEFAULT 'private',
ADD COLUMN allowed_user_ids JSON NULL,
ADD COLUMN allowed_tenant_ids JSON NULL;
```
建议第一期仅实现:`private`、`shared`、`system`。不要把“已发布”当作“所有用户可执行”;发布只代表版本不可变并可被授权对象使用。
### 4.2 接口改动
以下接口均应由 `principal` 过滤:
| 接口能力 | 整改要求 |
|---|---|
| 已发布工作流目录 | 仅返回当前用户可执行的工作流 |
| 获取版本详情 | owner / 被分享对象 / 管理员才可读 |
| 创建根运行 | 检查 workflow 与 version 的 ACL |
| 子工作流节点 | 按父 run 的 principal 再次检查子流程 ACL |
| 复制流程 | 允许复制公开定义不等于允许继承数据源和凭证引用 |
子工作流执行应改为:
```python
child_version = await workflow_store.get_published_for_execution(
workflow_id=node.workflow_id,
version_id=node.version_id,
principal=context.principal,
)
if child_version is None:
raise ResourceNotAvailableError(node.workflow_id)
```
并在 child run 写入:
```python
parent_run_id=parent_run.id
owner_id=parent_run.owner_id
tenant_id=parent_run.tenant_id
```
### 4.3 验收测试
- 用户 B 的资源目录不包含用户 A 的私有已发布流程。
- 用户 B 手工构造 A 的 workflow/version ID 创建运行,被拒绝。
- 用户 B 的父流程引用 A 的子流程,被拒绝。
- A 共享子流程给 B 后,B 可运行;收回授权后,历史草稿仍可保存但新运行失败。
## 5. P1-01:人工介入恢复的原子性
> **状态(2026-08-29):已实施。** Store 新增 `resume_with_payload(run_id, resume_token, payload, *, context=None)`(SQL/memory 双实现),暂停侧 `set_awaiting_input` 扩展 `context` 参数一次性写入 checkpoint。路由 `resume_run` 与执行器 `_pause` 均已切换到单条条件更新;`consume_resume_token` 仅为兼容保留。回归测试 `tests/test_workflow_concurrency.py`(内存 + SQLite 并发恢复、错 token、单 winner)。
### 5.1 问题与目标
当前恢复操作先写 `resume_payload`,再消费 token。两个并发请求可能先后覆盖 payload,最终执行的数据与收到成功响应的请求不一致。
目标是把“校验 token、写入输入、清除 token、变更状态”为单条条件更新。
### 5.2 Store 方法
新增方法,替代 `update_run(...) + consume_resume_token(...)` 组合调用:
```python
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
) -> WorkflowRun | None:
"""成功时返回已恢复的 run,token 无效或已消费时返回 None。"""
```
SQL 示例,JSON 写法按当前数据库方言封装:
```sql
UPDATE workflow_runs
SET
context_json = :new_context_json,
resume_token = NULL,
pending_human_node_id = NULL,
status = 'queued',
updated_at = NOW()
WHERE id = :run_id
AND status = 'awaiting_input'
AND resume_token = :resume_token;
```
应用层规则:
1. `rowcount != 1` 时返回 409,且不得投递调度器。
2. `rowcount == 1` 后再投递调度器。
3. 投递失败时保留 `queued` 状态,由扫描器补投;不要把 token 恢复。
4. 对人工输入 payload 先完成 schema 校验、大小限制和脱敏审计,再进入事务。
暂停节点也应使用单次 store 调用,同时写入:状态、token、等待节点 ID 和上下文,避免执行器后续的独立 `update_run` 覆盖已恢复的数据。
### 5.3 验收测试
- 两个协程并发用同一 token 提交不同 payload,只有一个 200,另一个 409。
- run context 中只存在 200 对应 payload。
- worker 最终读取的 payload 与成功响应一致。
- token 被消费后,多次重试均为 409。
## 6. P1-02:草稿与发布版本并发控制
> **状态(2026-08-29):已实施。** 草稿保存改为 `UPDATE ... WHERE id AND draft_revision = :expected`,rowcount=0 时回读当前 revision 抛 `WorkflowDraftConflictError`(409 带 `currentRevision`)。发布版本号改由 `workflow_definitions.next_version_number` 计数器 CAS 分配(`max(counter, MAX(version)+1)` 兼容存量行),唯一约束兜底、冲突重试而非 500。计数器列由迁移 `20260828_04_workflow_counter_columns.py` 增加(server_default,存量库由启动列同步自动补列)。回归测试:并发草稿双写单 winner、5 并发发布得相邻版本号、存量版本行计数器自愈。
### 6.1 草稿乐观锁
禁止“先读取 revision,再修改 ORM 对象,再 commit”的方式。改为带 revision 的条件更新:
```sql
UPDATE workflows
SET graph_json = :graph_json,
draft_revision = draft_revision + 1,
updated_at = NOW()
WHERE id = :workflow_id
AND draft_revision = :expected_revision;
```
若受影响行数为零,则读取当前 revision,并返回:
```json
{
"code": "WORKFLOW_DRAFT_CONFLICT",
"currentRevision": 18
}
```
前端收到 409 后应提示“流程已被其他编辑者保存”,支持刷新、比较或另存副本;不能静默覆盖。
### 6.2 发布版本号
禁止 `MAX(version_number) + 1`。建议在 workflow 行维护 `next_version_number`,在一个事务内锁定并递增:
```sql
SELECT next_version_number FROM workflows WHERE id = :id FOR UPDATE;
INSERT INTO workflow_versions (..., version_number) VALUES (..., :next_number);
UPDATE workflows
SET next_version_number = next_version_number + 1
WHERE id = :id;
```
若现有数据库不方便 `FOR UPDATE`,可采用具备唯一约束的 sequence/counter 表,捕获唯一冲突后重试;不要将冲突直接抛为 500。
### 6.3 验收测试
- 两个独立 SQLAlchemy session 同时保存同一 revision,严格只有一个成功。
- 两个发布请求同时执行,生成相邻但不同的版本号。
- 冲突被转换为稳定的 409 业务错误,而非数据库异常。
## 7. P1-03:事件持久化、序号和 SSE 一致性
> **状态(2026-08-29):已实施(采用 7.2 计数器方案 + 7.3 的"丢弃不发布"策略)。** `workflow_runs.next_event_seq` 计数器在同一事务内 `UPDATE ... SET next_event_seq = next_event_seq + 1` 后读自增并插入事件,行写锁串行化并发追加(seq 严格递增、唯一、无空洞);唯一约束冲突时 `_resync_counter` 以独立事务回写 `MAX(seq)` 自愈存量行后重试。落库失败的事件**直接丢弃、绝不向 live hub 发布**(原负序号幽灵事件实现已删除),信封带 `eventPersistenceDegraded` 标记、`sink.degraded_count` 计数并记结构化告警日志;SSE 端点对无终态事件的终态 run 合成关闭帧。回归测试:10 并发追加 seq 连续唯一、存量事件计数器自愈、落库失败零发布。
### 7.1 改造原则
业务事件必须遵循:**获得持久序号 → 持久化成功 → 实时发布**。不能生成负序号临时事件,也不能让前端看到无法通过重连补回的事件。
### 7.2 序号分配
不要使用 `MAX(seq) + 1`。在 `workflow_runs` 增加计数器:
```sql
ALTER TABLE workflow_runs
ADD COLUMN next_event_seq BIGINT NOT NULL DEFAULT 1;
```
追加事件时在同一个数据库事务中:
1. 对 run 原子递增 `next_event_seq` 并得到当前 seq。
2. 以该 seq 插入 `workflow_events`。
3. 提交事务。
4. 仅提交成功后调用 `live_hub.publish(event)`。
不同数据库可选择 `UPDATE ... RETURNING` 或者用 run 行锁实现,但不可回退为 `MAX(seq)` 查询。
### 7.3 持久化失败处理
事件库短暂失败时:
1. 在有限次数内重试;
2. 仍失败则将运行标记为 `retryable_failure` 或 `paused_system`;
3. 记录包含 `run_id`、node ID、异常分类的结构化日志和告警;
4. SSE 至少发送一个已成功持久化的终态或系统状态事件;
5. 用户重连后只能从数据库事件日志恢复,不依赖当前进程内存。
不要使用“事件落库失败后仍向 live hub 发负数 seq”的实现,因为按 cursor 追尾的 SSE 客户端通常会忽略它。
### 7.4 验收测试
- 并行节点大量写事件,序号严格递增且无重复。
- 模拟一次插入事件失败,前端不会收到不可回放事件。
- 客户端从 `Last-Event-ID=N` 重连,得到所有 `seq>N` 的事件且顺序一致。
- 多实例下任一实例订阅 SSE,均可通过数据库补放完整事件。
## 8. P1-04:智能体和技能最小权限
> **决策(2026-08-29):暂缓。** 属安全隔离类项目,本轮整改只做功能性优化(用户决策:"P1-04/P1-05 偏安全隔离不用做,安全性的问题咱们暂时不用做,咱们只改功能性的,优化的")。后续若对外开放多租户再评估。
### 8.1 智能体可见性
`agent` 节点收到 `agent_id` 后,必须在 `WorkflowAgentRunner` 内部再次调用可见性查询:
```python
agent = await agent_store.get_visible(
agent_id=node.agent_id,
user_id=context.principal.user_id,
)
if agent is None or not agent.enabled:
raise ResourceNotAvailableError(node.agent_id)
```
不能仅依赖资源目录接口已过滤,也不能仅依赖发布阶段校验。
### 8.2 技能白名单
现有将 `skill_names` 传给 Lead Agent 的方式必须核实是否被真正消费。建议新增语义明确的运行时参数:
```python
configurable["workflow_allowed_skill_ids"] = selected_skill_ids
```
并在 Lead Agent 加载技能、MCP 工具和工具调用路由之前强制过滤:
```python
requested = set(configurable.get("workflow_allowed_skill_ids", []))
available = [skill for skill in all_enabled_skills if skill.id in requested]
```
还应校验:
1. 选中的 skill 对当前用户可见。
2. skill 是否允许在 workflow 场景执行。
3. skill 的网络、数据库、文件权限是否与工作流 principal 相兼容。
4. 系统级高风险 skill 是否只能由管理员在数据源/策略中启用。
### 8.3 验收测试
- 用户手工填写不可见 agent ID,运行被拒绝。
- 未选中的技能不出现在模型可调用工具列表中。
- 提示词诱导模型调用未选技能,调用被拒绝且记录审计事件。
- 选中的技能正常可执行,节点事件中仅暴露必要的脱敏参数。
## 9. P1-05:代码节点沙箱化
> **决策(2026-08-29):暂缓,保持现状即安全。** 本轮不迁移到容器沙箱;`config.yaml` 中 `workflows.code.enabled` 保持默认 `false`(代码节点全局关闭,`tests/test_workflow_engine.py::test_code_node_disabled_by_default` 已锁定该默认值),如需启用由运维在受限宿主上自行开启。现有 `code_runner.py` 子进程加固(净化环境、临时目录、rlimit、断网、超时杀进程组)在可信内网下作为过渡方案。
### 9.1 上线策略
在 DeerFlow 沙箱实现并通过安全验证前,保持:
```yaml
workflows:
code:
enabled: false
```
不得以本地 `exec`、`subprocess` 或 monkey-patch `socket` 作为生产隔离方案。用户代码可借助原生扩展、子进程、文件系统或环境变量绕过此类限制。
### 9.2 目标接口
代码节点改为只调用现有 DeerFlow sandbox 抽象,而不自行启动宿主解释器:
```python
result = await sandbox.run(
code=rendered_code,
language="python",
timeout_seconds=30,
memory_limit_mb=256,
cpu_limit=1,
max_processes=8,
network_enabled=False,
writable_paths=["/mnt/user-data/workflows/{run_id}"],
)
```
沙箱最低要求:
- 独立容器或受限命名空间,不能读取 Gateway 项目、环境变量与宿主文件。
- 默认无网络;若未来允许网络,也需复用 HTTP 节点的数据源、SSRF 和凭证策略。
- CPU、内存、执行时长、磁盘、进程数和文件数限制。
- stdout/stderr 使用流式读取并在读取中截断,例如各 1MB;禁止 `communicate()` 后再截断。
- 制品只能输出到工作流专属目录,并通过既有 artifact 机制登记。
- 记录代码 hash、镜像版本、资源消耗、退出码,日志不得泄露源代码中的密钥。
### 9.3 验收测试
- 代码读取 `/etc`、项目目录、环境变量失败。
- 导入 socket、创建子进程、连接外网失败。
- 死循环超时结束,运行正确失败或可重试。
- 大量 stdout/stderr 不造成 Gateway 内存膨胀。
- 制品只能写入限定目录。
## 10. P1-06:测试纳入版本控制与 CI
> **状态(2026-08-29):已实施。** 采用方案 1(白名单):仓库根 `.gitignore` 对 `tests/test_workflow_{schemas,definitions,engine,runs}.py` 逐个 `!` 白名单,并新增 `tests/test_workflow_concurrency.py`(本轮整改的内存 + 真实 SQLite 并发回归,含 P1-01/02/03 与 schema 校验)。CI 需在该清单基础上加 `tests/test_workflow_concurrency.py`。
当前工作流测试文件位于后端 `tests/`,但仓库根目录的 `.gitignore` 规则会忽略这一目录内容。必须在合并前处理,否则本地通过的测试不会进入提交和 CI。
推荐方案:
1. 调整 `.gitignore`,仅忽略真正的生成物,不要忽略后端测试源码;或
2. 若忽略规则短期不能改,使用 `git add -f` 纳入现有测试文件,但仍应补充注释说明原因;
3. CI 显式执行:
```bash
cd offline-backend-20260512/backend
PYTHONPATH=. uv run pytest \
tests/test_workflow_schemas.py \
tests/test_workflow_definitions.py \
tests/test_workflow_engine.py \
tests/test_workflow_runs.py -v
```
4. 新增 SQL 集成测试 job,至少覆盖真实事务并发;不能只依赖 memory store。
## 11. P2:契约校验与 iframe 票据
> **状态(2026-08-29):§11.1 已实施;§11.2 暂缓**(单实例部署,进程内 replay cache + 60s TTL 足够,多实例化时再引入 Redis)。
### 11.1 完整 JSON Schema 校验
> **实施说明(2026-08-29)**:新增纯函数模块 `packages/harness/deerflow/workflows/schema_validation.py` 的 `schema_issues(value, schema)`(Draft202012,中文短消息、字段路径、去重、上限 20 条、schema 本身非法时返回友好错误而非 500)。接入点:路由 `start_run`(启动前 400 `WORKFLOW_INPUT_INVALID` + `details.errors`)、引擎 Start/Output 节点(`WORKFLOW_INPUT_INVALID` / `WORKFLOW_OUTPUT_SCHEMA_MISMATCH`)。空 schema 不强制(未声明即不校验)。回归测试见 `tests/test_workflow_concurrency.py` 与 `tests/test_workflow_runs.py::test_typed_input_schema_violations_return_structured_400`。
输入节点和输出节点不应只校验 `required` 字段。应使用完整 JSON Schema validator,至少支持:`type`、`enum`、`items`、`properties`、`required`、`additionalProperties`、嵌套对象和数组。
```python
from jsonschema import Draft202012Validator
validator = Draft202012Validator(schema)
errors = sorted(validator.iter_errors(value), key=lambda item: list(item.path))
if errors:
raise WorkflowSchemaValidationError(
errors=[{"path": list(err.path), "message": err.message} for err in errors]
)
```
错误响应需要包含节点 ID、字段路径和可展示的消息;不应把内部 Python trace 返回给前端。
### 11.2 iframe 一次性票据
“一次性”语义不能使用单进程内存字典。若部署多实例,应使用 Redis 或数据库的原子写入:
```text
SET workflow_embed_nonce:{jti} consumed NX EX 60
```
只有返回成功的实例可以继续验证。服务端还必须读取真实请求 `Origin`,并要求以下三者一致:
1. ticket 签发时绑定的 origin;
2. 浏览器请求头中的 `Origin`;
3. 后台 `workflows.embed.allowed_origins` 白名单。
不能只比较前端 body/query 参数中的 origin。
## 12. 建议实施顺序与分批发布
> 2026-08-29 实况:第二批全部落地;第四批仅 §11.1 落地(§11.2 暂缓);第一批(P0-01/02)经部署方决策不实施;第三批(P1-04/05)暂缓。
### 第一批:安全阻断项
范围:资源 principal、数据源授权、HTTP host 绑定、子工作流 ACL。
发布策略:先迁移数据库字段,默认所有历史资源为 `private`;只允许管理员手工将必要资源改为 shared。此批完成并通过越权测试后,才允许开放 SQL/HTTP/子工作流节点。
### 第二批:一致性与可靠性
范围:人工恢复原子操作、草稿 CAS、发布版本锁、事件序号与持久化失败策略。
发布策略:需要停写窗口或兼容性迁移时,先双写/双读一版;对运行中的旧 run 保持旧 schema 可读,新增 run 使用新序号和新恢复逻辑。
### 第三批:能力隔离
范围:agent 可见性、skill 白名单、代码节点沙箱。
发布策略:代码节点继续全局关闭;先灰度给管理员和测试租户。技能节点在完成真实白名单前,不得宣称“只调用所选技能”。
### 第四批:契约与嵌入完善
范围:JSON Schema 完整校验、iframe Redis nonce、接口错误码统一、运维监控。
## 13. 发布准入清单
> 2026-08-29 更新:本清单按原始审计口径保留。结合「0. 整改决策记录」,P0 两项经部署方决策**不实施**(内网可信环境,风险已接受),与 P0 相关的准入项在内网部署中视为豁免;✅ 为本轮已达成项。
以下条件全部满足才可开放给普通用户:
- [ ] 所有资源解析接口都需要 `WorkflowExecutionPrincipal`。(❌ 不实施——内网可信环境,见 §3/§4 决策)
- [ ] 私有数据源、私有工作流、私有智能体均有跨用户拒绝测试。(❌ 不实施——同上)
- [ ] HTTP 凭证绑定 host/method,跨域重定向不携带敏感认证头。(❌ 不实施——内网需访问任意地址,见 §3 决策)
- [ ] SQL 数据源使用数据库只读账号,且连接/查询审计可追踪到 run ID。(部分:SQL 只读校验已有;账号隔离不在本轮范围)
- [ ] 子工作流按运行身份校验 ACL。(❌ 不实施——见 §4 决策)
- [x] 人工输入恢复是单事务条件更新,并发测试通过。(✅ `test_workflow_concurrency.py`)
- [x] 草稿保存和发布版本具备数据库级并发保护。(✅ 同上)
- [x] 事件序号单调、唯一、可重放;没有负序号临时事件。(✅ 同上)
- [ ] agent / skill 白名单在实际工具加载链路生效。(⏸ 暂缓——P1-04)
- [x] 代码节点运行在受限 sandbox;否则保持关闭。(✅ 保持关闭:`workflows.code.enabled: false` 为默认且被测试锁定)
- [x] 所有 workflow 测试文件已纳入 Git 与 CI。(✅ `.gitignore` 白名单 + 新增并发回归)
- [x] 安全测试、SQL 并发集成测试、SSE 重连测试全部通过。(✅ 88 个 workflow 测试全绿;安全类测试随 P0/P1-04 决策豁免)
## 14. 相关代码入口
| 主题 | 当前主要代码入口 |
|---|---|
| 路由与运行 API | `app/gateway/routers/workflow_runs.py` |
| 工作流执行与依赖装配 | `app/gateway/workflow_executor.py` |
| 智能体/技能运行 | `app/gateway/workflow_agent_runner.py` |
| 数据源 API | `app/gateway/routers/workflow_data_sources.py` |
| 工作流资源目录 | `app/gateway/routers/workflow_resources.py` |
| iframe ticket | `app/gateway/routers/workflow_embed.py` |
| 数据源持久化 | `packages/harness/deerflow/persistence/workflow_data_sources/` |
| 工作流/版本持久化 | `packages/harness/deerflow/persistence/workflows/` |
| 运行与租约持久化 | `packages/harness/deerflow/persistence/workflow_runs/` |
| 事件持久化 | `packages/harness/deerflow/persistence/workflow_events/` |
| SQL/HTTP 节点 | `packages/harness/deerflow/workflows/nodes/integrations.py` |
| 事件 sink | `packages/harness/deerflow/workflows/runtime/sink.py` |
| 代码节点运行器 | `packages/harness/deerflow/workflows/runtime/code_runner.py` |
| 工作流安全策略 | `packages/harness/deerflow/workflows/security/` |
| 数据库迁移 | `packages/harness/deerflow/persistence/migrations/` |
| 现有工作流测试 | `tests/test_workflow_*.py` |
## 15. 非目标与注意事项
- 本整改不改变 Coze Studio 画布的基础交互,也不要求先改前端;前端仅需后续适配资源权限失败、409 冲突和结构化 schema 错误。
- 不要把资源密钥、数据库 DSN、人工输入 token 或工具原始参数写入前端 SSE 事件。
- 不要依赖“前端资源选择器只显示可见资源”作为授权机制;浏览器请求和画布 JSON 均视为不可信输入。
- 新字段上线后,历史资源默认按私有处理,避免一次迁移导致所有历史凭证意外共享。
- 每项整改完成后都要更新 `WORKFLOW_STUDIO_BACKEND_DEV_ZH.md` 中对应的“已完成”状态,避免文档声明与实际安全边界不一致。
@@ -0,0 +1,486 @@
# Workflow Studio:Coze 独立前端对接 DeerFlow 后端适配开发文档
> 状态:待实施
> 日期:2026-08-29
> 前端范围:F:/react01/code-stutsiu/frontend/apps/coze-studio/src/workflow-standalone/
> 后端范围:offline-backend-20260512/backend/
> 关联文档:
>
> - docs/WORKFLOW_STUDIO_BACKEND_DEV_ZH.md:工作流运行时总体设计与已有能力。
> - docs/WORKFLOW_STUDIO_BACKEND_REMEDIATION_ZH.md:安全、并发、租约与事件持久化整改。
>
> 本文只处理“当前独立 Coze 页面与真实 DeerFlow 后端之间”的适配,不重新设计工作流运行内核。
## 1. 目标与结论
当前独立页的交互、三栏布局、资源卡、运行历史、SSE reducer 与 mock 演示已可用,但其保存、校验、发布和部分运行事件仍使用 mock 协议。真实 DeerFlow 后端采用 WorkflowGraph v1.0 和持久化运行时,两者不能直接互换。
本次适配目标:
1. Coze 画布能够保存、重新加载、校验、发布,并由 DeerFlow 运行。
2. 画布上的 Agent、Skill、SQL、HTTP、Code、Input、Output 节点都有真实、可审计的执行配置。
3. 左侧对话运行和右侧执行过程能从同一条 DeerFlow SSE 事件流稳定恢复工具步骤、人工介入与文件制品。
4. 不把 Coze 的内部数值节点类型、任意画布 JSON、凭据明文或完整工具参数泄漏到运行时协议中。
结论:不能仅在现有 POST /api/workflows/{id}/runs 中增加几个兼容字段。必须先建立“画布表示”和“执行图表示”的明确边界;否则保存成功的流程仍无法发布或运行。
## 2. 当前差异清单
| 编号 | 当前独立页行为 | DeerFlow 当前行为 | 影响 | 所属 |
| --- | --- | --- | --- | --- |
| A01 | 保存 body 为 schemaJson,读取 workflowId / draftRevision / schemaJson | 草稿接口要求 graph,读取返回 id / draft_graph / draft_revision | 自动保存、冲突恢复、发布均无法真实联调 | 前端主改,后端提供工作室 DTO |
| A02 | 校验期待 valid 字段 | 后端返回 ok 字段 | 前端始终判定校验不通过 | 前后端契约统一 |
| A03 | Coze JSON 被原样持久化 | 后端运行时只接受 WorkflowGraph v1.0 | Coze 数值节点类型不可执行 | 前端编译,后端持久化与验证 |
| A04 | 拖入资源只预填节点标题 | Agent / Skill 执行器要求 agentId、skillNames、promptTemplate 等 | 资源卡看起来可用,真实运行必失败 | 前端表单 + 后端发布校验 |
| A05 | 工具事件读取 name、status | 后端发送 toolName、ok | 工具步骤丢失,失败状态错误 | 后端规范化,前端兼容旧字段 |
| A06 | 人工介入读取顶层 prompt / formSchema | 后端放在 pendingInput 内 | 介入卡没有问题文本和表单 | 后端事件扁平化,前端兼容 |
| A07 | 文件预览请求单个 artifact 内容接口 | 后端只提供制品列表 | 文件卡可见但无法打开或下载 | 后端新增制品内容接口 |
| A08 | 资源 hook 请求 credentials 目录 | 后端没有该路由 | 整个资源加载 Promise.all 可能显示失败 | 后端补只读引用目录,或前端暂停请求 |
| A09 | UI 可选“当前草稿试运行”和 mock 场景 | 后端忽略 target / scenario,只运行发布版本 | 用户以为执行草稿,实际执行旧版本 | 前端先隐藏;后端后续可实现草稿快照 |
| A10 | 连续输入会创建新的 run | 后端无 conversation / turn 模型 | 不是 ChatDev 式多轮会话 | 二期可选能力 |
## 3. 必须冻结的总体设计
### 3.1 两种表示,两个职责
同一工作流必须保留两份有明确用途的数据,不能让任何一份冒充另一份。
| 数据 | 责任方 | 用途 | 是否可直接执行 |
| --- | --- | --- | --- |
| canvasSchemaJson | Coze 前端产生;后端仅作为受限 JSON 文档保存 | 画布恢复、节点位置、Coze 表单内部字段、编辑体验 | 否 |
| executionGraph | 前端适配器编译;后端以 WorkflowGraph 校验并版本化 | 发布校验、权限检查、调度、恢复、SSE、审计 | 是 |
约束:
1. DeerFlow harness 只接收 executionGraph,绝不解析 Coze StandardNodeType 或 Coze 内部服务字段。
2. canvasSchemaJson 不进入 SSE,不进入运行快照,不作为表达式或节点配置来源。
3. 发布版本至少冻结 executionGraph;建议同时保存其来源 canvasSchemaJson 的哈希,便于审计与排错。
4. Coze 到 WorkflowGraph 的逐节点转换放在前端 adapters/coze-canvas-adapter.ts 或其同级编译器中。后端不耦合 Coze 的数值节点类型。
5. 后端应拒绝“只有 canvasSchemaJson、没有合法 executionGraph”的发布请求,不能静默将其当作空图。
### 3.2 推荐的工作室适配层
保持现有 /api/workflows 的标准运行时 API 语义,不在其核心请求中混入 Coze 专属字段。新增一层仅面向独立页的 studio facade:
GET /api/workflows/{workflow_id}/studio-document
PUT /api/workflows/{workflow_id}/studio-draft
POST /api/workflows/{workflow_id}/studio-validate
POST /api/workflows/{workflow_id}/studio-publish
现有以下端点继续作为运行时和其他客户端的标准接口:
POST /api/workflows/{workflow_id}/runs
GET /api/workflows/{workflow_id}/runs
GET /api/workflows/runs/{run_id}/events
GET /api/workflows/runs/{run_id}/stream
这样既避免污染通用 API,也让前端能获得它需要的 camelCase DTO。
## 4. Studio facade API 契约
### 4.1 获取文档
请求:
GET /api/workflows/{workflow_id}/studio-document
响应:
{
"workflowId": "wf_001",
"name": "周报生成",
"draftRevision": 12,
"publishedVersionId": "ver_009",
"canvasSchemaJson": "{...}",
"executionGraph": { "schemaVersion": "1.0", "id": "wf_001", "nodes": [], "edges": [] },
"updatedAt": "2026-08-29T10:00:00+08:00"
}
要求:
- 从现有 definition 的 id、draft_revision、draft_graph、published_version_id 等字段适配生成。
- canvasSchemaJson 为空时返回空画布 JSON,而不是 null。
- 权限沿用现有 owner/admin 规则。
- 不返回 DSN、HTTP 请求头、resume token、完整 agent 配置或任何私密字段。
### 4.2 保存草稿
请求:
PUT /api/workflows/{workflow_id}/studio-draft
{
"expectedRevision": 12,
"canvasSchemaJson": "{...}",
"executionGraph": {
"schemaVersion": "1.0",
"id": "wf_001",
"nodes": [],
"edges": []
}
}
响应:
{ "revision": 13, "updatedAt": "2026-08-29T10:01:00+08:00" }
规则:
1. 先做 WorkflowGraph 结构解析;无法解析返回 400,错误码 WORKFLOW_SCHEMA_INVALID。
2. 画布 JSON 只做大小、JSON 格式、深度和危险字段控制,不做执行解释。
3. canvasSchemaJson 与 executionGraph 必须在同一次 CAS 更新中落库,不能出现画布和执行图不同 revision。
4. revision 冲突返回 409:
{
"code": "WORKFLOW_DRAFT_CONFLICT",
"message": "草稿版本冲突,请刷新后重试或另存副本",
"currentRevision": 13
}
5. 不建议让旧 PUT /draft 同时接受 schemaJson 与 graph。若为灰度必须暂时兼容,兼容逻辑只能位于 facade 路由,且必须在一个发布周期后删除。
### 4.3 校验与发布
校验请求:
POST /api/workflows/{workflow_id}/studio-validate
{ "executionGraph": { ... } }
校验响应:
{
"valid": false,
"issues": [
{
"code": "WORKFLOW_RESOURCE_MISSING",
"nodeId": "agent_writer",
"field": "config.agentId",
"message": "未选择可访问的智能体"
}
],
"graphHash": null
}
发布请求:
POST /api/workflows/{workflow_id}/studio-publish
{ "expectedRevision": 13, "changeNote": "配置报告智能体" }
发布响应:
{ "versionId": "ver_010", "publishedAt": "2026-08-29T10:02:00+08:00" }
发布必须复用标准发布版本表和不可变 graph 快照。facade 只负责 DTO 命名、画布关联和调用标准发布服务,不能复制一套版本逻辑。
## 5. 后端实施项
### 5.1 持久化 canvas schema 与草稿原子更新
现有 workflow_definitions 只有 draft_graph_json。新增独立字段,例如 draft_canvas_schema_json;不要把 Coze 画布塞进 draft_graph_json。
需要修改:
| 路径 | 改动 |
| --- | --- |
| packages/harness/deerflow/persistence/workflows/model.py | WorkflowDefinitionRow 增加 draft_canvas_schema_json;如需审计可增加 published_canvas_schema_hash |
| packages/harness/deerflow/persistence/migrations/versions/ | 新增 Alembic migration,存量行默认 {} |
| packages/harness/deerflow/persistence/workflows/base.py | save_studio_draft 或扩展 save_draft 的原子接口,输入两份文档和 expected_revision |
| packages/harness/deerflow/persistence/workflows/sql.py | 单条 CAS UPDATE 同时更新两份文档、revision、updated_at |
| packages/harness/deerflow/persistence/workflows/memory.py | 同步实现,保证测试环境语义一致 |
禁止:
- 先保存 canvas、后保存 graph 的两次写入。
- 在 SQL store 之外用“先读 revision、再无条件写”的方式实现 CAS。
- 把 canvasSchemaJson 输出到运行事件或 run context。
### 5.2 Studio facade 路由与 DTO
新增 app/gateway/routers/workflow_studio.py,负责:
1. studio-document、studio-draft、studio-validate、studio-publish 四个端点。
2. camelCase DTO 与现有 persistence snake_case 字段之间的映射。
3. 统一错误结构和 owner/admin 鉴权。
4. 调用通用 WorkflowGraph 结构校验与 app 层资源校验。
在 app/gateway/app.py 或当前工作流 router 注册位置挂载该 router。不要让 packages/harness/deerflow 导入 app.gateway。
前端移交项:
- api/workflows.ts 改为调用 studio facade。
- coze-canvas-adapter.ts 产出 executionGraph,不再只做 JSON.stringify。
- 画布新增或修改节点时,同时更新可编译的节点配置。
### 5.3 发布前资源和权限校验
现有 validator 支持 resource_checker 钩子,但通用 harness 不应直接导入应用层的 agent、skill、data-source store。应在 app 层构造 checker 后传入。
在 studio-validate、studio-publish 和标准 publish 入口都执行:
| 节点 | 必检字段 | 后端校验 |
| --- | --- | --- |
| agent | agentId、promptTemplate、responseSchema | 智能体存在、调用者可见、提示词非空、输出 schema 合法 |
| skill | mode、agentId、skillNames、promptTemplate | 技能存在且启用;agent_skill 必须属于智能体白名单;callable_skill 未开放时明确拒绝 |
| sql_read | dataSourceId、queryTemplate、parameters | 数据源可见、只读、表白名单、SQL 解析与 maxRows 上限 |
| http | dataSourceId 或 credentialRef、method、path、headers 映射 | 资源可见、方法白名单、URL/内网策略、凭据引用可用 |
| code | language、source、timeoutSeconds、output mapping | 沙箱已启用、语言白名单、超时和输出目录限制 |
| output | schema、field mapping | 输出字段可解析,输出 schema 可校验 |
| subworkflow | workflowId、versionId | 子工作流已发布、调用者有权限、深度限制 |
校验必须返回 nodeId 与字段路径,便于前端定位 Coze 节点表单。运行时仍保留二次校验,防止资源被禁用或权限在发布后撤销。
### 5.4 资源目录与凭据引用
保留现有:
- app/gateway/routers/workflow_resources.py:Agent、Skill、数据源目录。
- app/gateway/routers/workflow_data_sources.py:SQL/HTTP 数据源创建、加密与只读策略。
补充:
GET /api/workflows/resources/credentials
响应只允许:
{
"credentials": [
{
"credentialId": "http_source_001",
"name": "CRM 接口密钥",
"type": "http_headers",
"sourceKind": "http"
}
]
}
绝不返回 headers、token、DSN、加密密文或可逆提示。若一期 UI 没有凭据选择器,前端必须停止请求该端点,不能让一个 404 使整个资源面板显示“加载失败”。
### 5.5 运行目标、输入 schema 与草稿试运行
一期决策:
1. POST /runs 只接受发布版本或明确的 versionId。
2. 前端隐藏“当前草稿(试运行快照)”和 mock scenario 选择器。
3. 后端若收到 target=draft,返回 422 WORKFLOW_DRAFT_RUN_UNSUPPORTED,禁止静默回退到发布版本。
二期若需要草稿试运行,新增明确的 test-run API,而不是篡改正式 runs:
POST /api/workflows/{workflow_id}/test-runs
流程为:CAS 读取草稿 → 生成不可变 test snapshot → 记录 snapshot id → 运行 → 保留期清理。运行记录必须注明 test=true,不能被误认为发布版本运行。
运行输入:
- studio-document 响应必须带 inputSchema。
- 前端根据 inputSchema 渲染输入,而非永远发送 input.query。
- POST /runs 继续使用后端 JSON Schema 验证;错误返回 field/path/expected,前端在表单附近展示。
### 5.6 SSE 事件契约收敛
后端事件是前端执行过程唯一事实来源。所有事件须先持久化再广播,seq 单调递增且可 REST 重放。
规范化以下 payload:
node.tool.started
{
"name": "web_search",
"toolCallId": "call_001",
"title": "联网检索",
"args": { "path": "/mnt/user-data/outputs/report.md" }
}
node.tool.finished
{
"name": "web_search",
"toolCallId": "call_001",
"status": "succeeded",
"summary": "获得 12 条结果"
}
run.awaiting_input
{
"resumeToken": "one-time-token",
"nodeId": "review",
"prompt": "请确认是否发布",
"formSchema": { "type": "object", "properties": {} },
"actions": ["approve", "reject"]
}
artifact.created
{
"artifactId": "artifact_001",
"name": "weekly-report.docx",
"kind": "file",
"mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"sizeBytes": 1234
}
规则:
- 工具字段统一为 name;不要只发 toolName。
- 失败工具必须明确 status=failed 与脱敏 error。
- pendingInput 可以继续存储在数据库,但对外事件须扁平化为 prompt、formSchema、actions。
- agent 节点继续发送安全展示元数据 agent、parallelGroupId、roundIndex;不发送系统提示词、技能白名单、凭据和完整配置。
- 不将每个模型 token 当作 aria/live 或持久化事件;继续使用节流后的 node.output.delta。
- 事件类型表若仍不含 node.tool.input_delta,应从前端 mock 中删除该依赖,或后端正式实现该事件;不能只有 mock 有。
涉及路径:
| 路径 | 改动 |
| --- | --- |
| packages/harness/deerflow/workflows/nodes/agents.py | on_tool 输出 name、status、最小安全 args |
| app/gateway/workflow_executor.py | run.awaiting_input 对外 payload 扁平化 |
| packages/harness/deerflow/workflows/events.py | 冻结事件类型与 payload schema 样例 |
| app/gateway/routers/workflow_runs.py | REST 重放与 SSE 输出保持同一 envelope |
| packages/harness/deerflow/workflows/runtime/engine.py | 保持 node.started 的 agent / parallelGroupId / roundIndex 安全投影 |
### 5.7 制品内容、预览和下载
新增:
GET /api/workflows/runs/{run_id}/artifacts/{artifact_id}
GET /api/workflows/runs/{run_id}/artifacts/{artifact_id}/download
内容接口返回受限预览:
{
"artifactId": "artifact_001",
"name": "weekly-report.md",
"kind": "markdown",
"mimeType": "text/markdown",
"text": "# 周报",
"sizeBytes": 2048
}
策略:
1. 每次读取先校验 run owner/admin,再校验 artifact 属于该 run。
2. 文本、JSON、表格只返回大小受限预览;大文件返回 truncated=true。
3. 二进制文件不塞入 JSON;download 接口以附件或受控 inline 响应流返回。
4. 路径只能从已登记的 artifact id 解析,禁止接收任意 sandbox 路径。
5. 制品存储生命周期沿用现有 sandbox/artifact 机制;工作流表只存引用与元数据。
涉及路径:
- app/gateway/routers/workflow_runs.py:新增两个路由。
- packages/harness/deerflow/persistence/workflow_runs/*:按 run_id + artifact_id 查询的 store 方法。
- app/gateway/routers/artifacts.py:仅可参考现有 thread artifact 的鉴权和文件响应,不可直接跨 thread 复用。
### 5.8 取消状态和事件游标
统一运行状态:
| 后端状态 | 前端显示 | 说明 |
| --- | --- | --- |
| queued | 排队中 | 已入队未领取 |
| running | 运行中 | 正在执行 |
| awaiting_input | 等待人工介入 | 暂停并持有一次性 token |
| cancel_requested | 取消中 | 等待运行器确认,非终态 |
| completed / failed / cancelled | 终态 | SSE 停止重连 |
后端工作:
1. _public_run 输出 lastSeq 时映射 persistence 的 next_event_seq,不能读取不存在的 last_seq。
2. 前端 RunStatus 类型和 RunStatusBadge 增加 cancel_requested;后端不应把它伪装为 cancelled。
3. list events 空页必须返回请求 cursor,防止前端游标回退。
### 5.9 连续对话运行(二期,可选)
当前“对话运行”是单个 run 的视图投影:每次发送创建一个新 run。若业务要 ChatDev 式连续问答,需要独立设计,不能复用 run_id 作为会话 id。
建议新增:
workflow_conversations
workflow_conversation_turns
workflow_runs.conversation_id (nullable)
接口:
POST /api/workflows/{workflow_id}/conversations
GET /api/workflows/{workflow_id}/conversations
POST /api/workflows/conversations/{conversation_id}/turns
每个 turn 创建一个 run,并记录输入、run id、可展示摘要和制品引用。默认不把完整模型上下文拼接给每个节点;由 graph 明确配置哪些字段、哪些历史摘要可见。
## 6. 推荐实施顺序
### 阶段 A:真实保存与发布闭环(阻断项)
1. 增加 canvas schema 持久化迁移和 store 原子接口。
2. 实现 studio facade 四个端点。
3. 前端实现 Coze JSON 到 WorkflowGraph 编译,并改接 facade。
4. 完成输入/输出、代码、Agent、Skill、SQL、HTTP 的最小节点配置表单。
5. 加入发布前 app 层资源检查。
验收:编辑一个“Input → SQL → Agent → Output”流程,刷新后画布与配置不丢失,发布后执行的是同一版本的标准图。
### 阶段 B:真实执行过程闭环
1. 统一工具、人工介入、制品的 SSE payload。
2. 实现制品内容与下载接口。
3. 修复 lastSeq、cancel_requested。
4. 增加跨 REST 重放、SSE 断线重连、人工介入恢复的契约测试。
验收:报告工作流在 iframe 刷新后能恢复步骤、工具摘要、文件卡和人工介入状态。
### 阶段 C:草稿试运行与连续会话
1. 先由产品确认是否真的需要草稿试运行;需要时实现独立 test-run snapshot。
2. 再由业务确认是否需要跨 run 的上下文记忆;需要时实现 conversation/turn。
在阶段 C 前,前端必须隐藏对应入口,禁止用“看似可点、实际忽略”的体验替代能力。
## 7. 测试与验收
### 7.1 后端新增测试
建议新增:
| 文件 | 重点场景 |
| --- | --- |
| tests/test_workflow_studio_facade.py | studio DTO、camelCase、CAS、错误结构、owner/admin 鉴权 |
| tests/test_workflow_studio_graph_contract.py | Coze 编译产物可被 WorkflowGraph 接受;非法 graph 被拒绝 |
| tests/test_workflow_resource_validation.py | Agent/Skill/SQL/HTTP/Code/Output 配置与权限校验 |
| tests/test_workflow_event_contract.py | tool、awaiting_input、artifact、parallel agent payload 与 REST/SSE 一致 |
| tests/test_workflow_artifact_content.py | 文本预览、二进制下载、越权、路径穿越、超大文件 |
| tests/test_workflow_runs.py | target=draft 显式拒绝、cancel_requested、lastSeq |
保留并扩展:
- tests/test_workflow_engine.py:并行、循环、节点 started 元数据。
- tests/test_workflow_runs.py:运行、取消、恢复、SSE、重放。
- tests/test_workflow_data_sources.py:SQL 只读、HTTP allowlist、秘密不回显。
### 7.2 前后端契约 fixture
在 backend/tests/fixtures/workflow_studio/ 固化 JSON fixture,并让前端测试复制或通过生成脚本消费:
1. studio-document。
2. 保存成功与 409 冲突。
3. 资源缺失校验。
4. Agent 工具 started/finished。
5. human_input 暂停与恢复。
6. markdown/json/table/docx 制品。
7. 并行三智能体与取消中状态。
禁止前后端各自手写近似 mock 事件。fixture 变更必须同时更新双方测试。
### 7.3 发布准入
- [ ] Coze 画布和 executionGraph 采用两份数据、一次 CAS 保存。
- [ ] 真实后端不再依赖 schemaJson、valid 等 mock 专用字段。
- [ ] 所有可拖入资源都能在 graph 中找到真实绑定字段。
- [ ] 发布前可定位到节点和字段的资源/权限错误。
- [ ] 工具、人工介入、制品在 SSE 与 REST replay 中字段一致。
- [ ] 制品内容和下载经过 run 级鉴权,不能按任意路径读取。
- [ ] target=draft 不会被静默执行为发布版本。
- [ ] typecheck、后端 pytest、ruff 与至少一条 iframe E2E 全部通过。
## 8. 明确不做的事项
1. 不把整个 Coze Go 后端或 ChatDev 后端迁入 DeerFlow。
2. 不在 DeerFlow harness 中导入 app.gateway 或 Coze 前端类型。
3. 不为了兼容 mock 而让标准 WorkflowGraph API 永久接受任意 schemaJson。
4. 不向浏览器返回数据库 DSN、HTTP headers、token、resume token、沙箱真实文件路径或模型隐藏思维链。
5. 不在未实现草稿快照、连续会话前保留误导性的 UI 入口。
@@ -0,0 +1,54 @@
# ADR 0001:工作流编排运行时边界
- **状态**:Accepted(阶段 0 冻结;决策 3 已按阶段 2 实现修订,见文末修订记录)
- **日期**:2026-08-28
- **关联**:`docs/WORKFLOW_STUDIO_BACKEND_DEV_ZH.md`
## 背景
需要在 DeerFlow 中支持可持久化、可恢复、可审计的工作流运行,并用 Coze Studio 独立画布作为编辑器。可选路径包括:整包接入 Coze 后端、整包接入 ChatDev 2.0、或在 DeerFlow 内自建运行时。
## 决策
1. **Coze Studio 只负责画布与节点配置 UI**
不复用其完整后端协议、内置试运行、Trace/Debug。Gateway 仅提供薄兼容层(canvas/save/create/node_template)。
2. **ChatDev 2.0 只作语义参考**
可参考 DAG、并行、循环、动态边、会话执行边界;**不**引入其 WebSocket、SQLite 存储、宿主机 `subprocess` 代码执行、自有 Agent/模型 runtime。
3. **执行内核 = DeerFlow 自建 DAG 调度器;LangGraph 只作智能体内核**
见下方「阶段 2 修订」。智能体 / 技能 / 沙箱 / 模型 / 制品全部复用现有 harness 能力。`deerflow.workflows` 不得导入 `app.*`。
4. **流式协议 = 持久事件日志 + SSE**
先写 `workflow_run_events`,再推 live;断线用 `Last-Event-ID` / `after_seq` 补放。不与 ChatDev WebSocket 并存。
## 阶段 0 配套确认
| 议题 | 结论 |
|------|------|
| SQL 节点驱动 | 首期仅 **MySQL 和/或 PostgreSQL**(与项目实际库一致);不抽象全数据库 |
| 凭据存储 | 画布只存 `credentialRef` / `dataSourceId`;明文密钥进服务端凭据存储,不进 JSON / SSE / 日志 |
| iframe 域名 | 由 `config.yaml` → `workflows.embed.allowed_origins`(阶段 6 落地配置项);票据绑定 origin + workflow + user + actions,约 60s 一次性兑换 |
| 应用库 vs 业务查询库 | DeerFlow 自身业务库 **不得**直接暴露给 SQL 节点;必须管理员登记独立只读数据源 |
## 阶段 2 修订:工作流级编排不编译成 LangGraph
阶段 0 原定「标准图编译为 `StateGraph`」。实现阶段 2 时改为**自建 DAG 调度器**(`deerflow/workflows/runtime/engine.py`),LangGraph 退到**单个 `agent` / `skill` 节点内部**继续做智能体内核。
**原因**:运行状态必须只有一个权威来源。工作流的可恢复性建立在 `workflow_node_runs`(已完成节点的输出)+ `workflow_run_events`(事件日志)之上——重新租约或人工恢复时,执行器从这两张表回放,只跑缺失的节点。若同时用 LangGraph checkpoint 保存工作流级状态,就会出现两份状态需要保持一致(checkpoint 与 node_runs 谁说了算?租约被抢走后 checkpoint 归谁?),恢复语义无法自证。此外 `human_input` 的暂停需要把 `resume_token` 与 `pending_input` 暴露到 HTTP/SSE 层,条件分支需要「未选中的端口整条子图跳过」的显式边状态,循环需要按 `bodyEntry` 精确失效 body 内的 node_runs——这些都要求调度器直接读写运行存储。
**代价**:调度、并行度、重试、超时、循环上限这些机制要自己实现和测试(`tests/test_workflow_engine.py`、`tests/test_workflow_runs.py`)。**收益**:崩溃恢复、取消、暂停/恢复都只有一条真相路径,且不需要为工作流额外引入一套 checkpoint 生命周期。
调度规则本身很小:每条边有 `pending` / `active` / `pruned` 三态,节点在「非回边入边都已定型且至少一条 `active`」时就绪,全部 `pruned` 则跳过并继续下推剪枝。`loop` 节点的 `continue` 回边是唯一允许的环,不参与就绪判定。
## 后果
- 短期需自建 schema / store / dispatcher / SSE,工作量高于“整包移植”。
- 长期只维护一套模型调用、鉴权、沙箱与审计,避免双运行时分叉。
- 前端时间线必须与本仓库冻结的事件 envelope(`deerflow.workflows.events`)契约一致。
## 否决方案
- 把 Coze `workflow_api` 全量试运行当作执行引擎。
- 把 ChatDev `python_executor.py`(宿主机 subprocess)当作代码节点。
- 在画布 JSON 或 URL query 中传递 JWT / DB 密码 / API Key。
@@ -42,6 +42,7 @@ from deerflow.config.token_usage_config import TokenUsageConfig
from deerflow.config.tool_config import ToolConfig, ToolGroupConfig
from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_search_config_from_dict
from deerflow.config.vision_config import VisionConfig
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.config.y_log_config import YLogConfig
load_dotenv()
@@ -149,6 +150,7 @@ class AppConfig(BaseModel):
knowledge_ingest: KnowledgeIngestConfig = Field(default_factory=KnowledgeIngestConfig, description="External knowledge-ingest endpoint configuration")
task_deeplink: TaskDeeplinkConfig = Field(default_factory=TaskDeeplinkConfig, description="任务深链(rwfx)目的树闸门与跳转地址配置(前端经 /api/public/config/task-deeplink 读取)")
y_log: YLogConfig = Field(default_factory=YLogConfig, description="操作日志上报配置(前端经 /api/public/config/y-log 读取)")
workflows: WorkflowConfig = Field(default_factory=WorkflowConfig, description="Workflow Studio limits / iframe origins")
sentiment_agent: SentimentAgentConfig = Field(
default_factory=SentimentAgentConfig,
description="舆情分析虚拟智能体上游代理超时(地址由前端 runtime-config.js 随请求传入)",
@@ -0,0 +1,81 @@
"""Workflow Studio configuration (limits, egress policy, sandbox, iframe origins)."""
from __future__ import annotations
from pydantic import BaseModel, Field
class WorkflowEmbedConfig(BaseModel):
"""iframe ticket / origin policy."""
allowed_origins: list[str] = Field(
default_factory=list,
description="Allowed parent origins for workflow embed tickets (exact match)",
)
ticket_ttl_seconds: int = Field(default=60, ge=5, le=600)
class WorkflowHttpPolicyConfig(BaseModel):
"""Egress policy for the ``http`` node.
Default-deny on private address space. ``allowed_hosts`` is an explicit
allowlist (exact host or ``.suffix`` match); when it is empty every public
host is allowed but private/loopback/link-local targets are still refused,
which is what stops SSRF against cloud metadata and internal services.
"""
enabled: bool = Field(default=True)
allowed_hosts: list[str] = Field(default_factory=list)
blocked_hosts: list[str] = Field(default_factory=list)
allow_private_networks: bool = Field(
default=False,
description="Allow loopback/private/link-local targets (only for trusted intranet deployments)",
)
allowed_ports: list[int] = Field(default_factory=lambda: [80, 443, 8080, 8443])
max_response_bytes: int = Field(default=2_097_152, ge=1024)
max_redirects: int = Field(default=0, ge=0, le=5)
timeout_seconds: int = Field(default=60, ge=1)
class WorkflowSqlPolicyConfig(BaseModel):
"""Read-only policy for the ``sql_read`` node."""
enabled: bool = Field(default=True)
max_rows: int = Field(default=1000, ge=1)
statement_timeout_seconds: int = Field(default=30, ge=1)
max_cell_chars: int = Field(default=4000, ge=64)
class WorkflowCodeNodeConfig(BaseModel):
"""``code`` node execution. Off by default: the local runner is a hardened
subprocess, not a container, so operators must opt in knowingly."""
enabled: bool = Field(default=False)
timeout_seconds: int = Field(default=30, ge=1, le=600)
max_source_chars: int = Field(default=20_000, ge=64)
max_output_chars: int = Field(default=100_000, ge=256)
memory_limit_mb: int = Field(default=512, ge=64)
allow_network: bool = Field(default=False)
class WorkflowRetentionConfig(BaseModel):
event_retention_days: int = Field(default=30, ge=1)
run_retention_days: int = Field(default=90, ge=1)
class WorkflowConfig(BaseModel):
"""System-level workflow limits and policies."""
enabled: bool = Field(default=True, description="Master switch for workflow studio APIs")
max_steps: int = Field(default=500, ge=1)
max_loop_iterations: int = Field(default=20, ge=1)
max_parallelism: int = Field(default=16, ge=1)
run_timeout_seconds: int = Field(default=3600, ge=1)
node_timeout_seconds: int = Field(default=600, ge=1)
max_concurrent_runs_per_user: int = Field(default=5, ge=1)
lease_ttl_seconds: int = Field(default=60, ge=10)
embed: WorkflowEmbedConfig = Field(default_factory=WorkflowEmbedConfig)
http: WorkflowHttpPolicyConfig = Field(default_factory=WorkflowHttpPolicyConfig)
sql: WorkflowSqlPolicyConfig = Field(default_factory=WorkflowSqlPolicyConfig)
code: WorkflowCodeNodeConfig = Field(default_factory=WorkflowCodeNodeConfig)
retention: WorkflowRetentionConfig = Field(default_factory=WorkflowRetentionConfig)
@@ -0,0 +1,64 @@
"""Create workflow_definitions / workflow_versions tables.
Revision ID: 20260828_01
Revises: 20260825_01
Create Date: 2026-08-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_01"
down_revision = "20260825_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_definitions"):
op.create_table(
"workflow_definitions",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("name", sa.String(length=255), nullable=False),
sa.Column("description", sa.Text(), nullable=False, server_default=""),
sa.Column("owner_id", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="draft"),
sa.Column("draft_revision", sa.Integer(), nullable=False, server_default="0"),
sa.Column("draft_graph_json", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_workflow_definitions_owner_id", "workflow_definitions", ["owner_id"])
op.create_index("ix_workflow_definitions_status", "workflow_definitions", ["status"])
if not inspector.has_table("workflow_versions"):
op.create_table(
"workflow_versions",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("workflow_id", sa.String(length=64), nullable=False),
sa.Column("version_number", sa.Integer(), nullable=False),
sa.Column("graph_json", sa.Text(), nullable=False),
sa.Column("graph_hash", sa.String(length=64), nullable=False),
sa.Column("input_schema", sa.Text(), nullable=True),
sa.Column("output_schema", sa.Text(), nullable=True),
sa.Column("published_by", sa.String(length=64), nullable=True),
sa.Column("published_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("change_note", sa.Text(), nullable=False, server_default=""),
sa.ForeignKeyConstraint(["workflow_id"], ["workflow_definitions.id"], ondelete="CASCADE"),
sa.UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),
)
op.create_index("ix_workflow_versions_workflow_id", "workflow_versions", ["workflow_id"])
op.create_index("ix_workflow_versions_graph_hash", "workflow_versions", ["graph_hash"])
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_versions"):
op.drop_table("workflow_versions")
if inspector.has_table("workflow_definitions"):
op.drop_table("workflow_definitions")
@@ -0,0 +1,127 @@
"""Create workflow run / node-run / event / artifact tables.
Revision ID: 20260828_02
Revises: 20260828_01
Create Date: 2026-08-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_02"
down_revision = "20260828_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_runs"):
op.create_table(
"workflow_runs",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("workflow_id", sa.String(length=64), nullable=False),
sa.Column("workflow_version_id", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
sa.Column("owner_id", sa.String(length=64), nullable=False),
sa.Column("idempotency_key", sa.String(length=128), nullable=True),
sa.Column("input_json", sa.Text(), nullable=True),
sa.Column("output_json", sa.Text(), nullable=True),
sa.Column("error_json", sa.Text(), nullable=True),
sa.Column("context_json", sa.Text(), nullable=True),
sa.Column("pending_input_json", sa.Text(), nullable=True),
sa.Column("resume_token", sa.String(length=64), nullable=True),
sa.Column("lease_owner", sa.String(length=64), nullable=True),
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("attempt", sa.Integer(), nullable=False, server_default="0"),
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
sa.Column("version", sa.Integer(), nullable=False, server_default="0"),
sa.Column("retry_of_run_id", sa.String(length=64), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),
)
op.create_index("ix_workflow_runs_workflow_id", "workflow_runs", ["workflow_id"])
op.create_index("ix_workflow_runs_workflow_version_id", "workflow_runs", ["workflow_version_id"])
op.create_index("ix_workflow_runs_owner_id", "workflow_runs", ["owner_id"])
op.create_index("ix_workflow_runs_status", "workflow_runs", ["status"])
op.create_index("ix_workflow_runs_resume_token", "workflow_runs", ["resume_token"])
if not inspector.has_table("workflow_node_runs"):
op.create_table(
"workflow_node_runs",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("run_id", sa.String(length=64), nullable=False),
sa.Column("node_id", sa.String(length=128), nullable=False),
sa.Column("node_type", sa.String(length=32), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
sa.Column("attempt", sa.Integer(), nullable=False, server_default="1"),
sa.Column("iteration", sa.Integer(), nullable=False, server_default="0"),
sa.Column("is_current", sa.Boolean(), nullable=False, server_default=sa.text("1")),
sa.Column("input_snapshot", sa.Text(), nullable=True),
sa.Column("output_snapshot", sa.Text(), nullable=True),
sa.Column("error_json", sa.Text(), nullable=True),
sa.Column("parent_node_run_id", sa.String(length=64), nullable=True),
sa.Column("duration_ms", sa.Integer(), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint(
"run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"
),
)
op.create_index("ix_workflow_node_runs_run_id", "workflow_node_runs", ["run_id"])
op.create_index("ix_workflow_node_runs_status", "workflow_node_runs", ["status"])
if not inspector.has_table("workflow_run_events"):
op.create_table(
"workflow_run_events",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("run_id", sa.String(length=64), nullable=False),
sa.Column("workflow_id", sa.String(length=64), nullable=False),
sa.Column("version_id", sa.String(length=64), nullable=False),
sa.Column("seq", sa.Integer(), nullable=False),
sa.Column("event_type", sa.String(length=64), nullable=False),
sa.Column("node_id", sa.String(length=128), nullable=True),
sa.Column("node_run_id", sa.String(length=64), nullable=True),
sa.Column("payload_json", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
)
op.create_index("ix_workflow_run_events_run_id", "workflow_run_events", ["run_id"])
op.create_index("ix_workflow_run_events_run_seq", "workflow_run_events", ["run_id", "seq"])
if not inspector.has_table("workflow_run_artifacts"):
op.create_table(
"workflow_run_artifacts",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("run_id", sa.String(length=64), nullable=False),
sa.Column("node_id", sa.String(length=128), nullable=False),
sa.Column("name", sa.String(length=512), nullable=False),
sa.Column("mime_type", sa.String(length=191), nullable=False),
sa.Column("path", sa.Text(), nullable=False),
sa.Column("size_bytes", sa.Integer(), nullable=True),
sa.Column("preview", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_workflow_run_artifacts_run_id", "workflow_run_artifacts", ["run_id"])
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
for table in (
"workflow_run_artifacts",
"workflow_run_events",
"workflow_node_runs",
"workflow_runs",
):
if inspector.has_table(table):
op.drop_table(table)
@@ -0,0 +1,49 @@
"""Create workflow_data_sources table.
Revision ID: 20260828_03
Revises: 20260828_02
Create Date: 2026-08-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_03"
down_revision = "20260828_02"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_data_sources"):
return
op.create_table(
"workflow_data_sources",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("name", sa.String(length=191), nullable=False),
sa.Column("description", sa.String(length=512), nullable=False, server_default=""),
sa.Column("kind", sa.String(length=32), nullable=False, server_default="sql"),
sa.Column("driver", sa.String(length=64), nullable=False, server_default=""),
sa.Column("secret_ciphertext", sa.Text(), nullable=False),
sa.Column("secret_encrypted", sa.Boolean(), nullable=False, server_default=sa.text("1")),
sa.Column("masked_target", sa.String(length=512), nullable=False, server_default=""),
sa.Column("max_rows", sa.Integer(), nullable=False, server_default="1000"),
sa.Column("allowed_tables", sa.Text(), nullable=True),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.text("1")),
sa.Column("owner_id", sa.String(length=64), nullable=True),
sa.Column("created_by", sa.String(length=64), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("name", name="uq_workflow_data_sources_name"),
)
op.create_index("ix_workflow_data_sources_owner_id", "workflow_data_sources", ["owner_id"])
def downgrade() -> None:
bind = op.get_bind()
if sa.inspect(bind).has_table("workflow_data_sources"):
op.drop_table("workflow_data_sources")
@@ -0,0 +1,66 @@
"""Add workflow counter columns: version allocator + event seq counter.
Revision ID: 20260828_04
Revises: 20260828_03
Create Date: 2026-08-28
Functional-robustness remediation (audit items P1-02 / P1-03):
* ``workflow_definitions.next_version_number`` — publish allocates the version
number by CAS-bumping this counter instead of ``MAX(version_number) + 1``,
so two concurrent publishes get adjacent distinct numbers instead of a
unique-constraint 500.
* ``workflow_runs.next_event_seq`` — the event log draws each event's ``seq``
from this counter in the insert's own transaction instead of ``MAX(seq)+1``.
Both columns carry a ``server_default`` so existing databases get them via the
startup ``_ensure_orm_columns_sync`` sweep even without running Alembic; rows
keep the default (1 / 0) and the first write self-heals from the existing
maximum on unique-conflict.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_04"
down_revision = "20260828_03"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_definitions"):
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
if "next_version_number" not in existing:
op.add_column(
"workflow_definitions",
sa.Column("next_version_number", sa.Integer(), nullable=False, server_default="1"),
)
if inspector.has_table("workflow_runs"):
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
if "next_event_seq" not in existing:
op.add_column(
"workflow_runs",
sa.Column("next_event_seq", sa.Integer(), nullable=False, server_default="0"),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_definitions"):
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
if "next_version_number" in existing:
op.drop_column("workflow_definitions", "next_version_number")
if inspector.has_table("workflow_runs"):
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
if "next_event_seq" in existing:
op.drop_column("workflow_runs", "next_event_seq")
@@ -0,0 +1,50 @@
"""Add non-secret HTTP metadata to workflow data sources.
Revision ID: 20260829_01
Revises: 20260828_04
Create Date: 2026-08-29
HTTP request headers remain in the encrypted secret blob. The endpoint and
method allowlist are intentionally separate non-secret fields so the canvas
can show a reusable interface resource without exposing credentials.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260829_01"
down_revision = "20260828_04"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_data_sources"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
if "base_url" not in existing:
op.add_column(
"workflow_data_sources",
sa.Column("base_url", sa.String(length=2048), nullable=False, server_default=""),
)
if "allowed_methods" not in existing:
op.add_column(
"workflow_data_sources",
sa.Column("allowed_methods", sa.Text(), nullable=True),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_data_sources"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
if "allowed_methods" in existing:
op.drop_column("workflow_data_sources", "allowed_methods")
if "base_url" in existing:
op.drop_column("workflow_data_sources", "base_url")
@@ -0,0 +1,51 @@
"""Add Coze canvas draft column to workflow_definitions.
Revision ID: 20260829_02
Revises: 20260829_01
Create Date: 2026-08-29
The canvas document (positions, Coze form internals) is stored separately from
the executable WorkflowGraph so neither representation can impersonate the
other. ``published_canvas_schema_hash`` records which canvas the current
published version was compiled from, for audit only.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260829_02"
down_revision = "20260829_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_definitions"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
if "draft_canvas_schema_json" not in existing:
op.add_column(
"workflow_definitions",
sa.Column("draft_canvas_schema_json", sa.Text(), nullable=False, server_default="'{}'"),
)
if "published_canvas_schema_hash" not in existing:
op.add_column(
"workflow_definitions",
sa.Column("published_canvas_schema_hash", sa.String(length=64), nullable=True),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_definitions"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
if "published_canvas_schema_hash" in existing:
op.drop_column("workflow_definitions", "published_canvas_schema_hash")
if "draft_canvas_schema_json" in existing:
op.drop_column("workflow_definitions", "draft_canvas_schema_json")
@@ -50,6 +50,14 @@ from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
@@ -128,6 +136,13 @@ __all__ = [
"KnowledgeSourceRow",
"KnowledgeTagRow",
"LightAppRow",
"WorkflowDataSourceRow",
"WorkflowDefinitionRow",
"WorkflowNodeRunRow",
"WorkflowRunArtifactRow",
"WorkflowRunEventRow",
"WorkflowRunRow",
"WorkflowVersionRow",
"LlmCallMetricRow",
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
@@ -0,0 +1,19 @@
"""Workflow data-source persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
from deerflow.persistence.workflow_data_sources.store import (
MemoryWorkflowDataSourceStore,
SqlWorkflowDataSourceStore,
WorkflowDataSourceStore,
make_workflow_data_source_store,
)
__all__ = [
"MemoryWorkflowDataSourceStore",
"SqlWorkflowDataSourceStore",
"WorkflowDataSourceRow",
"WorkflowDataSourceStore",
"make_workflow_data_source_store",
]
@@ -0,0 +1,51 @@
"""ORM model for workflow SQL data sources and HTTP credentials."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, UniqueConstraint
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class WorkflowDataSourceRow(Base):
"""A read-only database connection usable by ``sql_read`` nodes.
``secret_ciphertext`` holds the DSN encrypted with the workflow secret key;
it is never returned by any API. ``owner_id`` NULL means the data source is
shared (admin-managed)."""
__tablename__ = "workflow_data_sources"
__table_args__ = (UniqueConstraint("name", name="uq_workflow_data_sources_name"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(191), nullable=False)
description: Mapped[str] = mapped_column(String(512), nullable=False, default="")
kind: Mapped[str] = mapped_column(String(32), nullable=False, default="sql")
driver: Mapped[str] = mapped_column(String(64), nullable=False, default="")
base_url: Mapped[str] = mapped_column(
String(2048),
nullable=False,
default="",
server_default=sa_text("''"),
)
# Nullable so the startup column synchronizer can safely patch existing databases.
allowed_methods: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
secret_ciphertext: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
secret_encrypted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
masked_target: Mapped[str] = mapped_column(String(512), nullable=False, default="")
max_rows: Mapped[int] = mapped_column(Integer, nullable=False, default=1000, server_default=sa_text("1000"))
allowed_tables: Mapped[list] = mapped_column(PortableJSON(), default=list)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
owner_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,293 @@
"""Workflow data-source store (SQL + in-memory).
Secrets are encrypted before they reach the database and only ever decrypted
inside :meth:`resolve_dsn`, which the run executor calls. No read path returns
the ciphertext.
"""
from __future__ import annotations
import copy
import uuid
from abc import ABC, abstractmethod
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
from deerflow.workflows.security.secret_box import decrypt_secret, encrypt_secret, mask_dsn
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _public_dict(row: WorkflowDataSourceRow) -> dict[str, Any]:
return {
"id": row.id,
"name": row.name,
"description": row.description or "",
"kind": row.kind,
"driver": row.driver or "",
"base_url": row.base_url or "",
"allowed_methods": list(row.allowed_methods or []),
"masked_target": row.masked_target or "",
"secret_encrypted": bool(row.secret_encrypted),
"max_rows": row.max_rows,
"allowed_tables": list(row.allowed_tables or []),
"enabled": bool(row.enabled),
"owner_id": row.owner_id,
"created_by": row.created_by,
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
class WorkflowDataSourceStore(ABC):
@abstractmethod
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_source(self, source_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def delete_source(self, source_id: str) -> bool:
raise NotImplementedError
@abstractmethod
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
"""Decrypted connection info for the executor. Never expose via HTTP."""
raise NotImplementedError
class SqlWorkflowDataSourceStore(WorkflowDataSourceStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
stmt = select(WorkflowDataSourceRow).order_by(WorkflowDataSourceRow.name.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
visible = []
for row in rows:
if row.owner_id is None and include_shared:
visible.append(row)
elif owner_id is not None and row.owner_id == owner_id:
visible.append(row)
return [_public_dict(row) for row in visible]
async def get_source(self, source_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
return _public_dict(row) if row else None
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
dsn = str(data.get("dsn") or "")
kind = str(data.get("kind") or "sql")
base_url = str(data.get("base_url") or "")
row = WorkflowDataSourceRow(
id=str(data.get("id") or uuid.uuid4()),
name=str(data["name"])[:191],
description=str(data.get("description") or "")[:512],
kind=kind,
driver=str(
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
)[:64],
base_url=base_url[:2048],
allowed_methods=list(data.get("allowed_methods") or []),
secret_ciphertext=encrypt_secret(dsn) if dsn else "",
secret_encrypted=bool(dsn),
masked_target=(base_url if kind == "http" else mask_dsn(dsn))[:512],
max_rows=int(data.get("max_rows") or 1000),
allowed_tables=list(data.get("allowed_tables") or []),
enabled=bool(data.get("enabled", True)),
owner_id=data.get("owner_id"),
created_by=data.get("created_by"),
)
async with self._sf() as session:
session.add(row)
await session.flush()
out = _public_dict(row)
await session.commit()
return out
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
if row is None:
return None
if data.get("name"):
row.name = str(data["name"])[:191]
if "description" in data:
row.description = str(data.get("description") or "")[:512]
if "base_url" in data:
row.base_url = str(data.get("base_url") or "")[:2048]
if row.kind == "http":
row.masked_target = row.base_url[:512]
if "allowed_methods" in data:
row.allowed_methods = list(data.get("allowed_methods") or [])
if data.get("dsn"):
dsn = str(data["dsn"])
row.secret_ciphertext = encrypt_secret(dsn)
row.secret_encrypted = True
if row.kind == "http":
row.driver = "HTTP API"
else:
row.masked_target = mask_dsn(dsn)[:512]
row.driver = dsn.split("://", 1)[0][:64]
if data.get("max_rows") is not None:
row.max_rows = int(data["max_rows"])
if "allowed_tables" in data:
row.allowed_tables = list(data.get("allowed_tables") or [])
if "enabled" in data:
row.enabled = bool(data["enabled"])
row.updated_at = datetime.now(UTC)
await session.flush()
out = _public_dict(row)
await session.commit()
return out
async def delete_source(self, source_id: str) -> bool:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
if row is None or not row.enabled or not row.secret_ciphertext:
return None
return {
"id": row.id,
"dsn": decrypt_secret(row.secret_ciphertext),
"kind": row.kind,
"base_url": row.base_url or "",
"allowed_methods": list(row.allowed_methods or []),
"max_rows": row.max_rows,
"allowed_tables": list(row.allowed_tables or []),
"owner_id": row.owner_id,
}
class MemoryWorkflowDataSourceStore(WorkflowDataSourceStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._secrets: dict[str, str] = {}
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
out = []
for row in self._rows.values():
if row["owner_id"] is None and include_shared:
out.append(row)
elif owner_id is not None and row["owner_id"] == owner_id:
out.append(row)
return [copy.deepcopy(r) for r in sorted(out, key=lambda r: r["name"])]
async def get_source(self, source_id: str) -> dict[str, Any] | None:
row = self._rows.get(source_id)
return copy.deepcopy(row) if row else None
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
source_id = str(data.get("id") or uuid.uuid4())
dsn = str(data.get("dsn") or "")
kind = str(data.get("kind") or "sql")
base_url = str(data.get("base_url") or "")
now = _iso(datetime.now(UTC))
row = {
"id": source_id,
"name": str(data["name"]),
"description": str(data.get("description") or ""),
"kind": kind,
"driver": str(
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
),
"base_url": base_url,
"allowed_methods": list(data.get("allowed_methods") or []),
"masked_target": base_url if kind == "http" else mask_dsn(dsn),
"secret_encrypted": bool(dsn),
"max_rows": int(data.get("max_rows") or 1000),
"allowed_tables": list(data.get("allowed_tables") or []),
"enabled": bool(data.get("enabled", True)),
"owner_id": data.get("owner_id"),
"created_by": data.get("created_by"),
"created_at": now,
"updated_at": now,
}
self._rows[source_id] = row
self._secrets[source_id] = dsn
return copy.deepcopy(row)
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._rows.get(source_id)
if row is None:
return None
for key in (
"name",
"description",
"base_url",
"allowed_methods",
"max_rows",
"allowed_tables",
"enabled",
):
if key in data and data[key] is not None:
row[key] = data[key]
if "base_url" in data and row["kind"] == "http":
row["masked_target"] = str(row["base_url"])
if data.get("dsn"):
self._secrets[source_id] = str(data["dsn"])
if row["kind"] == "http":
row["driver"] = "HTTP API"
else:
row["masked_target"] = mask_dsn(str(data["dsn"]))
row["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(row)
async def delete_source(self, source_id: str) -> bool:
self._secrets.pop(source_id, None)
return self._rows.pop(source_id, None) is not None
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
row = self._rows.get(source_id)
if row is None or not row["enabled"]:
return None
return {
"id": source_id,
"dsn": self._secrets.get(source_id, ""),
"kind": row["kind"],
"base_url": row.get("base_url", ""),
"allowed_methods": list(row.get("allowed_methods") or []),
"max_rows": row["max_rows"],
"allowed_tables": row["allowed_tables"],
"owner_id": row["owner_id"],
}
def make_workflow_data_source_store(session_factory):
if session_factory is None:
return MemoryWorkflowDataSourceStore()
return SqlWorkflowDataSourceStore(session_factory)
__all__ = [
"MemoryWorkflowDataSourceStore",
"SqlWorkflowDataSourceStore",
"WorkflowDataSourceStore",
"make_workflow_data_source_store",
]
@@ -0,0 +1,22 @@
"""Workflow run event-log persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_events.memory import MemoryWorkflowEventStore
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_events.sql import SqlWorkflowEventStore
__all__ = [
"MemoryWorkflowEventStore",
"SqlWorkflowEventStore",
"WorkflowEventStore",
"WorkflowRunEventRow",
"make_workflow_event_store",
]
def make_workflow_event_store(session_factory, run_store=None):
if session_factory is None:
return MemoryWorkflowEventStore(run_store=run_store)
return SqlWorkflowEventStore(session_factory)
@@ -0,0 +1,42 @@
"""Abstract workflow run event store (durable, replayable)."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class WorkflowEventStore(ABC):
@abstractmethod
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> dict[str, Any]:
"""Persist one event, assigning the next per-run ``seq``."""
raise NotImplementedError
@abstractmethod
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def max_seq(self, run_id: str) -> int:
raise NotImplementedError
@abstractmethod
async def purge_before(self, *, older_than_days: int) -> int:
"""Retention purge by event age; returns deleted row count."""
raise NotImplementedError
@abstractmethod
async def purge_for_runs(self, run_ids: list[str]) -> int:
"""Delete every event belonging to the given runs (used when the runs
themselves are retained away). Returns deleted row count."""
raise NotImplementedError
@@ -0,0 +1,103 @@
"""In-memory workflow event log (tests / no-database deployments)."""
from __future__ import annotations
import asyncio
import copy
from datetime import UTC, datetime, timedelta
from typing import Any
from deerflow.persistence.workflow_events.base import WorkflowEventStore
class MemoryWorkflowEventStore(WorkflowEventStore):
def __init__(self, *, max_per_run: int = 20_000, run_store: Any = None) -> None:
self._events: dict[str, list[dict[str, Any]]] = {}
self._max_per_run = max_per_run
self._run_store = run_store
self._lock = asyncio.Lock()
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> dict[str, Any]:
async with self._lock:
rows = self._events.setdefault(run_id, [])
event = {
"seq": len(rows) + 1 if not rows else rows[-1]["seq"] + 1,
"run_id": run_id,
"workflow_id": workflow_id,
"version_id": version_id,
"event_type": event_type,
"node_id": node_id,
"node_run_id": node_run_id,
"payload": payload or {},
"created_at": datetime.now(UTC).isoformat(),
}
rows.append(event)
if len(rows) > self._max_per_run:
del rows[: len(rows) - self._max_per_run]
if self._run_store is not None and hasattr(self._run_store, "note_event_seq"):
# Keep the run row's lastSeq counter in sync (mirrors the SQL store,
# where the counter bump and the insert share one transaction).
try:
await self._run_store.note_event_seq(run_id, event["seq"])
except Exception: # noqa: BLE001 - counter sync is best effort
pass
return copy.deepcopy(event)
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
rows = [r for r in self._events.get(run_id, []) if r["seq"] > after_seq]
return [copy.deepcopy(r) for r in rows[:limit]]
async def max_seq(self, run_id: str) -> int:
rows = self._events.get(run_id, [])
return int(rows[-1]["seq"]) if rows else 0
async def purge_before(self, *, older_than_days: int) -> int:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._lock:
deleted = 0
for run_id, rows in list(self._events.items()):
kept: list[dict[str, Any]] = []
for row in rows:
created = _parse_dt(row.get("created_at"))
if created is not None and created < cutoff:
deleted += 1
else:
kept.append(row)
if kept:
self._events[run_id] = kept
else:
del self._events[run_id]
return deleted
async def purge_for_runs(self, run_ids: list[str]) -> int:
if not run_ids:
return 0
async with self._lock:
deleted = 0
for run_id in run_ids:
rows = self._events.pop(run_id, None)
if rows:
deleted += len(rows)
return deleted
def _parse_dt(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, str) and value:
try:
parsed = datetime.fromisoformat(value)
except ValueError:
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
return None
@@ -0,0 +1,34 @@
"""ORM model for the durable workflow run event log."""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class WorkflowRunEventRow(Base):
"""A single persisted event. ``seq`` is monotonic per run and gap-free."""
__tablename__ = "workflow_run_events"
__table_args__ = (
UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
sa.Index("ix_workflow_run_events_run_seq", "run_id", "seq"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False)
version_id: Mapped[str] = mapped_column(String(64), nullable=False)
seq: Mapped[int] = mapped_column(Integer, nullable=False)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
node_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
# JSON text rather than PortableJSON: payloads can hold bounded deltas.
payload_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="{}")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -0,0 +1,157 @@
"""SQLAlchemy-backed workflow event log."""
from __future__ import annotations
import json
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_runs.model import WorkflowRunRow
_MAX_PAYLOAD_CHARS = 64_000
_APPEND_ATTEMPTS = 5
def _dump_payload(payload: dict[str, Any] | None) -> str:
text = json.dumps(payload or {}, ensure_ascii=False, separators=(",", ":"), default=str)
if len(text) > _MAX_PAYLOAD_CHARS:
return json.dumps(
{"truncated": True, "preview": text[:_MAX_PAYLOAD_CHARS]},
ensure_ascii=False,
separators=(",", ":"),
)
return text
def event_to_dict(row: WorkflowRunEventRow) -> dict[str, Any]:
try:
payload = json.loads(row.payload_json or "{}")
except json.JSONDecodeError:
payload = {}
return {
"seq": row.seq,
"run_id": row.run_id,
"workflow_id": row.workflow_id,
"version_id": row.version_id,
"event_type": row.event_type,
"node_id": row.node_id,
"node_run_id": row.node_run_id,
"payload": payload if isinstance(payload, dict) else {},
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else None,
}
class SqlWorkflowEventStore(WorkflowEventStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
max_retries: int = _APPEND_ATTEMPTS,
) -> dict[str, Any]:
"""Persist one event, taking its ``seq`` from the run's own counter.
``MAX(seq) + 1`` let two appenders draw the same number and burn a
unique-constraint retry; here the counter increment and the insert share
one transaction, and the row write-lock held by the UPDATE serialises
concurrent appenders until commit — so seqs are strictly increasing,
unique and gap-free per run, in commit order.
"""
text = _dump_payload(payload)
for _ in range(max_retries):
async with self._sf() as session:
seq = await self._take_seq(session, run_id)
row = WorkflowRunEventRow(
run_id=run_id,
workflow_id=workflow_id,
version_id=version_id,
seq=seq,
event_type=event_type,
node_id=node_id,
node_run_id=node_run_id,
payload_json=text,
)
session.add(row)
try:
await session.flush()
except IntegrityError:
# Only reachable for legacy rows whose counter was backfilled
# to 0 while events already existed. Re-sync the counter to
# MAX(seq) in its own committed transaction, then retry.
await session.rollback()
await self._resync_counter(run_id)
continue
out = event_to_dict(row)
await session.commit()
return out
raise RuntimeError(f"failed to append workflow event after {max_retries} attempts (run={run_id})")
async def _take_seq(self, session: Any, run_id: str) -> int:
"""Draw the next seq. Falls back to MAX(seq)+1 when the run row is
missing (event store used standalone in tests)."""
result = await session.execute(
update(WorkflowRunRow).where(WorkflowRunRow.id == run_id).values(next_event_seq=WorkflowRunRow.next_event_seq + 1)
)
if result.rowcount:
# Our own uncommitted increment; the row lock guards it until commit.
current = (
await session.execute(select(WorkflowRunRow.next_event_seq).where(WorkflowRunRow.id == run_id))
).scalar()
return int(current or 1)
current = (
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
).scalar()
return int(current or 0) + 1
async def _resync_counter(self, run_id: str) -> None:
"""Point a legacy/stale run counter back at MAX(seq)."""
async with self._sf() as session:
max_seq = (
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
).scalar() or 0
await session.execute(
update(WorkflowRunRow)
.where(WorkflowRunRow.id == run_id, WorkflowRunRow.next_event_seq < int(max_seq))
.values(next_event_seq=int(max_seq))
)
await session.commit()
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
stmt = select(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id == run_id, WorkflowRunEventRow.seq > after_seq).order_by(WorkflowRunEventRow.seq.asc()).limit(limit)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [event_to_dict(r) for r in rows]
async def max_seq(self, run_id: str) -> int:
async with self._sf() as session:
value = (await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))).scalar()
return int(value or 0)
async def purge_before(self, *, older_than_days: int) -> int:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._sf() as session:
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.created_at < cutoff))
await session.commit()
return int(result.rowcount or 0)
async def purge_for_runs(self, run_ids: list[str]) -> int:
if not run_ids:
return 0
async with self._sf() as session:
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id.in_(list(run_ids))))
await session.commit()
return int(result.rowcount or 0)
@@ -0,0 +1,38 @@
"""Workflow run / node-run / artifact persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunConflictError,
WorkflowRunNotFoundError,
WorkflowRunStore,
)
from deerflow.persistence.workflow_runs.memory import MemoryWorkflowRunStore
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
from deerflow.persistence.workflow_runs.sql import SqlWorkflowRunStore
__all__ = [
"CANCELLABLE_RUN_STATUSES",
"MemoryWorkflowRunStore",
"SqlWorkflowRunStore",
"TERMINAL_RUN_STATUSES",
"WorkflowNodeRunRow",
"WorkflowRunArtifactRow",
"WorkflowRunConflictError",
"WorkflowRunNotFoundError",
"WorkflowRunRow",
"WorkflowRunStore",
"make_workflow_run_store",
]
def make_workflow_run_store(session_factory):
if session_factory is None:
return MemoryWorkflowRunStore()
return SqlWorkflowRunStore(session_factory)
@@ -0,0 +1,161 @@
"""Abstract workflow run / node-run / artifact store."""
from __future__ import annotations
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any
TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled")
CANCELLABLE_RUN_STATUSES = ("queued", "running", "awaiting_input")
class WorkflowRunNotFoundError(Exception):
def __init__(self, run_id: str) -> None:
super().__init__(f"workflow run not found: {run_id}")
self.run_id = run_id
class WorkflowRunConflictError(Exception):
"""Raised when a state transition is not legal for the current status."""
def __init__(self, run_id: str, status: str) -> None:
super().__init__(f"workflow run {run_id} not in a valid state (status={status})")
self.run_id = run_id
self.status = status
class WorkflowRunStore(ABC):
# ── runs ────────────────────────────────────────────────────────────
@abstractmethod
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
"""Create a run. Returns ``(run, created)``; an existing
``idempotency_key`` yields the stored run with ``created=False``."""
raise NotImplementedError
@abstractmethod
async def get_run(self, run_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
"""Run ids that are ``queued`` or ``running`` with an expired lease."""
raise NotImplementedError
@abstractmethod
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
raise NotImplementedError
@abstractmethod
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
"""CAS update. When ``lease_owner`` is given the row must still be leased
by that owner. Terminal statuses clear the lease."""
raise NotImplementedError
@abstractmethod
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Pause atomically: status + token + pending descriptor (+ optional run
context) in one conditional update, so a concurrent resume can never be
overwritten by a follow-up context write."""
raise NotImplementedError
@abstractmethod
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
"""Atomically consume a one-shot resume token: ``awaiting_input`` →
``queued``. Returns None when the token is wrong/already used."""
raise NotImplementedError
@abstractmethod
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Atomic human-input resume in a single conditional update.
Validates the token, stores ``payload`` as ``context.resume_payload``
(replacing the whole run context with ``context`` when given), clears the
token/pending descriptor and flips the run to ``queued`` — all under
``WHERE status = 'awaiting_input' AND resume_token = :token``. Returns
the resumed run on success, ``None`` when the token is wrong, already
used, or the run is no longer awaiting input (caller maps that to 409).
"""
raise NotImplementedError
# ── node runs ───────────────────────────────────────────────────────
@abstractmethod
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
"""Insert-or-update a node run keyed by (run_id, node_id, attempt, iteration)."""
raise NotImplementedError
@abstractmethod
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
"""Mark node runs stale (used when a loop re-enters its body)."""
raise NotImplementedError
# ── artifacts ───────────────────────────────────────────────────────
@abstractmethod
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
"""Fetch one artifact row scoped to its run (authorization key)."""
raise NotImplementedError
# ── retention ───────────────────────────────────────────────────────
@abstractmethod
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
"""Delete finished runs older than the cutoff, with their node runs and
artifact rows. Only terminal runs are eligible. Returns deleted run ids
(bounded per call so a sweeper can loop)."""
raise NotImplementedError
@@ -0,0 +1,388 @@
"""In-memory workflow run store (tests / no-database deployments)."""
from __future__ import annotations
import asyncio
import copy
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunStore,
)
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
class MemoryWorkflowRunStore(WorkflowRunStore):
def __init__(self) -> None:
self._runs: dict[str, dict[str, Any]] = {}
self._node_runs: dict[str, list[dict[str, Any]]] = {}
self._artifacts: dict[str, list[dict[str, Any]]] = {}
self._lock = asyncio.Lock()
# ── runs ────────────────────────────────────────────────────────────
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
async with self._lock:
key = data.get("idempotency_key")
if key:
for run in self._runs.values():
if run.get("idempotency_key") == str(key):
return copy.deepcopy(run), False
now = datetime.now(UTC)
run_id = str(data.get("id") or uuid.uuid4())
run = {
"id": run_id,
"workflow_id": str(data["workflow_id"]),
"workflow_version_id": str(data["workflow_version_id"]),
"status": str(data.get("status") or "queued"),
"owner_id": str(data["owner_id"]),
"idempotency_key": str(key) if key else None,
"input": data.get("input") or {},
"output": None,
"error": None,
"context": data.get("context") or {},
"pending_input": None,
"resume_token": None,
"lease_owner": None,
"lease_until": None,
"heartbeat_at": None,
"attempt": 0,
"max_attempts": int(data.get("max_attempts") or 3),
"version": 0,
"retry_of_run_id": data.get("retry_of_run_id"),
"next_event_seq": 0,
"started_at": None,
"finished_at": None,
"created_at": _iso(now),
"updated_at": _iso(now),
"_created": now,
}
self._runs[run_id] = run
return copy.deepcopy(run), True
async def get_run(self, run_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
return copy.deepcopy(run) if run else None
async def note_event_seq(self, run_id: str, seq: int) -> None:
"""Mirror the SQL counter semantics for no-DB mode: the memory event
store reports each handed-out seq so run views can expose lastSeq."""
async with self._lock:
run = self._runs.get(run_id)
if run is not None and int(seq) > int(run.get("next_event_seq") or 0):
run["next_event_seq"] = int(seq)
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
rows = sorted(self._runs.values(), key=lambda r: r["_created"], reverse=True)
if workflow_id:
rows = [r for r in rows if r["workflow_id"] == workflow_id]
if owner_id:
rows = [r for r in rows if r["owner_id"] == owner_id]
if status:
rows = [r for r in rows if r["status"] == status]
return [copy.deepcopy(r) for r in rows[offset : offset + limit]]
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
out: list[str] = []
for run in sorted(self._runs.values(), key=lambda r: r["_created"]):
if run["status"] == "queued":
out.append(run["id"])
elif run["status"] == "running" and run["lease_until"]:
if datetime.fromisoformat(run["lease_until"]) < now:
out.append(run["id"])
if len(out) >= limit:
break
return out
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None:
return None
now = datetime.now(UTC)
claimable = run["status"] == "queued" or (run["status"] == "running" and run["lease_until"] and datetime.fromisoformat(run["lease_until"]) < now)
if not claimable:
return None
run.update(
status="running",
lease_owner=lease_owner,
lease_until=_iso(lease_until),
heartbeat_at=_iso(now),
attempt=run["attempt"] + 1,
version=run["version"] + 1,
started_at=run["started_at"] or _iso(now),
updated_at=_iso(now),
)
return copy.deepcopy(run)
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
run = self._runs.get(run_id)
if run is None or run["lease_owner"] != lease_owner or run["status"] != "running":
return False
run["lease_until"] = _iso(lease_until)
run["heartbeat_at"] = _iso(datetime.now(UTC))
return True
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None:
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
mapping = {
"output_json": "output",
"error_json": "error",
"context_json": "context",
"pending_input_json": "pending_input",
}
for key, value in fields.items():
run[mapping.get(key, key)] = value
status = fields.get("status")
if status in TERMINAL_RUN_STATUSES:
run["lease_owner"] = None
run["lease_until"] = None
run["resume_token"] = None
run["pending_input"] = None
run["finished_at"] = run.get("finished_at") or _iso(now)
for key in ("started_at", "finished_at"):
if isinstance(run.get(key), datetime):
run[key] = _iso(run[key])
run["version"] += 1
run["updated_at"] = _iso(now)
return copy.deepcopy(run)
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None:
return None
if run["status"] in ("cancel_requested", "cancelled"):
return copy.deepcopy(run)
if run["status"] not in CANCELLABLE_RUN_STATUSES:
return None
run["status"] = "cancel_requested"
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None or run["status"] != "cancel_requested":
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
run.update(
status="cancelled",
lease_owner=None,
lease_until=None,
resume_token=None,
pending_input=None,
finished_at=_iso(now),
updated_at=_iso(now),
)
run["version"] += 1
return copy.deepcopy(run)
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "running":
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
run.update(
status="awaiting_input",
pending_input=copy.deepcopy(pending_input),
resume_token=resume_token,
lease_owner=None,
lease_until=None,
updated_at=_iso(now),
)
if context is not None:
run["context"] = copy.deepcopy(context)
run["version"] += 1
return copy.deepcopy(run)
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "awaiting_input":
return None
if not run.get("resume_token") or run["resume_token"] != resume_token:
return None
run.update(status="queued", resume_token=None, pending_input=None)
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "awaiting_input":
return None
if not run.get("resume_token") or run["resume_token"] != resume_token:
return None
base_context = context if context is not None else run.get("context") or {}
run.update(
context={**base_context, "resume_payload": copy.deepcopy(payload)},
status="queued",
resume_token=None,
pending_input=None,
)
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
# ── node runs ───────────────────────────────────────────────────────
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
run_id = str(data["run_id"])
node_id = str(data["node_id"])
attempt = int(data.get("attempt") or 1)
iteration = int(data.get("iteration") or 0)
rows = self._node_runs.setdefault(run_id, [])
existing = next(
(r for r in rows if r["node_id"] == node_id and r["attempt"] == attempt and r["iteration"] == iteration),
None,
)
if existing is None:
existing = {
"id": str(data.get("id") or uuid.uuid4()),
"run_id": run_id,
"node_id": node_id,
"node_type": str(data.get("node_type") or "unknown"),
"status": "queued",
"attempt": attempt,
"iteration": iteration,
"is_current": True,
"input": None,
"output": None,
"error": None,
"parent_node_run_id": None,
"duration_ms": None,
"started_at": None,
"finished_at": None,
"created_at": _iso(datetime.now(UTC)),
}
rows.append(existing)
for key in (
"status",
"node_type",
"input",
"output",
"error",
"parent_node_run_id",
"duration_ms",
"started_at",
"finished_at",
):
if key in data and data[key] is not None:
existing[key] = _iso(data[key]) if isinstance(data[key], datetime) else data[key]
if "is_current" in data:
existing["is_current"] = bool(data["is_current"])
return copy.deepcopy(existing)
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
rows = self._node_runs.get(run_id, [])
if current_only:
rows = [r for r in rows if r["is_current"]]
return [copy.deepcopy(r) for r in rows]
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
count = 0
for row in self._node_runs.get(run_id, []):
if row["node_id"] in node_ids and row["is_current"]:
row["is_current"] = False
count += 1
return count
# ── artifacts ───────────────────────────────────────────────────────
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
row = {
"id": str(data.get("id") or uuid.uuid4()),
"run_id": str(data["run_id"]),
"node_id": str(data.get("node_id") or ""),
"name": str(data.get("name") or ""),
"mime_type": str(data.get("mime_type") or "application/octet-stream"),
"path": str(data.get("path") or ""),
"size_bytes": data.get("size_bytes"),
"preview": data.get("preview"),
"created_at": _iso(datetime.now(UTC)),
}
self._artifacts.setdefault(row["run_id"], []).append(row)
return copy.deepcopy(row)
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
return [copy.deepcopy(r) for r in self._artifacts.get(run_id, [])]
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
for row in self._artifacts.get(run_id, []):
if row["id"] == artifact_id:
return copy.deepcopy(row)
return None
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._lock:
deleted: list[str] = []
for run_id, run in list(self._runs.items()):
if run["status"] not in TERMINAL_RUN_STATUSES:
continue
finished = _parse_dt(run.get("finished_at"))
if finished is None or finished >= cutoff:
continue
deleted.append(run_id)
for run_id in deleted:
self._runs.pop(run_id, None)
self._node_runs.pop(run_id, None)
self._artifacts.pop(run_id, None)
return deleted
def _parse_dt(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, str) and value:
try:
parsed = datetime.fromisoformat(value)
except ValueError:
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
return None
@@ -0,0 +1,96 @@
"""ORM models for workflow runs and per-node runs."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class WorkflowRunRow(Base):
"""One workflow execution bound to an immutable published version."""
__tablename__ = "workflow_runs"
__table_args__ = (UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
workflow_version_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# queued | running | awaiting_input | completed | failed | cancel_requested | cancelled
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
input_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
output_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
# Non-sensitive run context (never credentials).
context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
# Pending human-input descriptor when status == awaiting_input.
pending_input_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
resume_token: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
heartbeat_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3, server_default=sa_text("3"))
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
# Last seq handed out by the event log for this run (0 = none yet). The
# event store increments it in the same transaction that inserts the event
# row, replacing the MAX(seq)+1 race; "0" backfills existing rows and the
# first append self-heals from MAX(seq) on conflict.
next_event_seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
retry_of_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
class WorkflowNodeRunRow(Base):
"""Per-node execution record (also the resume checkpoint for a paused run)."""
__tablename__ = "workflow_node_runs"
__table_args__ = (UniqueConstraint("run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
node_type: Mapped[str] = mapped_column(String(32), nullable=False)
# queued | running | completed | failed | skipped | cancelled
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default=sa_text("1"))
iteration: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
is_current: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
output_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
parent_node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class WorkflowRunArtifactRow(Base):
"""Artifact reference produced by a run (bytes live in the sandbox/thread dir)."""
__tablename__ = "workflow_run_artifacts"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
name: Mapped[str] = mapped_column(String(512), nullable=False)
mime_type: Mapped[str] = mapped_column(String(191), nullable=False, default="application/octet-stream")
path: Mapped[str] = mapped_column(Text, nullable=False, default="")
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
preview: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -0,0 +1,566 @@
"""SQLAlchemy-backed workflow run store (lease dispatch + resume checkpoints)."""
from __future__ import annotations
import json
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunStore,
)
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
_RUN_UPDATABLE = {
"status",
"output_json",
"error_json",
"context_json",
"pending_input_json",
"resume_token",
"started_at",
"finished_at",
}
# Retention deletes in bounded batches so a long-neglected table cannot produce
# one enormous transaction.
_PURGE_BATCH = 500
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _load_text_json(raw: str | None) -> Any:
if not raw:
return None
try:
return json.loads(raw)
except (TypeError, json.JSONDecodeError):
return None
def _dump_text_json(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
def run_to_dict(row: WorkflowRunRow) -> dict[str, Any]:
return {
"id": row.id,
"workflow_id": row.workflow_id,
"workflow_version_id": row.workflow_version_id,
"status": row.status,
"owner_id": row.owner_id,
"idempotency_key": row.idempotency_key,
"input": row.input_json or {},
"output": row.output_json,
"error": row.error_json,
"context": row.context_json or {},
"pending_input": row.pending_input_json,
"resume_token": row.resume_token,
"lease_owner": row.lease_owner,
"lease_until": _iso(row.lease_until),
"heartbeat_at": _iso(row.heartbeat_at),
"attempt": row.attempt,
"max_attempts": row.max_attempts,
"version": row.version,
"retry_of_run_id": row.retry_of_run_id,
# Last seq handed out by the event log (0 = no events yet).
"next_event_seq": int(row.next_event_seq or 0),
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
def node_run_to_dict(row: WorkflowNodeRunRow) -> dict[str, Any]:
return {
"id": row.id,
"run_id": row.run_id,
"node_id": row.node_id,
"node_type": row.node_type,
"status": row.status,
"attempt": row.attempt,
"iteration": row.iteration,
"is_current": bool(row.is_current),
"input": _load_text_json(row.input_snapshot),
"output": _load_text_json(row.output_snapshot),
"error": row.error_json,
"parent_node_run_id": row.parent_node_run_id,
"duration_ms": row.duration_ms,
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"created_at": _iso(row.created_at),
}
def artifact_to_dict(row: WorkflowRunArtifactRow) -> dict[str, Any]:
return {
"id": row.id,
"run_id": row.run_id,
"node_id": row.node_id,
"name": row.name,
"mime_type": row.mime_type,
"path": row.path,
"size_bytes": row.size_bytes,
"preview": row.preview,
"created_at": _iso(row.created_at),
}
class SqlWorkflowRunStore(WorkflowRunStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
# ── runs ────────────────────────────────────────────────────────────
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
key = data.get("idempotency_key")
if key:
existing = await self._get_by_idempotency(str(key))
if existing is not None:
return existing, False
row = WorkflowRunRow(
id=str(data.get("id") or uuid.uuid4()),
workflow_id=str(data["workflow_id"]),
workflow_version_id=str(data["workflow_version_id"]),
status=str(data.get("status") or "queued"),
owner_id=str(data["owner_id"]),
idempotency_key=str(key) if key else None,
input_json=data.get("input") or {},
context_json=data.get("context") or {},
max_attempts=int(data.get("max_attempts") or 3),
retry_of_run_id=data.get("retry_of_run_id"),
)
async with self._sf() as session:
session.add(row)
try:
await session.flush()
except IntegrityError:
await session.rollback()
if key:
existing = await self._get_by_idempotency(str(key))
if existing is not None:
return existing, False
raise
out = run_to_dict(row)
await session.commit()
return out, True
async def _get_by_idempotency(self, key: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = (await session.execute(select(WorkflowRunRow).where(WorkflowRunRow.idempotency_key == key))).scalar_one_or_none()
return run_to_dict(row) if row else None
async def get_run(self, run_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
stmt = select(WorkflowRunRow)
if workflow_id:
stmt = stmt.where(WorkflowRunRow.workflow_id == workflow_id)
if owner_id:
stmt = stmt.where(WorkflowRunRow.owner_id == owner_id)
if status:
stmt = stmt.where(WorkflowRunRow.status == status)
stmt = stmt.order_by(WorkflowRunRow.created_at.desc()).limit(limit).offset(offset)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [run_to_dict(r) for r in rows]
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
stmt = (
select(WorkflowRunRow.id)
.where(
or_(
WorkflowRunRow.status == "queued",
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
)
)
.order_by(WorkflowRunRow.created_at.asc())
.limit(limit)
)
async with self._sf() as session:
return [str(v) for v in (await session.execute(stmt)).scalars().all()]
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
or_(
WorkflowRunRow.status == "queued",
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
),
)
.values(
status="running",
lease_owner=lease_owner,
lease_until=lease_until,
heartbeat_at=now,
attempt=WorkflowRunRow.attempt + 1,
version=WorkflowRunRow.version + 1,
started_at=func.coalesce(WorkflowRunRow.started_at, now),
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.lease_owner == lease_owner,
WorkflowRunRow.status == "running",
)
.values(lease_until=lease_until, heartbeat_at=now, updated_at=now)
)
if result.rowcount == 0:
await session.rollback()
return False
await session.commit()
return True
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
values: dict[str, Any] = {"updated_at": now, "version": WorkflowRunRow.version + 1}
for key, value in fields.items():
if key in _RUN_UPDATABLE:
values[key] = value
status = fields.get("status")
if status in TERMINAL_RUN_STATUSES:
values["lease_owner"] = None
values["lease_until"] = None
values["resume_token"] = None
values["pending_input_json"] = None
values.setdefault("finished_at", now)
where = [WorkflowRunRow.id == run_id]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
async with self._sf() as session:
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status.in_(list(CANCELLABLE_RUN_STATUSES)),
)
.values(status="cancel_requested", version=WorkflowRunRow.version + 1, updated_at=now)
)
if result.rowcount == 0:
await session.rollback()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row and row.status in ("cancel_requested", "cancelled") else None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
now = datetime.now(UTC)
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "cancel_requested"]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(*where)
.values(
status="cancelled",
lease_owner=None,
lease_until=None,
resume_token=None,
pending_input_json=None,
finished_at=now,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "running"]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
values: dict[str, Any] = {
"status": "awaiting_input",
"pending_input_json": pending_input,
"resume_token": resume_token,
"lease_owner": None,
"lease_until": None,
"version": WorkflowRunRow.version + 1,
"updated_at": now,
}
if context is not None:
values["context_json"] = context
async with self._sf() as session:
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status == "awaiting_input",
WorkflowRunRow.resume_token == resume_token,
)
.values(
status="queued",
resume_token=None,
pending_input_json=None,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
if context is None:
async with self._sf() as session:
row = await session.get(WorkflowRunRow, run_id)
if row is None:
return None
context = dict(row.context_json or {})
new_context = {**context, "resume_payload": payload}
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status == "awaiting_input",
WorkflowRunRow.resume_token == resume_token,
)
.values(
context_json=new_context,
status="queued",
resume_token=None,
pending_input_json=None,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
# ── node runs ───────────────────────────────────────────────────────
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
run_id = str(data["run_id"])
node_id = str(data["node_id"])
attempt = int(data.get("attempt") or 1)
iteration = int(data.get("iteration") or 0)
async with self._sf() as session:
row = (
await session.execute(
select(WorkflowNodeRunRow).where(
WorkflowNodeRunRow.run_id == run_id,
WorkflowNodeRunRow.node_id == node_id,
WorkflowNodeRunRow.attempt == attempt,
WorkflowNodeRunRow.iteration == iteration,
)
)
).scalar_one_or_none()
if row is None:
row = WorkflowNodeRunRow(
id=str(data.get("id") or uuid.uuid4()),
run_id=run_id,
node_id=node_id,
node_type=str(data.get("node_type") or "unknown"),
attempt=attempt,
iteration=iteration,
)
session.add(row)
row.status = str(data.get("status") or row.status)
row.node_type = str(data.get("node_type") or row.node_type)
row.is_current = bool(data.get("is_current", True))
if "input" in data:
row.input_snapshot = _dump_text_json(data.get("input"))
if "output" in data:
row.output_snapshot = _dump_text_json(data.get("output"))
if "error" in data:
row.error_json = data.get("error")
if data.get("parent_node_run_id"):
row.parent_node_run_id = str(data["parent_node_run_id"])
if data.get("duration_ms") is not None:
row.duration_ms = int(data["duration_ms"])
if data.get("started_at") is not None:
row.started_at = data["started_at"]
if data.get("finished_at") is not None:
row.finished_at = data["finished_at"]
await session.flush()
out = node_run_to_dict(row)
await session.commit()
return out
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
stmt = select(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id == run_id)
if current_only:
stmt = stmt.where(WorkflowNodeRunRow.is_current.is_(True))
stmt = stmt.order_by(WorkflowNodeRunRow.created_at.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [node_run_to_dict(r) for r in rows]
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
if not node_ids:
return 0
async with self._sf() as session:
result = await session.execute(
update(WorkflowNodeRunRow)
.where(
WorkflowNodeRunRow.run_id == run_id,
WorkflowNodeRunRow.node_id.in_(node_ids),
WorkflowNodeRunRow.is_current.is_(True),
)
.values(is_current=False)
)
await session.commit()
return int(result.rowcount or 0)
# ── artifacts ───────────────────────────────────────────────────────
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
row = WorkflowRunArtifactRow(
id=str(data.get("id") or uuid.uuid4()),
run_id=str(data["run_id"]),
node_id=str(data.get("node_id") or ""),
name=str(data.get("name") or "")[:512],
mime_type=str(data.get("mime_type") or "application/octet-stream")[:191],
path=str(data.get("path") or ""),
size_bytes=data.get("size_bytes"),
preview=data.get("preview"),
)
async with self._sf() as session:
session.add(row)
await session.flush()
out = artifact_to_dict(row)
await session.commit()
return out
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
stmt = select(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id == run_id).order_by(WorkflowRunArtifactRow.created_at.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [artifact_to_dict(r) for r in rows]
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
stmt = select(WorkflowRunArtifactRow).where(
WorkflowRunArtifactRow.id == artifact_id,
WorkflowRunArtifactRow.run_id == run_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return artifact_to_dict(row) if row is not None else None
# ── retention ───────────────────────────────────────────────────────
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._sf() as session:
run_ids = (
(
await session.execute(
select(WorkflowRunRow.id)
.where(
WorkflowRunRow.status.in_(TERMINAL_RUN_STATUSES),
WorkflowRunRow.finished_at.is_not(None),
WorkflowRunRow.finished_at < cutoff,
)
.limit(_PURGE_BATCH)
)
)
.scalars()
.all()
)
if not run_ids:
return []
ids = list(run_ids)
await session.execute(delete(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id.in_(ids)))
await session.execute(delete(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id.in_(ids)))
await session.execute(delete(WorkflowRunRow).where(WorkflowRunRow.id.in_(ids)))
await session.commit()
return ids
@@ -0,0 +1,31 @@
"""Workflow definition / version persistence."""
from __future__ import annotations
from deerflow.persistence.workflows.base import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
WorkflowVersionNotFoundError,
)
from deerflow.persistence.workflows.memory import MemoryWorkflowStore
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
from deerflow.persistence.workflows.sql import WorkflowRepository
__all__ = [
"MemoryWorkflowStore",
"WorkflowDefinitionRow",
"WorkflowDraftConflictError",
"WorkflowNotFoundError",
"WorkflowRepository",
"WorkflowStore",
"WorkflowVersionNotFoundError",
"WorkflowVersionRow",
"make_workflow_store",
]
def make_workflow_store(session_factory):
if session_factory is None:
return MemoryWorkflowStore()
return WorkflowRepository(session_factory)
@@ -0,0 +1,112 @@
"""Abstract workflow definition / version store."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class WorkflowDraftConflictError(Exception):
"""Raised when ``expected_revision`` does not match the stored draft revision."""
def __init__(self, workflow_id: str, current_revision: int) -> None:
super().__init__(f"draft conflict for {workflow_id}: current={current_revision}")
self.workflow_id = workflow_id
self.current_revision = current_revision
class WorkflowNotFoundError(Exception):
def __init__(self, workflow_id: str) -> None:
super().__init__(f"workflow not found: {workflow_id}")
self.workflow_id = workflow_id
class WorkflowVersionNotFoundError(Exception):
def __init__(self, version_id: str) -> None:
super().__init__(f"workflow version not found: {version_id}")
self.version_id = version_id
class WorkflowStore(ABC):
@abstractmethod
async def list_definitions(
self,
*,
owner_id: str | None = None,
status: str | None = None,
include_archived: bool = False,
) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def save_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
updated_by: str | None = None,
) -> dict[str, Any]:
"""CAS update of ``draft_graph_json``. Raises ``WorkflowDraftConflictError``."""
raise NotImplementedError
@abstractmethod
async def save_studio_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
canvas_schema: str = "{}",
updated_by: str | None = None,
) -> dict[str, Any]:
"""Atomically CAS-update BOTH the execution graph and the canvas document.
A single conditional UPDATE bumps ``draft_revision`` and writes
``draft_graph_json`` + ``draft_canvas_schema_json`` together, so the two
representations can never diverge across revisions. Raises
``WorkflowDraftConflictError`` on a stale ``expected_revision``.
"""
raise NotImplementedError
@abstractmethod
async def publish_version(
self,
workflow_id: str,
*,
graph: dict[str, Any],
graph_hash: str,
published_by: str | None = None,
change_note: str = "",
canvas_schema_hash: str | None = None,
) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_version(self, version_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
"""Lightweight list for subworkflow resource catalog."""
raise NotImplementedError
@@ -0,0 +1,216 @@
"""In-memory workflow store for tests / no-DB mode."""
from __future__ import annotations
import copy
import uuid
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.workflows.base import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
)
def _now() -> str:
return datetime.now(UTC).isoformat()
class MemoryWorkflowStore(WorkflowStore):
def __init__(self) -> None:
self._definitions: dict[str, dict[str, Any]] = {}
self._versions: dict[str, dict[str, Any]] = {}
async def list_definitions(
self,
*,
owner_id: str | None = None,
status: str | None = None,
include_archived: bool = False,
) -> list[dict[str, Any]]:
rows = list(self._definitions.values())
if owner_id:
rows = [r for r in rows if r["owner_id"] == owner_id]
if status:
rows = [r for r in rows if r["status"] == status]
elif not include_archived:
rows = [r for r in rows if r["status"] != "archived"]
rows.sort(key=lambda r: r.get("updated_at") or "", reverse=True)
return [
{k: v for k, v in r.items() if k not in ("draft_graph", "draft_canvas_schema", "next_version_number")}
for r in rows
]
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
row = self._definitions.get(workflow_id)
if row is None:
return None
out = copy.deepcopy(row)
if not include_draft:
out.pop("draft_graph", None)
return out
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
graph = copy.deepcopy(data.get("draft_graph") or data.get("graph") or {
"schemaVersion": "1.0",
"id": workflow_id,
"name": data.get("name") or "未命名工作流",
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
"edges": [],
"inputSchema": {"type": "object", "properties": {}},
"outputSchema": {"type": "object", "properties": {}},
"settings": {},
})
graph.setdefault("id", workflow_id)
now = _now()
row = {
"id": workflow_id,
"name": data.get("name") or graph.get("name") or "未命名工作流",
"description": data.get("description") or "",
"owner_id": data["owner_id"],
"status": data.get("status") or "draft",
"draft_revision": 0,
"next_version_number": 1,
"draft_graph": graph,
"draft_canvas_schema": str(data.get("draft_canvas_schema") or "{}"),
"created_at": now,
"updated_at": now,
}
self._definitions[workflow_id] = row
return copy.deepcopy(row)
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._definitions.get(workflow_id)
if row is None:
return None
if "name" in data and data["name"] is not None:
row["name"] = data["name"]
if "description" in data and data["description"] is not None:
row["description"] = data["description"]
if "status" in data and data["status"] is not None:
row["status"] = data["status"]
row["updated_at"] = _now()
return copy.deepcopy(row)
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
return await self.update_definition(workflow_id, {"status": "archived"})
async def save_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by
row = self._definitions.get(workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row["draft_revision"] != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
payload = copy.deepcopy(graph)
payload.setdefault("id", workflow_id)
row["draft_graph"] = payload
row["draft_revision"] = expected_revision + 1
if payload.get("name"):
row["name"] = str(payload["name"])[:255]
row["updated_at"] = _now()
return copy.deepcopy(row)
async def save_studio_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
canvas_schema: str = "{}",
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by
row = self._definitions.get(workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row["draft_revision"] != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
payload = copy.deepcopy(graph)
payload.setdefault("id", workflow_id)
row["draft_graph"] = payload
row["draft_canvas_schema"] = canvas_schema or "{}"
row["draft_revision"] = expected_revision + 1
if payload.get("name"):
row["name"] = str(payload["name"])[:255]
row["updated_at"] = _now()
return copy.deepcopy(row)
async def publish_version(
self,
workflow_id: str,
*,
graph: dict[str, Any],
graph_hash: str,
published_by: str | None = None,
change_note: str = "",
canvas_schema_hash: str | None = None,
) -> dict[str, Any]:
definition = self._definitions.get(workflow_id)
if definition is None:
raise WorkflowNotFoundError(workflow_id)
existing = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
# Allocate from the definition's own counter (mirrors the SQL store) so
# the number never goes backwards even if events are purged/replayed.
counter = int(definition.get("next_version_number") or 1)
version_number = max(counter, max((v["version_number"] for v in existing), default=0) + 1)
definition["next_version_number"] = version_number + 1
version_id = f"wv_{uuid.uuid4().hex[:12]}"
version = {
"id": version_id,
"workflow_id": workflow_id,
"version_number": version_number,
"graph": copy.deepcopy(graph),
"graph_hash": graph_hash,
"input_schema": graph.get("inputSchema") or graph.get("input_schema") or {},
"output_schema": graph.get("outputSchema") or graph.get("output_schema") or {},
"published_by": published_by,
"published_at": _now(),
"change_note": change_note or "",
}
self._versions[version_id] = version
definition["status"] = "active"
if canvas_schema_hash:
definition["published_canvas_schema_hash"] = str(canvas_schema_hash)[:64]
definition["updated_at"] = _now()
return copy.deepcopy(version)
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
rows = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
rows.sort(key=lambda r: r["version_number"], reverse=True)
return [{k: v for k, v in r.items() if k != "graph"} for r in rows]
async def get_version(self, version_id: str) -> dict[str, Any] | None:
row = self._versions.get(version_id)
return copy.deepcopy(row) if row is not None else None
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
by_wf: dict[str, dict[str, Any]] = {}
for version in self._versions.values():
wf_id = version["workflow_id"]
if exclude_workflow_id and wf_id == exclude_workflow_id:
continue
definition = self._definitions.get(wf_id)
if definition is None or definition.get("status") == "archived":
continue
current = by_wf.get(wf_id)
if current is None or version["version_number"] > current["version_number"]:
by_wf[wf_id] = {
"workflow_id": wf_id,
"name": definition["name"],
"version_id": version["id"],
"version_number": version["version_number"],
"graph_hash": version["graph_hash"],
"published_at": version["published_at"],
}
return sorted(by_wf.values(), key=lambda r: r.get("published_at") or "", reverse=True)
@@ -0,0 +1,69 @@
"""ORM models for workflow definitions and immutable published versions."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class WorkflowDefinitionRow(Base):
"""Mutable workflow metadata + current draft canvas."""
__tablename__ = "workflow_definitions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str] = mapped_column(Text, default="", nullable=False)
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# draft | active | archived
status: Mapped[str] = mapped_column(String(32), default="draft", nullable=False, index=True)
draft_revision: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
# Monotonic version allocator. "1" = no version published yet; publish takes
# the value then CAS-bumps it, replacing the old MAX(version_number)+1 race.
next_version_number: Mapped[int] = mapped_column(
Integer, nullable=False, default=1, server_default=sa_text("1")
)
draft_graph_json: Mapped[str] = mapped_column(PortableLongText(), default="{}", nullable=False)
# Coze canvas document (opaque JSON text). Never interpreted by the runtime;
# the executable WorkflowGraph lives in draft_graph_json above.
draft_canvas_schema_json: Mapped[str] = mapped_column(
PortableLongText(), default="{}", server_default=sa_text("'{}'"), nullable=False
)
# sha256 of the canvas document the current published version was compiled
# from (audit trail only; publish always freezes the execution graph).
published_canvas_schema_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
class WorkflowVersionRow(Base):
"""Immutable published graph snapshot. Runs bind to version id."""
__tablename__ = "workflow_versions"
__table_args__ = (UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
workflow_id: Mapped[str] = mapped_column(
String(64),
ForeignKey("workflow_definitions.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
graph_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
graph_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
input_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
output_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
published_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
change_note: Mapped[str] = mapped_column(Text, default="", nullable=False)
@@ -0,0 +1,385 @@
"""SQLAlchemy-backed workflow definition / version store."""
from __future__ import annotations
import json
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.workflows.base import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
)
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
# Concurrent publishers CAS-retry against the workflow's own counter; a handful
# of attempts is far more than any realistic contention window needs.
_PUBLISH_ATTEMPTS = 6
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _parse_graph(raw: str | dict[str, Any] | None) -> dict[str, Any]:
if raw is None:
return {}
if isinstance(raw, dict):
return raw
text = str(raw).strip()
if not text:
return {}
try:
data = json.loads(text)
except json.JSONDecodeError:
return {}
return data if isinstance(data, dict) else {}
def _dump_graph(graph: dict[str, Any]) -> str:
return json.dumps(graph, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def _definition_dict(row: WorkflowDefinitionRow, *, include_draft: bool) -> dict[str, Any]:
data = {
"id": row.id,
"name": row.name,
"description": row.description or "",
"owner_id": row.owner_id,
"status": row.status,
"draft_revision": row.draft_revision,
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
if include_draft:
data["draft_graph"] = _parse_graph(row.draft_graph_json)
# Canvas document as stored (opaque JSON text; "{}" when never saved).
data["draft_canvas_schema"] = row.draft_canvas_schema_json or "{}"
return data
def _version_dict(row: WorkflowVersionRow, *, include_graph: bool = True) -> dict[str, Any]:
data = {
"id": row.id,
"workflow_id": row.workflow_id,
"version_number": row.version_number,
"graph_hash": row.graph_hash,
"input_schema": row.input_schema or {},
"output_schema": row.output_schema or {},
"published_by": row.published_by,
"published_at": _iso(row.published_at),
"change_note": row.change_note or "",
}
if include_graph:
data["graph"] = _parse_graph(row.graph_json)
return data
class WorkflowRepository(WorkflowStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_definitions(
self,
*,
owner_id: str | None = None,
status: str | None = None,
include_archived: bool = False,
) -> list[dict[str, Any]]:
stmt = select(WorkflowDefinitionRow)
if owner_id:
stmt = stmt.where(WorkflowDefinitionRow.owner_id == owner_id)
if status:
stmt = stmt.where(WorkflowDefinitionRow.status == status)
elif not include_archived:
stmt = stmt.where(WorkflowDefinitionRow.status != "archived")
stmt = stmt.order_by(WorkflowDefinitionRow.updated_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_definition_dict(row, include_draft=False) for row in result.scalars()]
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
return _definition_dict(row, include_draft=include_draft) if row is not None else None
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
graph = data.get("draft_graph") or data.get("graph") or {
"schemaVersion": "1.0",
"id": workflow_id,
"name": data.get("name") or "未命名工作流",
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
"edges": [],
"inputSchema": {"type": "object", "properties": {}},
"outputSchema": {"type": "object", "properties": {}},
"settings": {},
}
if isinstance(graph, dict) and not graph.get("id"):
graph = {**graph, "id": workflow_id}
if isinstance(graph, dict) and data.get("name") and not graph.get("name"):
graph = {**graph, "name": data["name"]}
row = WorkflowDefinitionRow(
id=workflow_id,
name=data.get("name") or graph.get("name") or "未命名工作流",
description=data.get("description") or "",
owner_id=data["owner_id"],
status=data.get("status") or "draft",
draft_revision=0,
draft_graph_json=_dump_graph(graph),
draft_canvas_schema_json=str(data.get("draft_canvas_schema") or "{}"),
)
async with self._sf() as session:
session.add(row)
await session.flush()
out = _definition_dict(row, include_draft=True)
await session.commit()
return out
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
if row is None:
return None
if "name" in data and data["name"] is not None:
row.name = data["name"]
if "description" in data and data["description"] is not None:
row.description = data["description"]
if "status" in data and data["status"] is not None:
row.status = data["status"]
row.updated_at = datetime.now(UTC)
await session.flush()
out = _definition_dict(row, include_draft=True)
await session.commit()
return out
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
return await self.update_definition(workflow_id, {"status": "archived"})
async def save_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by # audit reserved for later
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row.draft_revision != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
payload = dict(graph)
payload.setdefault("id", workflow_id)
if row.name and not payload.get("name"):
payload["name"] = row.name
# Conditional UPDATE on the revision itself: two sessions that both
# read the same revision cannot both win — the loser's WHERE clause
# matches zero rows and surfaces a deterministic conflict instead of
# silently overwriting the other editor's draft.
values: dict[str, Any] = {
"draft_graph_json": _dump_graph(payload),
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
"updated_at": datetime.now(UTC),
}
if payload.get("name"):
values["name"] = str(payload["name"])[:255]
result = await session.execute(
update(WorkflowDefinitionRow)
.where(
WorkflowDefinitionRow.id == workflow_id,
WorkflowDefinitionRow.draft_revision == expected_revision,
)
.values(**values)
)
if result.rowcount == 0:
await session.rollback()
current = await session.get(WorkflowDefinitionRow, workflow_id)
if current is None:
raise WorkflowNotFoundError(workflow_id)
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
await session.commit()
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
assert fresh is not None
return _definition_dict(fresh, include_draft=True)
async def save_studio_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
canvas_schema: str = "{}",
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by # audit reserved for later
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row.draft_revision != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
payload = dict(graph)
payload.setdefault("id", workflow_id)
if row.name and not payload.get("name"):
payload["name"] = row.name
# One conditional UPDATE writes graph + canvas + revision together:
# the two representations are always from the same save, and a
# concurrent editor loses deterministically instead of interleaving
# a new canvas onto someone else's graph.
values: dict[str, Any] = {
"draft_graph_json": _dump_graph(payload),
"draft_canvas_schema_json": canvas_schema or "{}",
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
"updated_at": datetime.now(UTC),
}
if payload.get("name"):
values["name"] = str(payload["name"])[:255]
result = await session.execute(
update(WorkflowDefinitionRow)
.where(
WorkflowDefinitionRow.id == workflow_id,
WorkflowDefinitionRow.draft_revision == expected_revision,
)
.values(**values)
)
if result.rowcount == 0:
await session.rollback()
current = await session.get(WorkflowDefinitionRow, workflow_id)
if current is None:
raise WorkflowNotFoundError(workflow_id)
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
await session.commit()
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
assert fresh is not None
return _definition_dict(fresh, include_draft=True)
async def publish_version(
self,
workflow_id: str,
*,
graph: dict[str, Any],
graph_hash: str,
published_by: str | None = None,
change_note: str = "",
canvas_schema_hash: str | None = None,
) -> dict[str, Any]:
"""Allocate the version number from the workflow's own counter.
``MAX(version_number) + 1`` races between concurrent publishes; here the
number is claimed by CAS-bumping ``next_version_number`` inside the same
transaction that inserts the version row, and the unique
``(workflow_id, version_number)`` constraint is the final safety net.
Concurrent publishers get adjacent, distinct numbers.
"""
last_error: Exception | None = None
for _ in range(_PUBLISH_ATTEMPTS):
try:
async with self._sf() as session:
definition = await session.get(WorkflowDefinitionRow, workflow_id)
if definition is None:
raise WorkflowNotFoundError(workflow_id)
counter_seen = int(definition.next_version_number or 1)
# Legacy rows got the column auto-added with server_default 1
# while versions already exist — never allocate below the max.
latest = (
await session.execute(
select(func.max(WorkflowVersionRow.version_number)).where(
WorkflowVersionRow.workflow_id == workflow_id
)
)
).scalar() or 0
number = max(counter_seen, int(latest) + 1)
claimed = await session.execute(
update(WorkflowDefinitionRow)
.where(
WorkflowDefinitionRow.id == workflow_id,
WorkflowDefinitionRow.next_version_number == counter_seen,
)
.values(next_version_number=number + 1)
)
if claimed.rowcount == 0:
# Another publish landed first — release nothing (the
# counter bump never happened) and retry with fresh state.
await session.rollback()
last_error = None
continue
version = WorkflowVersionRow(
id=f"wv_{uuid.uuid4().hex[:12]}",
workflow_id=workflow_id,
version_number=number,
graph_json=_dump_graph(graph),
graph_hash=graph_hash,
input_schema=graph.get("inputSchema") or graph.get("input_schema") or {},
output_schema=graph.get("outputSchema") or graph.get("output_schema") or {},
published_by=published_by,
change_note=change_note or "",
)
definition.status = "active"
definition.updated_at = datetime.now(UTC)
if canvas_schema_hash:
definition.published_canvas_schema_hash = canvas_schema_hash[:64]
session.add(version)
await session.flush()
out = _version_dict(version, include_graph=True)
await session.commit()
return out
except IntegrityError as exc: # pragma: no cover - CAS makes this near-impossible
last_error = exc
continue
if last_error is not None:
raise last_error
raise RuntimeError(f"failed to allocate a workflow version number for {workflow_id}")
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
stmt = (
select(WorkflowVersionRow)
.where(WorkflowVersionRow.workflow_id == workflow_id)
.order_by(WorkflowVersionRow.version_number.desc())
)
async with self._sf() as session:
result = await session.execute(stmt)
return [_version_dict(row, include_graph=False) for row in result.scalars()]
async def get_version(self, version_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowVersionRow, version_id)
return _version_dict(row, include_graph=True) if row is not None else None
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
stmt = (
select(WorkflowVersionRow, WorkflowDefinitionRow)
.join(WorkflowDefinitionRow, WorkflowDefinitionRow.id == WorkflowVersionRow.workflow_id)
.where(WorkflowDefinitionRow.status != "archived")
.order_by(WorkflowVersionRow.published_at.desc())
)
if exclude_workflow_id:
stmt = stmt.where(WorkflowVersionRow.workflow_id != exclude_workflow_id)
async with self._sf() as session:
result = await session.execute(stmt)
out: list[dict[str, Any]] = []
seen: set[str] = set()
for version, definition in result.all():
# Latest version per workflow only.
if version.workflow_id in seen:
continue
seen.add(version.workflow_id)
out.append(
{
"workflow_id": definition.id,
"name": definition.name,
"version_id": version.id,
"version_number": version.version_number,
"graph_hash": version.graph_hash,
"published_at": _iso(version.published_at),
}
)
return out
@@ -0,0 +1,64 @@
"""Workflow Studio protocol package (phase 0 — frozen contracts).
Execution, persistence, and Gateway routers land in later phases.
This package must never import ``app.*``.
"""
from deerflow.workflows.errors import (
ALL_WORKFLOW_ERROR_CODES,
WorkflowError,
WorkflowErrorBody,
WorkflowErrorCode,
)
from deerflow.workflows.events import (
ALL_WORKFLOW_EVENT_TYPES,
TERMINAL_EVENT_TYPES,
WorkflowEventEnvelope,
WorkflowEventSink,
WorkflowEventType,
)
from deerflow.workflows.schemas import (
ALL_NODE_TYPES,
ALL_RUN_STATUSES,
SCHEMA_VERSION,
TERMINAL_RUN_STATUSES,
ArtifactRef,
NodeResult,
NodeType,
ResumeRunRequest,
RunStatus,
StartRunRequest,
WorkflowEdge,
WorkflowGraph,
WorkflowGraphSettings,
WorkflowNode,
)
from deerflow.workflows.validator import assert_publishable, validate_workflow_graph
__all__ = [
"ALL_NODE_TYPES",
"ALL_RUN_STATUSES",
"ALL_WORKFLOW_ERROR_CODES",
"ALL_WORKFLOW_EVENT_TYPES",
"SCHEMA_VERSION",
"TERMINAL_EVENT_TYPES",
"TERMINAL_RUN_STATUSES",
"ArtifactRef",
"NodeResult",
"NodeType",
"ResumeRunRequest",
"RunStatus",
"StartRunRequest",
"WorkflowEdge",
"WorkflowError",
"WorkflowErrorBody",
"WorkflowErrorCode",
"WorkflowEventEnvelope",
"WorkflowEventSink",
"WorkflowEventType",
"WorkflowGraph",
"WorkflowGraphSettings",
"WorkflowNode",
"assert_publishable",
"validate_workflow_graph",
]
@@ -0,0 +1,150 @@
"""Stable workflow error codes and exception types (protocol v1).
User-visible ``message`` must never include stack traces, SQL passwords,
full response bodies, or internal host paths. Full exceptions go to
controlled server logs only.
"""
from __future__ import annotations
from typing import Any, Literal
from pydantic import BaseModel, Field
WorkflowErrorCode = Literal[
# graph / publish
"WORKFLOW_SCHEMA_INVALID",
"WORKFLOW_GRAPH_CYCLE",
"WORKFLOW_NODE_NOT_FOUND",
"WORKFLOW_EDGE_INVALID",
"WORKFLOW_EXPRESSION_INVALID",
"WORKFLOW_EXPRESSION_UPSTREAM_REQUIRED",
"WORKFLOW_RESOURCE_MISSING",
"WORKFLOW_RESOURCE_FORBIDDEN",
"WORKFLOW_LIMIT_EXCEEDED",
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"WORKFLOW_DRAFT_CONFLICT",
"WORKFLOW_VERSION_NOT_FOUND",
# run lifecycle
"WORKFLOW_RUN_NOT_FOUND",
"WORKFLOW_RUN_CONFLICT",
"WORKFLOW_RUN_NOT_RESUMABLE",
"WORKFLOW_RESUME_TOKEN_INVALID",
"WORKFLOW_CANCELLED",
"WORKFLOW_TIMEOUT",
"WORKFLOW_MAX_STEPS",
"WORKFLOW_MAX_LOOP_ITERATIONS",
"WORKFLOW_BUDGET_EXCEEDED",
# node categories
"WORKFLOW_INPUT_INVALID",
"WORKFLOW_AGENT_FAILED",
"WORKFLOW_SKILL_FAILED",
"WORKFLOW_HTTP_POLICY_DENIED",
"WORKFLOW_HTTP_FAILED",
"WORKFLOW_SQL_POLICY_DENIED",
"WORKFLOW_SQL_FAILED",
"WORKFLOW_CODE_SANDBOX_FAILED",
"WORKFLOW_TRANSFORM_FAILED",
"WORKFLOW_CONDITION_FAILED",
"WORKFLOW_MERGE_FAILED",
"WORKFLOW_HUMAN_INPUT_INVALID",
"WORKFLOW_SUBWORKFLOW_FAILED",
"WORKFLOW_LOOP_FAILED",
# auth / embed
"WORKFLOW_FORBIDDEN",
"WORKFLOW_EMBED_TICKET_INVALID",
"WORKFLOW_EMBED_ORIGIN_DENIED",
# generic
"WORKFLOW_INTERNAL",
]
ALL_WORKFLOW_ERROR_CODES: tuple[WorkflowErrorCode, ...] = (
"WORKFLOW_SCHEMA_INVALID",
"WORKFLOW_GRAPH_CYCLE",
"WORKFLOW_NODE_NOT_FOUND",
"WORKFLOW_EDGE_INVALID",
"WORKFLOW_EXPRESSION_INVALID",
"WORKFLOW_EXPRESSION_UPSTREAM_REQUIRED",
"WORKFLOW_RESOURCE_MISSING",
"WORKFLOW_RESOURCE_FORBIDDEN",
"WORKFLOW_LIMIT_EXCEEDED",
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"WORKFLOW_DRAFT_CONFLICT",
"WORKFLOW_VERSION_NOT_FOUND",
"WORKFLOW_RUN_NOT_FOUND",
"WORKFLOW_RUN_CONFLICT",
"WORKFLOW_RUN_NOT_RESUMABLE",
"WORKFLOW_RESUME_TOKEN_INVALID",
"WORKFLOW_CANCELLED",
"WORKFLOW_TIMEOUT",
"WORKFLOW_MAX_STEPS",
"WORKFLOW_MAX_LOOP_ITERATIONS",
"WORKFLOW_BUDGET_EXCEEDED",
"WORKFLOW_INPUT_INVALID",
"WORKFLOW_AGENT_FAILED",
"WORKFLOW_SKILL_FAILED",
"WORKFLOW_HTTP_POLICY_DENIED",
"WORKFLOW_HTTP_FAILED",
"WORKFLOW_SQL_POLICY_DENIED",
"WORKFLOW_SQL_FAILED",
"WORKFLOW_CODE_SANDBOX_FAILED",
"WORKFLOW_TRANSFORM_FAILED",
"WORKFLOW_CONDITION_FAILED",
"WORKFLOW_MERGE_FAILED",
"WORKFLOW_HUMAN_INPUT_INVALID",
"WORKFLOW_SUBWORKFLOW_FAILED",
"WORKFLOW_LOOP_FAILED",
"WORKFLOW_FORBIDDEN",
"WORKFLOW_EMBED_TICKET_INVALID",
"WORKFLOW_EMBED_ORIGIN_DENIED",
"WORKFLOW_INTERNAL",
)
class WorkflowErrorBody(BaseModel):
"""Wire-format error body shared by API responses and ``run.failed`` events."""
code: WorkflowErrorCode
message: str
retryable: bool = False
node_id: str | None = Field(default=None, alias="nodeId")
details: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
class WorkflowError(Exception):
"""Raised by validators / node executors; convertible to :class:`WorkflowErrorBody`."""
def __init__(
self,
code: WorkflowErrorCode,
message: str,
*,
retryable: bool = False,
node_id: str | None = None,
details: dict[str, Any] | None = None,
) -> None:
super().__init__(message)
self.code = code
self.message = message
self.retryable = retryable
self.node_id = node_id
self.details = details or {}
def to_body(self) -> WorkflowErrorBody:
return WorkflowErrorBody(
code=self.code,
message=self.message,
retryable=self.retryable,
node_id=self.node_id,
details=self.details,
)
__all__ = [
"ALL_WORKFLOW_ERROR_CODES",
"WorkflowError",
"WorkflowErrorBody",
"WorkflowErrorCode",
]
@@ -0,0 +1,158 @@
"""Workflow SSE event protocol v1.0.
Invariant: **persist first, then publish**. ``seq`` is monotonic per run.
Heartbeat comments (``: heartbeat``) are live-only and never stored.
Canonical ``data`` payloads (frozen contract — REST replay and SSE must agree):
node.tool.started {"name": "web_search", "toolCallId": "call_001",
"title": "联网检索", "args": {"path": "..."}}
node.tool.finished {"name": "web_search", "toolCallId": "call_001",
"status": "succeeded" | "failed", "summary": "获得 12 条结果"}
run.awaiting_input {"resumeToken": "one-time-token", "prompt": "请确认是否发布",
"formSchema": {...}, "actions": ["approve", "reject"]}
artifact.created {"artifactId": "artifact_001", "name": "weekly-report.docx",
"kind": "file" | "markdown" | "json" | "table",
"mimeType": "...", "sizeBytes": 1234, "preview": "..."}
node.started {"nodeType": ..., "name": ..., "attempt": 1,
"agent": {...}, "parallelGroupId": "...", "roundIndex": 2}
Rules: tool identity is ``name`` (never ``toolName``); outcomes are ``status``
(never ``ok``); human-input descriptors are flattened (never nested under
``pendingInput``); artifacts never carry filesystem paths; agent nodes expose
display metadata only (no system prompts, skill whitelists or credentials).
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any, Literal, Protocol, runtime_checkable
from pydantic import BaseModel, Field
SCHEMA_VERSION = "1.0"
WorkflowEventType = Literal[
"run.created",
"run.queued",
"run.started",
"node.queued",
"node.started",
"node.progress",
"node.output.delta",
"node.tool.started",
"node.tool.finished",
"artifact.created",
"node.completed",
"node.failed",
"run.awaiting_input",
"run.resumed",
"run.cancel_requested",
"run.cancelled",
"run.completed",
"run.failed",
]
ALL_WORKFLOW_EVENT_TYPES: tuple[WorkflowEventType, ...] = (
"run.created",
"run.queued",
"run.started",
"node.queued",
"node.started",
"node.progress",
"node.output.delta",
"node.tool.started",
"node.tool.finished",
"artifact.created",
"node.completed",
"node.failed",
"run.awaiting_input",
"run.resumed",
"run.cancel_requested",
"run.cancelled",
"run.completed",
"run.failed",
)
TERMINAL_EVENT_TYPES: frozenset[WorkflowEventType] = frozenset({"run.completed", "run.failed", "run.cancelled"})
_KIND_BY_MIME: tuple[tuple[tuple[str, ...], str], ...] = (
(("text/markdown", "text/x-markdown"), "markdown"),
(("application/json", "text/json", "application/javascript"), "json"),
(("text/csv", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), "table"),
)
def artifact_kind(mime_type: str) -> str:
"""Coarse wire-level artifact category: file | markdown | json | table."""
lowered = (mime_type or "").lower()
for prefixes, kind in _KIND_BY_MIME:
if any(lowered.startswith(prefix) for prefix in prefixes):
return kind
return "file"
class WorkflowEventEnvelope(BaseModel):
"""Common envelope for every persisted / SSE workflow event."""
schema_version: Literal["1.0"] = Field(default=SCHEMA_VERSION, alias="schemaVersion")
run_id: str = Field(alias="runId")
workflow_id: str = Field(alias="workflowId")
version_id: str = Field(alias="versionId")
seq: int = 0
event_type: WorkflowEventType = Field(alias="event")
node_id: str | None = Field(default=None, alias="nodeId")
node_run_id: str | None = Field(default=None, alias="nodeRunId")
timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
data: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
def to_sse_dict(self) -> dict[str, Any]:
"""JSON object for the SSE ``data:`` line (camelCase wire fields)."""
return {
"schemaVersion": self.schema_version,
"runId": self.run_id,
"workflowId": self.workflow_id,
"versionId": self.version_id,
"seq": self.seq,
"event": self.event_type,
"nodeId": self.node_id,
"nodeRunId": self.node_run_id,
"timestamp": self.timestamp.isoformat() if self.timestamp else None,
"data": self.data,
}
def to_sse_frame(self) -> str:
"""Full SSE frame including ``id`` / ``event`` / ``data`` lines."""
import json
payload = json.dumps(self.to_sse_dict(), ensure_ascii=False, separators=(",", ":"))
return f"id: {self.seq}\nevent: {self.event_type}\ndata: {payload}\n\n"
@runtime_checkable
class WorkflowEventSink(Protocol):
"""Persist-then-publish sink used by the future executor."""
async def emit(
self,
event_type: WorkflowEventType,
*,
data: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> WorkflowEventEnvelope:
"""Persist event (assign ``seq``), then fan out to live subscribers."""
...
__all__ = [
"ALL_WORKFLOW_EVENT_TYPES",
"SCHEMA_VERSION",
"TERMINAL_EVENT_TYPES",
"WorkflowEventEnvelope",
"WorkflowEventSink",
"WorkflowEventType",
"artifact_kind",
]
@@ -0,0 +1,264 @@
"""Safe expression / template evaluation for workflow node configs.
Two surfaces, both intentionally tiny:
* ``render_template("… {{ nodes.a.data.title }} …", state)`` — string
interpolation over a whitelisted state tree.
* ``evaluate_condition("nodes.a.data.score > 0.5", state)`` — boolean
predicate over the same tree.
Neither uses :func:`eval`. The condition grammar is a single comparison or a
chain joined by ``and`` / ``or``; operands are either a state path or a JSON
literal. Anything else raises ``WORKFLOW_EXPRESSION_INVALID`` so a malformed
graph fails loudly at publish time instead of silently at run time.
"""
from __future__ import annotations
import json
import re
from typing import Any
from deerflow.workflows.errors import WorkflowError
_TEMPLATE_RE = re.compile(r"\{\{\s*([^{}]+?)\s*\}\}")
_ROOT_KEYS = ("inputs", "nodes", "run", "loop", "env")
_PATH_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_\-]*|\[\d+\])*$")
_COMPARATORS = ("==", "!=", ">=", "<=", ">", "<", " in ", " contains ")
_MAX_RENDER_CHARS = 200_000
def _invalid(expr: str, reason: str) -> WorkflowError:
return WorkflowError(
"WORKFLOW_EXPRESSION_INVALID",
f"表达式不合法:{reason}",
details={"expression": expr[:200]},
)
def resolve_path(path: str, state: dict[str, Any]) -> Any:
"""Read ``a.b[0].c`` from the state tree. Missing segments yield ``None``."""
expr = path.strip()
if not _PATH_RE.match(expr):
raise _invalid(expr, "路径格式不合法")
root = expr.split(".", 1)[0].split("[", 1)[0]
if root not in _ROOT_KEYS:
raise _invalid(expr, f"根节点必须是 {'/'.join(_ROOT_KEYS)} 之一")
current: Any = state
for segment in re.findall(r"[A-Za-z_][A-Za-z0-9_\-]*|\[\d+\]", expr):
if current is None:
return None
if segment.startswith("["):
index = int(segment[1:-1])
if not isinstance(current, (list, tuple)) or index >= len(current):
return None
current = current[index]
elif isinstance(current, dict):
current = current.get(segment)
else:
return None
return current
def _stringify(value: Any) -> str:
if value is None:
return ""
if isinstance(value, str):
return value
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, (int, float)):
return str(value)
return json.dumps(value, ensure_ascii=False, default=str)
def render_template(template: str, state: dict[str, Any]) -> str:
"""Replace every ``{{ path }}`` with its stringified state value."""
if not template:
return ""
def _sub(match: re.Match[str]) -> str:
return _stringify(resolve_path(match.group(1), state))
rendered = _TEMPLATE_RE.sub(_sub, template)
if len(rendered) > _MAX_RENDER_CHARS:
raise WorkflowError(
"WORKFLOW_LIMIT_EXCEEDED",
"模板渲染结果超出长度上限",
details={"limit": _MAX_RENDER_CHARS},
)
return rendered
def render_value(value: Any, state: dict[str, Any]) -> Any:
"""Recursively render templates inside dict / list / str structures.
A string that is exactly one ``{{ path }}`` keeps the resolved value's
native type instead of being stringified.
"""
if isinstance(value, str):
match = _TEMPLATE_RE.fullmatch(value.strip())
if match:
return resolve_path(match.group(1), state)
return render_template(value, state)
if isinstance(value, dict):
return {k: render_value(v, state) for k, v in value.items()}
if isinstance(value, list):
return [render_value(v, state) for v in value]
return value
def _operand(token: str, state: dict[str, Any]) -> Any:
text = token.strip()
if not text:
raise _invalid(token, "操作数为空")
match = _TEMPLATE_RE.fullmatch(text)
if match:
return resolve_path(match.group(1), state)
lowered = text.lower()
if lowered in ("true", "false"):
return lowered == "true"
if lowered in ("null", "none"):
return None
if (text[0], text[-1]) in (('"', '"'), ("'", "'")) and len(text) >= 2:
return text[1:-1]
try:
return json.loads(text)
except json.JSONDecodeError:
pass
if _PATH_RE.match(text) and text.split(".", 1)[0].split("[", 1)[0] in _ROOT_KEYS:
return resolve_path(text, state)
return text
def _truthy(value: Any) -> bool:
if isinstance(value, str):
return value.strip().lower() not in ("", "false", "0", "null", "none")
return bool(value)
def _compare(left: Any, op: str, right: Any) -> bool:
if op == "==":
return left == right
if op == "!=":
return left != right
if op == "in":
try:
return left in right # type: ignore[operator]
except TypeError:
return False
if op == "contains":
try:
return right in left # type: ignore[operator]
except TypeError:
return False
try:
if op == ">":
return left > right
if op == "<":
return left < right
if op == ">=":
return left >= right
if op == "<=":
return left <= right
except TypeError:
return False
raise _invalid(op, f"不支持的比较运算符 {op}")
def _split_top_level(expr: str, keyword: str) -> list[str] | None:
"""Split on ``keyword`` outside quotes / braces; None when absent."""
parts: list[str] = []
depth = 0
quote: str | None = None
buffer: list[str] = []
index = 0
pattern = f" {keyword} "
while index < len(expr):
char = expr[index]
if quote:
buffer.append(char)
if char == quote:
quote = None
index += 1
continue
if char in ('"', "'"):
quote = char
buffer.append(char)
index += 1
continue
if char in "[{(":
depth += 1
elif char in "]})":
depth -= 1
if depth == 0 and expr[index : index + len(pattern)] == pattern:
parts.append("".join(buffer))
buffer = []
index += len(pattern)
continue
buffer.append(char)
index += 1
if not parts:
return None
parts.append("".join(buffer))
return parts
def evaluate_condition(expression: str, state: dict[str, Any]) -> bool:
"""Evaluate a boolean predicate without ``eval``."""
expr = (expression or "").strip()
if not expr:
return False
or_parts = _split_top_level(expr, "or")
if or_parts:
return any(evaluate_condition(part, state) for part in or_parts)
and_parts = _split_top_level(expr, "and")
if and_parts:
return all(evaluate_condition(part, state) for part in and_parts)
if expr.startswith("not "):
return not evaluate_condition(expr[4:], state)
if expr.startswith("(") and expr.endswith(")"):
return evaluate_condition(expr[1:-1], state)
for comparator in _COMPARATORS:
op = comparator.strip()
index = expr.find(comparator)
if index <= 0:
continue
left = _operand(expr[:index], state)
right = _operand(expr[index + len(comparator) :], state)
return _compare(left, op, right)
return _truthy(_operand(expr, state))
_NODES_IDENT_RE = re.compile(r"(?:\{\{\s*)?nodes\.([A-Za-z_][A-Za-z0-9_\-]*)")
def collect_referenced_nodes(value: Any) -> set[str]:
"""Node ids referenced through ``nodes.<id>`` anywhere in a config tree.
Covers both ``{{ nodes.x.data }}`` templates and bare condition paths
such as ``nodes.x.data.score > 0.5``.
"""
found: set[str] = set()
if isinstance(value, str):
found.update(_NODES_IDENT_RE.findall(value))
elif isinstance(value, dict):
for item in value.values():
found |= collect_referenced_nodes(item)
elif isinstance(value, list):
for item in value:
found |= collect_referenced_nodes(item)
return found
__all__ = [
"collect_referenced_nodes",
"evaluate_condition",
"render_template",
"render_value",
"resolve_path",
]
@@ -0,0 +1,65 @@
"""Node executors for the workflow runtime."""
from __future__ import annotations
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.workflows.nodes.agents import (
AgentNodeExecutor,
SkillNodeExecutor,
SubworkflowNodeExecutor,
)
from deerflow.workflows.nodes.base import ACTIVE_PORTS, NodeExecutor, NodeRegistry
from deerflow.workflows.nodes.core import (
ConditionNodeExecutor,
LoopNodeExecutor,
MergeNodeExecutor,
OutputNodeExecutor,
StartNodeExecutor,
TransformNodeExecutor,
)
from deerflow.workflows.nodes.human import HumanInputNodeExecutor
from deerflow.workflows.nodes.integrations import (
CodeNodeExecutor,
HttpNodeExecutor,
SqlReadNodeExecutor,
)
__all__ = [
"ACTIVE_PORTS",
"AgentNodeExecutor",
"CodeNodeExecutor",
"ConditionNodeExecutor",
"HttpNodeExecutor",
"HumanInputNodeExecutor",
"LoopNodeExecutor",
"MergeNodeExecutor",
"NodeExecutor",
"NodeRegistry",
"OutputNodeExecutor",
"SkillNodeExecutor",
"SqlReadNodeExecutor",
"StartNodeExecutor",
"SubworkflowNodeExecutor",
"TransformNodeExecutor",
"build_default_registry",
]
def build_default_registry(config: WorkflowConfig | None = None) -> NodeRegistry:
"""Registry with every executable node type wired up."""
cfg = config or WorkflowConfig()
registry = NodeRegistry()
registry.register("start", StartNodeExecutor())
registry.register("output", OutputNodeExecutor())
registry.register("transform", TransformNodeExecutor())
registry.register("condition", ConditionNodeExecutor())
registry.register("merge", MergeNodeExecutor())
registry.register("loop", LoopNodeExecutor())
registry.register("human_input", HumanInputNodeExecutor())
registry.register("http", HttpNodeExecutor(cfg))
registry.register("sql_read", SqlReadNodeExecutor(cfg))
registry.register("code", CodeNodeExecutor(cfg))
registry.register("agent", AgentNodeExecutor())
registry.register("skill", SkillNodeExecutor())
registry.register("subworkflow", SubworkflowNodeExecutor())
return registry
@@ -0,0 +1,361 @@
"""LLM-backed nodes: ``agent``, ``skill``, ``subworkflow``.
These are the only nodes that call the DeerFlow LangGraph lead agent. The agent
invocation itself is injected (``deps.run_agent``) so the harness keeps its
import firewall; this module owns prompt rendering, streaming fan-out, JSON
response coercion, and error typing.
"""
from __future__ import annotations
import asyncio
import json
import logging
import re
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import render_template, render_value
from deerflow.workflows.nodes.base import require_dep
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.schemas import ArtifactRef, NodeResult, WorkflowNode
logger = logging.getLogger(__name__)
_JSON_BLOCK = re.compile(r"```(?:json)?\s*(.+?)\s*```", re.DOTALL)
_DELTA_FLUSH_SECONDS = 0.05
_DELTA_FLUSH_CHARS = 1024
class _DeltaCoalescer:
"""Merge model tokens into 50ms / 1KB event payloads (§9.5)."""
def __init__(self, ctx: RunContext, node_id: str) -> None:
self._ctx = ctx
self._node_id = node_id
self._parts: list[str] = []
self._chars = 0
self._task: asyncio.Task | None = None
self._lock = asyncio.Lock()
async def push(self, chunk: str) -> None:
if not chunk:
return
async with self._lock:
self._parts.append(chunk)
self._chars += len(chunk)
if self._chars >= _DELTA_FLUSH_CHARS:
await self._flush_unlocked()
return
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._delayed())
async def _delayed(self) -> None:
await asyncio.sleep(_DELTA_FLUSH_SECONDS)
async with self._lock:
await self._flush_unlocked()
async def _flush_unlocked(self) -> None:
if not self._parts:
return
text = "".join(self._parts)
self._parts.clear()
self._chars = 0
await self._ctx.emit_event(
"node.output.delta",
data={"channel": "answer", "delta": text, "text": text},
node_id=self._node_id,
)
async def flush(self) -> None:
task = self._task
if task is not None and not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
async with self._lock:
await self._flush_unlocked()
def _extract_json(text: str) -> Any:
candidate = text.strip()
match = _JSON_BLOCK.search(candidate)
if match:
candidate = match.group(1).strip()
else:
start = min((i for i in (candidate.find("{"), candidate.find("[")) if i >= 0), default=-1)
if start > 0:
candidate = candidate[start:]
try:
return json.loads(candidate)
except json.JSONDecodeError:
return None
_SAFE_ARG_STRING_CHARS = 160
_SAFE_ARG_MAX_KEYS = 12
def _safe_tool_args(args: Any) -> dict[str, Any]:
"""Minimal safe projection of tool-call args for the event log.
Scalars only (strings truncated), no nested containers, bounded key count —
enough for the canvas to label the step without leaking full payloads such
as entire documents, prompts or credentials that a tool may carry."""
if not isinstance(args, dict):
return {}
out: dict[str, Any] = {}
for key, value in list(args.items())[:_SAFE_ARG_MAX_KEYS]:
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
continue
if isinstance(value, str):
value = value[:_SAFE_ARG_STRING_CHARS]
out[str(key)[:64]] = value
return out
class _AgentInvocation:
"""Shared plumbing for ``agent`` and ``skill`` (agent_skill mode)."""
async def _invoke(
self,
node: WorkflowNode,
ctx: RunContext,
*,
agent_id: str,
prompt: str,
skill_names: list[str],
response_mode: str,
response_schema: dict[str, Any],
thread_mode: str,
failure_code: str,
) -> NodeResult:
run_agent = require_dep(ctx.deps.run_agent, node=node, what="智能体执行")
coalescer = _DeltaCoalescer(ctx, node.id)
async def on_delta(chunk: str) -> None:
await coalescer.push(chunk)
async def on_tool(event: dict[str, Any]) -> None:
phase = str(event.get("phase") or "started")
name = str(event.get("name") or "")
call_id = str(event.get("id") or "")
if phase == "started":
await ctx.emit_event(
"node.tool.started",
data={
"name": name,
"toolCallId": call_id,
"title": str(event.get("title") or name),
"args": _safe_tool_args(event.get("args")),
},
node_id=node.id,
)
return
ok = event.get("ok")
await ctx.emit_event(
"node.tool.finished",
data={
"name": name,
"toolCallId": call_id,
"status": "succeeded" if ok in (True, None) else "failed",
"summary": event.get("summary"),
},
node_id=node.id,
)
try:
try:
outcome = await run_agent(
run_id=ctx.run_id,
node_id=node.id,
owner_id=ctx.owner_id,
agent_id=agent_id,
prompt=prompt,
skill_names=skill_names,
thread_mode=thread_mode,
cancel=ctx.cancel,
on_delta=on_delta,
on_tool=on_tool,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
except Exception as exc: # noqa: BLE001 - agent failures are expected
logger.warning("workflow agent node %s failed", node.id, exc_info=True)
raise WorkflowError(
failure_code,
"智能体执行失败",
retryable=True,
node_id=node.id,
details={"reason": type(exc).__name__},
) from exc
finally:
await coalescer.flush()
text = str(outcome.get("text") or "")
data: dict[str, Any] = {"text": text}
if response_mode == "json":
payload = outcome.get("json")
if payload is None:
payload = _extract_json(text)
if payload is None:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"智能体未返回可解析的 JSON",
retryable=True,
node_id=node.id,
details={"preview": text[:500]},
)
required = response_schema.get("required") if isinstance(response_schema, dict) else None
if isinstance(required, list) and isinstance(payload, dict):
missing = [str(k) for k in required if payload.get(str(k)) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
f"智能体返回缺少字段:{'、'.join(missing)}",
retryable=True,
node_id=node.id,
details={"missing": missing},
)
data["json"] = payload
artifacts: list[ArtifactRef] = []
for raw in outcome.get("artifacts") or []:
if not isinstance(raw, dict) or not raw.get("name"):
continue
ref = await ctx.register_artifact(
node,
name=str(raw.get("name")),
mime_type=str(raw.get("mime_type") or raw.get("mimeType") or "application/octet-stream"),
path=str(raw.get("path") or ""),
size_bytes=raw.get("size_bytes") or raw.get("sizeBytes"),
preview=raw.get("preview"),
)
if ref is not None:
artifacts.append(ref)
return NodeResult(
data=data,
messages=[m for m in (outcome.get("messages") or []) if isinstance(m, dict)],
artifacts=artifacts,
metadata={
"agentId": agent_id,
"threadId": outcome.get("thread_id"),
"usage": outcome.get("usage") or {},
},
)
class AgentNodeExecutor(_AgentInvocation):
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
agent_id = str(node.config.get("agentId") or node.config.get("agent_id") or "")
if not agent_id:
raise WorkflowError("WORKFLOW_AGENT_FAILED", "智能体节点未选择智能体", node_id=node.id)
template = str(node.config.get("promptTemplate") or node.config.get("prompt_template") or "")
prompt = render_template(template, ctx.state()).strip()
if not prompt:
raise WorkflowError("WORKFLOW_AGENT_FAILED", "智能体节点提示词为空", node_id=node.id)
schema = node.config.get("responseSchema") or node.config.get("response_schema") or {}
return await self._invoke(
node,
ctx,
agent_id=agent_id,
prompt=prompt,
skill_names=[str(s) for s in (node.config.get("skillNames") or node.config.get("skill_names") or [])],
response_mode=str(node.config.get("responseMode") or node.config.get("response_mode") or "text"),
response_schema=schema if isinstance(schema, dict) else {},
thread_mode=str(node.config.get("threadMode") or node.config.get("thread_mode") or "isolated_per_node"),
failure_code="WORKFLOW_AGENT_FAILED",
)
class SkillNodeExecutor(_AgentInvocation):
"""``agent_skill`` runs the agent with a restricted skill set.
``callable_skill`` (direct script invocation) is not executable yet."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
mode = str(node.config.get("mode") or "agent_skill")
if mode != "agent_skill":
raise WorkflowError(
"WORKFLOW_SKILL_FAILED",
"callable_skill 模式暂未开放执行,请改用 agent_skill",
node_id=node.id,
details={"mode": mode},
)
agent_id = str(node.config.get("agentId") or node.config.get("agent_id") or "")
skill_names = [str(s) for s in (node.config.get("skillNames") or node.config.get("skill_names") or [])]
if not agent_id or not skill_names:
raise WorkflowError("WORKFLOW_SKILL_FAILED", "技能节点需要同时指定智能体与技能", node_id=node.id)
template = str(node.config.get("promptTemplate") or node.config.get("prompt_template") or "")
prompt = render_template(template, ctx.state()).strip() or f"请使用技能 {skill_names[0]} 完成任务。"
return await self._invoke(
node,
ctx,
agent_id=agent_id,
prompt=prompt,
skill_names=skill_names,
response_mode="text",
response_schema={},
thread_mode="isolated_per_node",
failure_code="WORKFLOW_SKILL_FAILED",
)
class SubworkflowNodeExecutor:
"""Runs a published child workflow synchronously and folds in its output."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
run_subworkflow = require_dep(ctx.deps.run_subworkflow, node=node, what="子工作流执行")
workflow_id = str(node.config.get("workflowId") or node.config.get("workflow_id") or "")
version_id = str(node.config.get("versionId") or node.config.get("version_id") or "")
if not workflow_id or not version_id:
raise WorkflowError("WORKFLOW_SUBWORKFLOW_FAILED", "子工作流节点缺少 workflowId/versionId", node_id=node.id)
mapping = node.config.get("inputMapping") or node.config.get("input_mapping") or {}
state = ctx.state()
inputs = {str(key): render_value(template, state) for key, template in mapping.items()} if isinstance(mapping, dict) else {}
await ctx.emit_event(
"node.progress",
data={"phase": "subworkflow.start", "workflowId": workflow_id, "versionId": version_id},
node_id=node.id,
)
try:
outcome = await run_subworkflow(
parent_run_id=ctx.run_id,
node_id=node.id,
owner_id=ctx.owner_id,
workflow_id=workflow_id,
version_id=version_id,
inputs=inputs,
cancel=ctx.cancel,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
except Exception as exc: # noqa: BLE001
raise WorkflowError(
"WORKFLOW_SUBWORKFLOW_FAILED",
"子工作流执行失败",
retryable=True,
node_id=node.id,
details={"reason": type(exc).__name__},
) from exc
status = str(outcome.get("status") or "completed")
if status != "completed":
raise WorkflowError(
"WORKFLOW_SUBWORKFLOW_FAILED",
f"子工作流未成功完成({status})",
node_id=node.id,
details={"childRunId": outcome.get("run_id"), "status": status},
)
output = outcome.get("output")
return NodeResult(
data=output if isinstance(output, dict) else {"value": output},
metadata={"childRunId": outcome.get("run_id"), "workflowId": workflow_id},
)
__all__ = ["AgentNodeExecutor", "SkillNodeExecutor", "SubworkflowNodeExecutor"]
@@ -0,0 +1,52 @@
"""Node executor protocol and registry."""
from __future__ import annotations
from typing import Any, Protocol, runtime_checkable
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.ports import ACTIVE_PORTS
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.schemas import NodeResult, NodeType, WorkflowNode
@runtime_checkable
class NodeExecutor(Protocol):
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult: ...
class NodeRegistry:
"""Maps node type → executor. Unknown types fail with a typed error."""
def __init__(self) -> None:
self._executors: dict[str, NodeExecutor] = {}
def register(self, node_type: NodeType | str, executor: NodeExecutor) -> None:
self._executors[str(node_type)] = executor
def get(self, node_type: str) -> NodeExecutor:
executor = self._executors.get(node_type)
if executor is None:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
f"节点类型暂不支持执行:{node_type}",
details={"nodeType": node_type},
)
return executor
def supported(self) -> list[str]:
return sorted(self._executors)
def require_dep(value: Any, *, node: WorkflowNode, what: str) -> Any:
if value is None:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
f"运行环境缺少{what}能力,无法执行该节点",
node_id=node.id,
details={"capability": what},
)
return value
__all__ = ["ACTIVE_PORTS", "NodeExecutor", "NodeRegistry", "require_dep"]
@@ -0,0 +1,238 @@
"""Deterministic control / data nodes: start, output, transform, condition, merge, loop."""
from __future__ import annotations
import json
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import evaluate_condition, render_template, render_value
from deerflow.workflows.nodes.base import ACTIVE_PORTS
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.schema_validation import schema_issues
from deerflow.workflows.schemas import NodeResult, WorkflowNode
DEFAULT_PORT = "else"
LOOP_CONTINUE_PORT = "continue"
LOOP_DONE_PORT = "done"
def _required_keys(schema: dict[str, Any]) -> list[str]:
required = schema.get("required")
if isinstance(required, list):
return [str(k) for k in required]
return []
class StartNodeExecutor:
"""Validates run inputs against the graph input schema and exposes them."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
schema = ctx.graph.input_schema or {}
missing = [key for key in _required_keys(schema) if ctx.inputs.get(key) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_INPUT_INVALID",
f"缺少必填输入:{'、'.join(missing)}",
node_id=node.id,
details={"missing": missing},
)
# Full JSON Schema pass (types / enums / nested objects) beyond the
# friendly missing-key check above.
issues = schema_issues(ctx.inputs, schema)
if issues:
raise WorkflowError(
"WORKFLOW_INPUT_INVALID",
"输入不符合 schema:" + ";".join(i["message"] for i in issues[:3]),
node_id=node.id,
details={"errors": issues},
)
declared = node.config.get("fields")
data = dict(ctx.inputs)
if isinstance(declared, list) and declared:
names = {str(f.get("name")) for f in declared if isinstance(f, dict) and f.get("name")}
data = {k: v for k, v in data.items() if k in names} or data
return NodeResult(data=data)
class OutputNodeExecutor:
"""Projects the final run output. Empty mapping = merge all upstream data."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
mapping = node.config.get("mapping") or {}
state = ctx.state()
if isinstance(mapping, dict) and mapping:
data = {str(key): render_value(template, state) for key, template in mapping.items()}
else:
data = {}
for result in ctx.node_results.values():
payload = result.get("data")
if isinstance(payload, dict):
data.update(payload)
schema = ctx.graph.output_schema or {}
missing = [key for key in _required_keys(schema) if data.get(key) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
f"输出缺少必填字段:{'、'.join(missing)}",
node_id=node.id,
details={"missing": missing},
)
issues = schema_issues(data, schema)
if issues:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"输出不符合 schema:" + ";".join(i["message"] for i in issues[:3]),
node_id=node.id,
details={"errors": issues},
)
return NodeResult(data=data, metadata={"isRunOutput": True})
class TransformNodeExecutor:
"""Pure data reshaping. Supported ops: set / pick / omit / json_parse / join."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
state = ctx.state()
data: dict[str, Any] = {}
template = node.config.get("template")
if isinstance(template, str) and template:
data["text"] = render_template(template, state)
for raw in node.config.get("operations") or []:
if not isinstance(raw, dict):
continue
op = str(raw.get("op") or "set")
target = str(raw.get("target") or raw.get("name") or "value")
try:
if op == "set":
data[target] = render_value(raw.get("value"), state)
elif op == "pick":
source = render_value(raw.get("source"), state)
keys = [str(k) for k in (raw.get("keys") or [])]
data[target] = {k: source.get(k) for k in keys} if isinstance(source, dict) else None
elif op == "omit":
source = render_value(raw.get("source"), state)
keys = {str(k) for k in (raw.get("keys") or [])}
data[target] = {k: v for k, v in source.items() if k not in keys} if isinstance(source, dict) else None
elif op == "json_parse":
text = render_value(raw.get("source"), state)
data[target] = json.loads(text) if isinstance(text, str) and text.strip() else None
elif op == "join":
items = render_value(raw.get("source"), state)
separator = str(raw.get("separator") or "\n")
data[target] = separator.join(str(i) for i in items) if isinstance(items, list) else ""
else:
raise WorkflowError(
"WORKFLOW_TRANSFORM_FAILED",
f"不支持的转换操作:{op}",
node_id=node.id,
details={"op": op},
)
except WorkflowError:
raise
except Exception as exc: # noqa: BLE001 - surface as typed node failure
raise WorkflowError(
"WORKFLOW_TRANSFORM_FAILED",
f"转换操作 {op} 执行失败",
node_id=node.id,
details={"op": op, "target": target, "reason": type(exc).__name__},
) from exc
return NodeResult(data=data)
class ConditionNodeExecutor:
"""Picks the first matching branch; activates only that outgoing port."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
state = ctx.state()
branches = node.config.get("branches") or []
if not isinstance(branches, list) or not branches:
raise WorkflowError("WORKFLOW_CONDITION_FAILED", "条件节点未配置分支", node_id=node.id)
evaluated: list[dict[str, Any]] = []
chosen: str | None = None
for raw in branches:
if not isinstance(raw, dict):
continue
name = str(raw.get("name") or "")
expression = str(raw.get("expression") or "")
matched = evaluate_condition(expression, state)
evaluated.append({"name": name, "matched": matched})
if matched and chosen is None:
chosen = name
if chosen is None:
chosen = str(node.config.get("defaultBranch") or node.config.get("default_branch") or DEFAULT_PORT)
return NodeResult(
data={"branch": chosen, "evaluated": evaluated},
metadata={ACTIVE_PORTS: [chosen]},
)
class MergeNodeExecutor:
"""Collects upstream results per strategy (all / first_success / any_success)."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
strategy = str(node.config.get("strategy") or "all")
sources = [str(s) for s in (node.config.get("sources") or [])]
if not sources:
sources = [edge.source for edge in ctx.graph.edges if edge.target == node.id]
collected: dict[str, Any] = {}
succeeded: list[str] = []
for source in sources:
result = ctx.node_results.get(source)
if result is None:
continue
collected[source] = result.get("data")
succeeded.append(source)
if strategy in ("first_success", "any_success") and not succeeded:
raise WorkflowError(
"WORKFLOW_MERGE_FAILED",
"合并节点没有任何成功的上游结果",
node_id=node.id,
details={"strategy": strategy, "sources": sources},
)
if strategy == "first_success":
first = succeeded[0]
return NodeResult(data={"selected": first, "value": collected[first]})
return NodeResult(data={"branches": collected, "succeeded": succeeded})
class LoopNodeExecutor:
"""Iteration gate. Activates ``continue`` (back-edge) or ``done``.
Iteration accounting lives in ``ctx.loop_state`` so the cap is enforced even
across a resume, and the engine invalidates body node results before the
next pass.
"""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
iteration = int(ctx.loop_state.get(node.id, 0))
configured = int(node.config.get("maxIterations") or node.config.get("max_iterations") or 1)
cap = min(configured, ctx.graph.settings.max_loop_iterations)
expression = str(node.config.get("continueWhen") or node.config.get("continue_when") or "")
should_continue = bool(expression) and evaluate_condition(expression, ctx.state())
next_iteration = iteration + 1
if should_continue and next_iteration >= cap:
should_continue = False
port = LOOP_CONTINUE_PORT if should_continue else LOOP_DONE_PORT
if should_continue:
ctx.loop_state[node.id] = next_iteration
return NodeResult(
data={
"iteration": iteration,
"nextIteration": next_iteration if should_continue else iteration,
"maxIterations": cap,
"continue": should_continue,
},
metadata={ACTIVE_PORTS: [port]},
)
__all__ = [
"ConditionNodeExecutor",
"LoopNodeExecutor",
"MergeNodeExecutor",
"OutputNodeExecutor",
"StartNodeExecutor",
"TransformNodeExecutor",
]
@@ -0,0 +1,77 @@
"""``human_input`` node: suspends the run and validates the resumed payload."""
from __future__ import annotations
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import render_template
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
from deerflow.workflows.schemas import NodeResult, WorkflowNode
DEFAULT_ACTIONS = ("submit",)
def _form_schema(node: WorkflowNode) -> dict[str, Any]:
schema = node.config.get("formSchema") or node.config.get("form_schema") or {}
return schema if isinstance(schema, dict) else {}
def _actions(node: WorkflowNode) -> list[str]:
raw = node.config.get("actions")
actions = [str(a) for a in raw] if isinstance(raw, list) and raw else list(DEFAULT_ACTIONS)
return actions
class HumanInputNodeExecutor:
"""First pass raises :class:`WorkflowPaused`; the resume pass consumes the
payload the API stored on the context and validates it against the form."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
payload = ctx.resume_payload
if not payload or str(payload.get("nodeId") or "") != node.id:
raise WorkflowPaused(
node.id,
{
"nodeId": node.id,
"name": node.name,
"prompt": render_template(str(node.config.get("prompt") or ""), ctx.state()),
"formSchema": _form_schema(node),
"actions": _actions(node),
},
)
action = str(payload.get("action") or "submit")
allowed = _actions(node)
if action not in allowed:
raise WorkflowError(
"WORKFLOW_HUMAN_INPUT_INVALID",
f"不支持的操作:{action}",
node_id=node.id,
details={"allowed": allowed},
)
values = payload.get("values")
values = values if isinstance(values, dict) else {}
schema = _form_schema(node)
required = schema.get("required")
if isinstance(required, list) and action != "cancel":
missing = [str(key) for key in required if values.get(str(key)) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_HUMAN_INPUT_INVALID",
f"缺少必填项:{'、'.join(missing)}",
node_id=node.id,
details={"missing": missing},
)
if action == "cancel":
raise WorkflowError(
"WORKFLOW_CANCELLED",
"用户在人工节点选择终止运行",
node_id=node.id,
)
# One-shot: a later human node in the same run must pause again.
ctx.resume_payload = None
return NodeResult(data={"action": action, "values": values})
__all__ = ["HumanInputNodeExecutor"]
@@ -0,0 +1,276 @@
"""External-effect nodes: ``http``, ``sql_read``, ``code``."""
from __future__ import annotations
import json
import logging
from typing import Any
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import render_template, render_value
from deerflow.workflows.runtime.code_runner import run_python_snippet
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.runtime.sql_runner import execute_read_query
from deerflow.workflows.schemas import NodeResult, WorkflowNode
from deerflow.workflows.security.http_policy import enforce_http_target, enforce_redirect
from deerflow.workflows.security.sql_policy import (
assert_parameters_bound,
assert_read_only,
assert_tables_allowed,
enforce_limit,
)
logger = logging.getLogger(__name__)
# Response headers safe to echo back into the event log.
_SAFE_RESPONSE_HEADERS = ("content-type", "content-length", "date", "etag", "location")
_REDACTED_REQUEST_HEADERS = ("authorization", "cookie", "x-api-key", "proxy-authorization")
class HttpNodeExecutor:
def __init__(self, config: WorkflowConfig) -> None:
self._config = config
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
policy = self._config.http
state = ctx.state()
url = render_template(str(node.config.get("url") or ""), state).strip()
if not url:
raise WorkflowError("WORKFLOW_HTTP_FAILED", "HTTP 节点缺少 URL", node_id=node.id)
method = str(node.config.get("method") or "GET").upper()
headers = {str(k): str(v) for k, v in (render_value(node.config.get("headers") or {}, state) or {}).items()}
query = render_value(node.config.get("query") or {}, state) or {}
body = render_value(node.config.get("body"), state)
credential_ref = node.config.get("credentialRef") or node.config.get("credential_ref")
if credential_ref:
headers.update(await self._credential_headers(str(credential_ref), node, ctx))
try:
target = await enforce_http_target(url, policy)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
timeout = min(
int(node.config.get("timeoutSeconds") or node.config.get("timeout_seconds") or policy.timeout_seconds),
policy.timeout_seconds,
)
max_bytes = min(
int(node.config.get("maxResponseBytes") or node.config.get("max_response_bytes") or policy.max_response_bytes),
policy.max_response_bytes,
)
allow_redirects = bool(node.config.get("allowRedirects") or node.config.get("allow_redirects"))
max_hops = policy.max_redirects if allow_redirects else 0
await ctx.emit_event(
"node.progress",
data={
"phase": "http.request",
"method": method,
"host": target["host"],
"headers": [k for k in headers if k.lower() not in _REDACTED_REQUEST_HEADERS],
},
node_id=node.id,
)
import httpx
current_url = url
hops = 0
raw = b""
truncated = False
async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
while True:
ctx.cancel.raise_if_cancelled(node.id)
try:
request = client.build_request(
method,
current_url,
headers=headers,
params=query or None,
json=body if isinstance(body, (dict, list)) else None,
content=body if isinstance(body, (str, bytes)) else None,
)
response = await client.send(request, stream=True)
except httpx.HTTPError as exc:
raise WorkflowError(
"WORKFLOW_HTTP_FAILED",
"HTTP 请求失败",
retryable=True,
node_id=node.id,
details={"reason": type(exc).__name__},
) from exc
if response.status_code in (301, 302, 303, 307, 308) and hops < max_hops:
location = response.headers.get("location") or ""
await response.aclose()
if not location:
break
current_url = await enforce_redirect(location, current_url, policy)
hops += 1
continue
chunks: list[bytes] = []
size = 0
try:
async for chunk in response.aiter_bytes():
size += len(chunk)
if size > max_bytes:
truncated = True
break
chunks.append(chunk)
finally:
await response.aclose()
raw = b"".join(chunks)
break
text_body = raw.decode("utf-8", errors="replace")
parsed: Any = None
content_type = response.headers.get("content-type", "")
if "json" in content_type.lower():
try:
parsed = json.loads(text_body)
except json.JSONDecodeError:
parsed = None
data = {
"status": response.status_code,
"ok": 200 <= response.status_code < 300,
"headers": {k: v for k, v in response.headers.items() if k.lower() in _SAFE_RESPONSE_HEADERS},
"json": parsed,
"text": None if parsed is not None else text_body,
"truncated": truncated,
"redirects": hops,
}
if not data["ok"]:
raise WorkflowError(
"WORKFLOW_HTTP_FAILED",
f"HTTP 请求返回 {response.status_code}",
retryable=response.status_code >= 500 or response.status_code == 429,
node_id=node.id,
details={"status": response.status_code, "preview": text_body[:500]},
)
return NodeResult(data=data, metadata={"host": target["host"]})
async def _credential_headers(self, ref: str, node: WorkflowNode, ctx: RunContext) -> dict[str, str]:
if ctx.deps.resolve_credential is None:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
"运行环境未配置凭据解析能力",
node_id=node.id,
)
credential = await ctx.deps.resolve_credential(ref)
if not credential:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
"引用的凭据不存在或无权访问",
node_id=node.id,
details={"credentialRef": ref},
)
headers = credential.get("headers")
return {str(k): str(v) for k, v in headers.items()} if isinstance(headers, dict) else {}
class SqlReadNodeExecutor:
def __init__(self, config: WorkflowConfig) -> None:
self._config = config
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
policy = self._config.sql
if not policy.enabled:
raise WorkflowError("WORKFLOW_SQL_POLICY_DENIED", "SQL 节点已被系统禁用", node_id=node.id)
data_source_id = str(node.config.get("dataSourceId") or node.config.get("data_source_id") or "")
if not data_source_id:
raise WorkflowError("WORKFLOW_SQL_FAILED", "SQL 节点未选择数据源", node_id=node.id)
if ctx.deps.resolve_data_source is None:
raise WorkflowError("WORKFLOW_RESOURCE_MISSING", "运行环境未配置数据源解析能力", node_id=node.id)
source = await ctx.deps.resolve_data_source(data_source_id)
if not source or not source.get("dsn"):
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
"数据源不存在或无权访问",
node_id=node.id,
details={"dataSourceId": data_source_id},
)
statement = assert_read_only(str(node.config.get("statement") or ""))
parameters = render_value(node.config.get("parameters") or {}, ctx.state()) or {}
if not isinstance(parameters, dict):
raise WorkflowError("WORKFLOW_SQL_FAILED", "SQL 参数必须是对象", node_id=node.id)
assert_parameters_bound(statement, parameters)
assert_tables_allowed(statement, list(source.get("allowed_tables") or []))
max_rows = min(
int(node.config.get("maxRows") or node.config.get("max_rows") or policy.max_rows),
policy.max_rows,
int(source.get("max_rows") or policy.max_rows),
)
statement = enforce_limit(statement, max_rows)
await ctx.emit_event(
"node.progress",
data={"phase": "sql.query", "dataSourceId": data_source_id, "maxRows": max_rows},
node_id=node.id,
)
try:
result = await execute_read_query(
dsn=str(source["dsn"]),
statement=statement,
parameters={str(k): v for k, v in parameters.items()},
max_rows=max_rows,
timeout_seconds=policy.statement_timeout_seconds,
max_cell_chars=policy.max_cell_chars,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
return NodeResult(data=result)
class CodeNodeExecutor:
def __init__(self, config: WorkflowConfig) -> None:
self._config = config
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
policy = self._config.code
if not policy.enabled:
raise WorkflowError(
"WORKFLOW_CODE_SANDBOX_FAILED",
"代码节点未启用(需管理员在 config.yaml 中开启 workflows.code.enabled)",
node_id=node.id,
)
source = str(node.config.get("source") or "")
if not source.strip():
raise WorkflowError("WORKFLOW_CODE_SANDBOX_FAILED", "代码节点没有源代码", node_id=node.id)
if len(source) > policy.max_source_chars:
raise WorkflowError(
"WORKFLOW_LIMIT_EXCEEDED",
f"代码长度超过上限 {policy.max_source_chars}",
node_id=node.id,
)
inputs = render_value(node.config.get("inputs") or {}, ctx.state()) or {}
timeout = min(
int(node.config.get("timeoutSeconds") or node.config.get("timeout_seconds") or policy.timeout_seconds),
policy.timeout_seconds,
)
allow_network = bool((node.config.get("allowNetwork") or node.config.get("allow_network")) and policy.allow_network)
try:
outcome = await run_python_snippet(
source=source,
inputs=inputs if isinstance(inputs, dict) else {"value": inputs},
timeout_seconds=timeout,
max_output_chars=policy.max_output_chars,
memory_limit_mb=policy.memory_limit_mb,
allow_network=allow_network,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
return NodeResult(
data=outcome["result"],
metadata={"stdout": outcome["stdout"][-4000:], "stderr": outcome["stderr"][-2000:]},
)
__all__ = ["CodeNodeExecutor", "HttpNodeExecutor", "SqlReadNodeExecutor"]
@@ -0,0 +1,17 @@
"""Leaf constants shared by the scheduler and node executors.
Kept dependency-free so the engine and the node package can both import it
without forming a cycle.
"""
from __future__ import annotations
# Metadata key a node sets to restrict which outgoing ports the engine activates.
ACTIVE_PORTS = "activePorts"
# Reserved port names.
LOOP_CONTINUE_PORT = "continue"
LOOP_DONE_PORT = "done"
CONDITION_DEFAULT_PORT = "else"
__all__ = ["ACTIVE_PORTS", "CONDITION_DEFAULT_PORT", "LOOP_CONTINUE_PORT", "LOOP_DONE_PORT"]
@@ -0,0 +1,23 @@
"""Workflow execution runtime (scheduler, context, event sink)."""
from __future__ import annotations
from deerflow.workflows.runtime.context import (
CancelToken,
RunContext,
WorkflowPaused,
WorkflowRuntimeDeps,
)
from deerflow.workflows.runtime.engine import EngineResult, WorkflowEngine
from deerflow.workflows.runtime.sink import PersistingWorkflowEventSink, envelope_from_row
__all__ = [
"CancelToken",
"EngineResult",
"PersistingWorkflowEventSink",
"RunContext",
"WorkflowEngine",
"WorkflowPaused",
"WorkflowRuntimeDeps",
"envelope_from_row",
]
@@ -0,0 +1,173 @@
"""Constrained Python execution for the ``code`` node.
The snippet runs in a **separate interpreter process** with:
* a scrubbed environment (no API keys, tokens, or DSNs inherited),
* a fresh temporary working directory that is deleted afterwards,
* an address-space cap and no core dumps (POSIX ``setrlimit``),
* sockets disabled unless the node explicitly allows network,
* a hard wall-clock timeout with process-group kill,
* bounded stdout/stderr capture.
This is process isolation, not a container: a determined snippet can still read
world-readable files on the host. That is why ``workflows.code.enabled``
defaults to ``false`` — turn it on only where the container sandbox or an OS
level jail already confines the backend process.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import shutil
import sys
import tempfile
from pathlib import Path
from typing import Any
from deerflow.workflows.errors import WorkflowError
logger = logging.getLogger(__name__)
_RUNNER_TEMPLATE = """
import json, os, sys
_ALLOW_NETWORK = {allow_network}
if not _ALLOW_NETWORK:
import socket
def _blocked(*_args, **_kwargs):
raise PermissionError("network access is disabled for this code node")
socket.socket = _blocked
socket.create_connection = _blocked
socket.getaddrinfo = _blocked
try:
import resource
_limit = {memory_bytes}
resource.setrlimit(resource.RLIMIT_AS, (_limit, _limit))
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
except Exception:
pass
inputs = json.loads(os.environ.get("WORKFLOW_NODE_INPUT") or "{{}}")
output = {{}}
_globals = {{"__name__": "__workflow_code__", "inputs": inputs, "output": output}}
with open({source_path!r}, "r", encoding="utf-8") as handle:
_source = handle.read()
exec(compile(_source, "<workflow-code>", "exec"), _globals)
_result = _globals.get("output")
if not isinstance(_result, dict):
_result = {{"value": _result}}
sys.stdout.write("\\n__WORKFLOW_RESULT__" + json.dumps(_result, ensure_ascii=False, default=str))
"""
def _safe_env() -> dict[str, str]:
env = {
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
"LANG": os.environ.get("LANG", "C.UTF-8"),
"PYTHONIOENCODING": "utf-8",
"PYTHONDONTWRITEBYTECODE": "1",
"HOME": "",
}
if sys.platform == "win32":
for key in ("SYSTEMROOT", "COMSPEC", "TEMP", "TMP"):
if os.environ.get(key):
env[key] = os.environ[key]
return env
async def run_python_snippet(
*,
source: str,
inputs: dict[str, Any],
timeout_seconds: int,
max_output_chars: int,
memory_limit_mb: int,
allow_network: bool,
) -> dict[str, Any]:
workdir = Path(tempfile.mkdtemp(prefix="wf-code-"))
try:
source_path = workdir / "snippet.py"
source_path.write_text(source, encoding="utf-8")
runner_path = workdir / "runner.py"
runner_path.write_text(
_RUNNER_TEMPLATE.format(
allow_network=bool(allow_network),
memory_bytes=max(64, int(memory_limit_mb)) * 1024 * 1024,
source_path=str(source_path),
),
encoding="utf-8",
)
env = _safe_env()
env["WORKFLOW_NODE_INPUT"] = json.dumps(inputs, ensure_ascii=False, default=str)
process = await asyncio.create_subprocess_exec(
sys.executable,
"-I",
"-S",
str(runner_path),
cwd=str(workdir),
env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
start_new_session=sys.platform != "win32",
)
try:
stdout_bytes, stderr_bytes = await asyncio.wait_for(process.communicate(), timeout=timeout_seconds)
except TimeoutError as exc:
_terminate(process)
raise WorkflowError(
"WORKFLOW_TIMEOUT",
f"代码节点执行超时({timeout_seconds}s)",
details={"timeoutSeconds": timeout_seconds},
) from exc
stdout = stdout_bytes.decode("utf-8", errors="replace")
stderr = stderr_bytes.decode("utf-8", errors="replace")[:max_output_chars]
if process.returncode != 0:
raise WorkflowError(
"WORKFLOW_CODE_SANDBOX_FAILED",
"代码节点执行失败",
details={"exitCode": process.returncode, "stderr": stderr[-2000:]},
)
marker = "__WORKFLOW_RESULT__"
index = stdout.rfind(marker)
result: dict[str, Any] = {}
logs = stdout
if index >= 0:
logs = stdout[:index]
try:
parsed = json.loads(stdout[index + len(marker) :])
result = parsed if isinstance(parsed, dict) else {"value": parsed}
except json.JSONDecodeError:
result = {}
return {
"result": result,
"stdout": logs[:max_output_chars],
"stderr": stderr,
}
finally:
shutil.rmtree(workdir, ignore_errors=True)
def _terminate(process: asyncio.subprocess.Process) -> None:
try:
if sys.platform != "win32":
os.killpg(os.getpgid(process.pid), 9)
else:
process.kill()
except (ProcessLookupError, PermissionError, OSError):
logger.debug("code node process already gone", exc_info=True)
__all__ = ["run_python_snippet"]
@@ -0,0 +1,162 @@
"""Run-scoped execution context shared by the engine and node executors."""
from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.events import WorkflowEventType
from deerflow.workflows.schemas import ArtifactRef, WorkflowGraph, WorkflowNode
EmitFn = Callable[..., Awaitable[Any]]
def _public_artifact_payload(ref: ArtifactRef) -> dict[str, Any]:
"""Wire payload for ``artifact.created``.
The sandbox/host ``path`` stays internal: clients address artifacts by id
through the run-scoped content endpoints, never by filesystem path."""
from deerflow.workflows.events import artifact_kind
return {
"artifactId": ref.artifact_id,
"name": ref.name,
"kind": artifact_kind(ref.mime_type),
"mimeType": ref.mime_type,
"sizeBytes": ref.size_bytes,
"preview": ref.preview,
}
class WorkflowPaused(Exception): # noqa: N818 - control-flow signal, not an error
"""Raised by ``human_input`` to suspend the run until a resume arrives."""
def __init__(self, node_id: str, pending_input: dict[str, Any]) -> None:
super().__init__(f"workflow paused at {node_id}")
self.node_id = node_id
self.pending_input = pending_input
class CancelToken:
"""Cooperative cancellation shared between the executor and node code."""
def __init__(self) -> None:
self._event = asyncio.Event()
self._reason = "cancelled"
def cancel(self, reason: str = "cancelled") -> None:
self._reason = reason
self._event.set()
@property
def cancelled(self) -> bool:
return self._event.is_set()
async def wait(self) -> None:
await self._event.wait()
def raise_if_cancelled(self, node_id: str | None = None) -> None:
if self._event.is_set():
raise WorkflowError("WORKFLOW_CANCELLED", "运行已被取消", node_id=node_id)
@dataclass
class WorkflowRuntimeDeps:
"""Injected capabilities. The harness never imports the app layer, so every
integration point (agents, HTTP egress, SQL, sandbox, subworkflows) arrives
as an async callable supplied by the caller. ``None`` means "not available",
which node executors turn into a typed ``WORKFLOW_RESOURCE_MISSING``."""
run_agent: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_http: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_sql: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_code: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_subworkflow: Callable[..., Awaitable[dict[str, Any]]] | None = None
save_artifact: Callable[..., Awaitable[dict[str, Any]]] | None = None
resolve_credential: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
resolve_data_source: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
@dataclass
class RunContext:
"""Everything a node executor is allowed to see."""
run_id: str
workflow_id: str
version_id: str
owner_id: str
graph: WorkflowGraph
inputs: dict[str, Any]
deps: WorkflowRuntimeDeps
cancel: CancelToken
emit: EmitFn
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
loop_state: dict[str, int] = field(default_factory=dict)
resume_payload: dict[str, Any] | None = None
node_run_ids: dict[str, str] = field(default_factory=dict)
current_node_id: str | None = None
env: dict[str, Any] = field(default_factory=dict)
def state(self) -> dict[str, Any]:
"""Whitelisted tree visible to expressions. Credentials never appear."""
return {
"inputs": self.inputs,
"nodes": self.node_results,
"run": {"id": self.run_id, "workflowId": self.workflow_id, "versionId": self.version_id},
"loop": dict(self.loop_state),
"env": self.env,
}
async def emit_event(
self,
event_type: WorkflowEventType,
*,
data: dict[str, Any] | None = None,
node_id: str | None = None,
) -> None:
node = node_id or self.current_node_id
await self.emit(
event_type,
data=data or {},
node_id=node,
node_run_id=self.node_run_ids.get(node) if node else None,
)
async def register_artifact(
self,
node: WorkflowNode,
*,
name: str,
mime_type: str = "application/octet-stream",
path: str = "",
size_bytes: int | None = None,
preview: str | None = None,
) -> ArtifactRef | None:
"""Persist an artifact reference and emit ``artifact.created``."""
if self.deps.save_artifact is None:
return None
row = await self.deps.save_artifact(
run_id=self.run_id,
node_id=node.id,
name=name,
mime_type=mime_type,
path=path,
size_bytes=size_bytes,
preview=preview,
)
ref = ArtifactRef(
artifactId=str(row.get("id") or ""),
name=str(row.get("name") or name),
mimeType=str(row.get("mime_type") or mime_type),
path=str(row.get("path") or path),
sizeBytes=row.get("size_bytes"),
preview=row.get("preview"),
)
await self.emit_event("artifact.created", data=_public_artifact_payload(ref), node_id=node.id)
return ref
__all__ = ["CancelToken", "RunContext", "WorkflowPaused", "WorkflowRuntimeDeps"]
@@ -0,0 +1,531 @@
"""Persisted DAG scheduler for workflow runs.
Scheduling model
----------------
Every edge carries a state: ``pending`` → ``active`` (its source chose this
port) or ``pruned`` (the source chose a different port). A node is *ready* when
none of its non-back incoming edges is ``pending`` and at least one is
``active``; a node whose incoming edges all resolved to ``pruned`` is *skipped*
and prunes its own outgoing edges in turn. That single rule gives us fan-out
parallelism, condition branching, and merge joins without a topological
pre-pass, and it terminates because each superstep resolves at least one edge.
Loops are the one cycle we allow: the back edge (``loop`` node, port
``continue``) is excluded from readiness gating, and activating it clears the
body's results so the next iteration re-runs them. Every re-entry is capped by
``settings.max_loop_iterations``.
Durability: node results are written to the run store as they complete, so a
resumed or re-leased run replays only what is missing. ``human_input`` raises
:class:`WorkflowPaused`, which the caller turns into ``awaiting_input``.
"""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any, Protocol
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.ports import ACTIVE_PORTS, LOOP_CONTINUE_PORT
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
from deerflow.workflows.schemas import NodeResult, WorkflowEdge, WorkflowGraph, WorkflowNode
class ExecutorRegistry(Protocol):
"""Structural view of ``deerflow.workflows.nodes.NodeRegistry``.
Declared here rather than imported so the scheduler has no dependency on
the node package (which depends on the runtime context).
"""
def get(self, node_type: str) -> Any: ...
logger = logging.getLogger(__name__)
PENDING = "pending"
ACTIVE = "active"
PRUNED = "pruned"
@dataclass
class EngineResult:
output: dict[str, Any] = field(default_factory=dict)
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
loop_state: dict[str, int] = field(default_factory=dict)
steps: int = 0
def _is_back_edge(edge: WorkflowEdge, nodes: dict[str, WorkflowNode]) -> bool:
source = nodes.get(edge.source)
return bool(source and source.type == "loop" and (edge.source_port or "") == LOOP_CONTINUE_PORT)
def _node_started_event_data(node: WorkflowNode, node_attempt: int) -> dict[str, Any]:
"""Safe presentation metadata for workflow clients.
This stays intentionally small: UI clients need stable agent identity and
an optional explicit parallel group, but never prompt templates, tools,
credentials, or the full node config. A missing group is valid; clients
may still use their backwards-compatible name-based presentation.
"""
data: dict[str, Any] = {
"nodeType": node.type,
"nodeKind": node.type,
"name": node.name,
"attempt": node_attempt,
}
group_id = node.config.get("parallelGroupId") or node.config.get("parallel_group_id")
if isinstance(group_id, str) and group_id.strip():
data["parallelGroupId"] = group_id.strip()
round_index = node.config.get("roundIndex")
if round_index is None:
round_index = node.config.get("round_index")
if isinstance(round_index, int) and round_index >= 0:
data["roundIndex"] = round_index
if node.type == "agent":
agent_id = node.config.get("agentId") or node.config.get("agent_id")
if isinstance(agent_id, str) and agent_id.strip():
agent: dict[str, str] = {
"agentId": agent_id.strip(),
"name": str(
node.config.get("agentName")
or node.config.get("agent_name")
or node.name
or agent_id
),
}
avatar_url = node.config.get("agentAvatarUrl") or node.config.get("agent_avatar_url")
if isinstance(avatar_url, str) and avatar_url.strip():
agent["avatarUrl"] = avatar_url.strip()
data["agent"] = agent
return data
class WorkflowEngine:
def __init__(self, registry: ExecutorRegistry, *, run_store: WorkflowRunStore | None = None) -> None:
self._registry = registry
self._store = run_store
async def run(
self,
ctx: RunContext,
*,
preloaded: dict[str, dict[str, Any]] | None = None,
loop_state: dict[str, int] | None = None,
attempt: int = 1,
) -> EngineResult:
graph: WorkflowGraph = ctx.graph
nodes = graph.node_map()
outgoing: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
incoming: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
for edge in graph.edges:
outgoing[edge.source].append(edge)
incoming[edge.target].append(edge)
edge_state: dict[str, str] = {edge.id: PENDING for edge in graph.edges}
done: set[str] = set()
skipped: set[str] = set()
ctx.loop_state.update(loop_state or {})
if preloaded:
ctx.node_results.update(preloaded)
done |= set(preloaded)
# Re-derive edge states from the ports each replayed node selected.
for node_id in list(done):
self._apply_ports(
node_id,
ctx.node_results.get(node_id, {}).get("metadata") or {},
outgoing,
edge_state,
)
semaphore = asyncio.Semaphore(max(1, graph.settings.max_parallelism))
steps = 0
max_steps = max(1, graph.settings.max_steps)
while True:
ctx.cancel.raise_if_cancelled()
self._propagate_skips(nodes, incoming, outgoing, edge_state, done, skipped)
ready = [node_id for node_id in nodes if node_id not in done and node_id not in skipped and self._readiness(node_id, incoming, edge_state, nodes) == "ready"]
if not ready:
break
steps += len(ready)
if steps > max_steps:
raise WorkflowError(
"WORKFLOW_MAX_STEPS",
f"运行步数超过上限 {max_steps}",
details={"maxSteps": max_steps},
)
results = await asyncio.gather(
*(self._run_one(nodes[node_id], ctx, semaphore, attempt) for node_id in ready),
return_exceptions=True,
)
paused: WorkflowPaused | None = None
failure: BaseException | None = None
for node_id, outcome in zip(ready, results, strict=True):
if isinstance(outcome, WorkflowPaused):
paused = paused or outcome
continue
if isinstance(outcome, BaseException):
failure = failure or outcome
continue
ctx.node_results[node_id] = outcome
done.add(node_id)
metadata = outcome.get("metadata") or {}
self._apply_ports(node_id, metadata, outgoing, edge_state)
if nodes[node_id].type == "loop":
await self._maybe_reset_loop_body(nodes[node_id], ctx, nodes, outgoing, edge_state, done, skipped)
if failure is not None:
raise failure
if paused is not None:
raise paused
return EngineResult(
output=self._collect_output(nodes, ctx),
node_results=dict(ctx.node_results),
loop_state=dict(ctx.loop_state),
steps=steps,
)
# ── scheduling helpers ──────────────────────────────────────────────
def _readiness(
self,
node_id: str,
incoming: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
nodes: dict[str, WorkflowNode],
) -> str:
edges = incoming.get(node_id) or []
if not edges:
return "ready"
back = [e for e in edges if _is_back_edge(e, nodes)]
if any(edge_state[e.id] == ACTIVE for e in back):
return "ready"
normal = [e for e in edges if not _is_back_edge(e, nodes)]
if not normal:
return "blocked"
if any(edge_state[e.id] == PENDING for e in normal):
return "blocked"
if any(edge_state[e.id] == ACTIVE for e in normal):
return "ready"
return "skip"
def _apply_ports(
self,
node_id: str,
metadata: dict[str, Any],
outgoing: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
) -> None:
active_ports = metadata.get(ACTIVE_PORTS)
allowed = {str(p) for p in active_ports} if isinstance(active_ports, list) else None
for edge in outgoing.get(node_id) or []:
port = edge.source_port or ""
if allowed is None or not port or port in allowed:
edge_state[edge.id] = ACTIVE
else:
edge_state[edge.id] = PRUNED
def _propagate_skips(
self,
nodes: dict[str, WorkflowNode],
incoming: dict[str, list[WorkflowEdge]],
outgoing: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
done: set[str],
skipped: set[str],
) -> None:
changed = True
while changed:
changed = False
for node_id in nodes:
if node_id in done or node_id in skipped:
continue
if self._readiness(node_id, incoming, edge_state, nodes) != "skip":
continue
skipped.add(node_id)
for edge in outgoing.get(node_id) or []:
edge_state[edge.id] = PRUNED
changed = True
async def _maybe_reset_loop_body(
self,
loop_node: WorkflowNode,
ctx: RunContext,
nodes: dict[str, WorkflowNode],
outgoing: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
done: set[str],
skipped: set[str],
) -> None:
back_edges = [e for e in outgoing.get(loop_node.id) or [] if _is_back_edge(e, nodes) and edge_state[e.id] == ACTIVE]
if not back_edges:
return
body_entry = str(loop_node.config.get("bodyEntry") or loop_node.config.get("body_entry") or "")
if not body_entry or body_entry not in nodes:
raise WorkflowError(
"WORKFLOW_LOOP_FAILED",
"循环节点的 bodyEntry 不存在",
node_id=loop_node.id,
details={"bodyEntry": body_entry},
)
body = self._loop_body(body_entry, loop_node.id, outgoing, nodes)
for node_id in body:
done.discard(node_id)
skipped.discard(node_id)
ctx.node_results.pop(node_id, None)
for edge in outgoing.get(node_id) or []:
if not _is_back_edge(edge, nodes):
edge_state[edge.id] = PENDING
# The back edge stays ACTIVE so the body entry is reachable again; the
# loop node prunes it itself once it selects the ``done`` port.
if self._store is not None and body:
try:
await self._store.invalidate_node_runs(ctx.run_id, sorted(body))
except Exception: # noqa: BLE001 - bookkeeping must not kill the run
logger.warning("failed to invalidate node runs for loop reset", exc_info=True)
await ctx.emit_event(
"node.progress",
data={
"phase": "loop.iteration",
"iteration": ctx.loop_state.get(loop_node.id, 0),
"bodyEntry": body_entry,
},
node_id=loop_node.id,
)
def _loop_body(
self,
body_entry: str,
loop_id: str,
outgoing: dict[str, list[WorkflowEdge]],
nodes: dict[str, WorkflowNode],
) -> set[str]:
"""Nodes reachable from ``body_entry`` up to (and including) the loop node."""
body: set[str] = set()
stack = [body_entry]
while stack:
current = stack.pop()
if current in body:
continue
body.add(current)
if current == loop_id:
continue
for edge in outgoing.get(current) or []:
if not _is_back_edge(edge, nodes):
stack.append(edge.target)
return body
# ── node execution ──────────────────────────────────────────────────
async def _run_one(
self,
node: WorkflowNode,
ctx: RunContext,
semaphore: asyncio.Semaphore,
attempt: int,
) -> dict[str, Any]:
async with semaphore:
ctx.cancel.raise_if_cancelled(node.id)
retry_cfg = node.config.get("retry") if isinstance(node.config.get("retry"), dict) else {}
max_attempts = max(1, min(5, int(retry_cfg.get("maxAttempts") or 1)))
backoff = max(0.0, float(retry_cfg.get("backoffSeconds") or 0.5))
on_error = str(node.config.get("onError") or "fail")
timeout = self._node_timeout(node, ctx)
last_error: WorkflowError | None = None
for node_attempt in range(1, max_attempts + 1):
node_run = await self._begin_node_run(ctx, node, attempt, node_attempt)
started = time.monotonic()
await ctx.emit_event(
"node.started",
data=_node_started_event_data(node, node_attempt),
node_id=node.id,
)
try:
executor = self._registry.get(node.type)
result: NodeResult = await asyncio.wait_for(executor.execute(node, ctx), timeout=timeout)
except WorkflowPaused:
await self._finish_node_run(ctx, node_run, status="queued", output=None)
raise
except TimeoutError as exc:
last_error = WorkflowError(
"WORKFLOW_TIMEOUT",
f"节点执行超时({timeout}s)",
retryable=True,
node_id=node.id,
details={"timeoutSeconds": timeout},
)
await self._fail_node_run(ctx, node_run, last_error, started)
if node_attempt < max_attempts:
await asyncio.sleep(backoff * node_attempt)
continue
if on_error == "continue":
return self._error_result(last_error)
raise last_error from exc
except WorkflowError as exc:
last_error = exc
exc.node_id = exc.node_id or node.id
await self._fail_node_run(ctx, node_run, exc, started)
if exc.retryable and node_attempt < max_attempts:
await asyncio.sleep(backoff * node_attempt)
continue
if on_error == "continue":
return self._error_result(exc)
raise
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 - unexpected node bug
logger.exception("workflow node %s crashed", node.id)
last_error = WorkflowError(
"WORKFLOW_INTERNAL",
"节点执行发生内部错误",
node_id=node.id,
details={"reason": type(exc).__name__},
)
await self._fail_node_run(ctx, node_run, last_error, started)
if on_error == "continue":
return self._error_result(last_error)
raise last_error from exc
payload = result.model_dump(by_alias=False)
duration = int((time.monotonic() - started) * 1000)
await self._finish_node_run(ctx, node_run, status="completed", output=payload, duration_ms=duration)
await ctx.emit_event(
"node.completed",
data={
"nodeType": node.type,
"durationMs": duration,
"data": payload.get("data"),
"warnings": payload.get("warnings") or [],
},
node_id=node.id,
)
return payload
raise last_error or WorkflowError("WORKFLOW_INTERNAL", "节点执行失败", node_id=node.id)
def _node_timeout(self, node: WorkflowNode, ctx: RunContext) -> int:
configured = node.config.get("timeoutSeconds") or node.config.get("timeout_seconds")
graph_limit = ctx.graph.settings.node_timeout_seconds
try:
value = int(configured) if configured else graph_limit
except (TypeError, ValueError):
value = graph_limit
return max(1, min(value, graph_limit))
def _error_result(self, error: WorkflowError) -> dict[str, Any]:
return {
"data": {},
"messages": [],
"artifacts": [],
"metadata": {"failed": True, "error": error.to_body().model_dump(by_alias=True)},
"warnings": [error.message],
}
async def _begin_node_run(self, ctx: RunContext, node: WorkflowNode, run_attempt: int, node_attempt: int) -> dict[str, Any] | None:
if self._store is None:
return None
try:
row = await self._store.record_node_run(
{
"run_id": ctx.run_id,
"node_id": node.id,
"node_type": node.type,
"status": "running",
"attempt": node_attempt,
"iteration": int(ctx.loop_state.get(node.id, 0)),
"started_at": datetime.now(UTC),
"is_current": True,
}
)
except Exception: # noqa: BLE001
logger.warning("failed to record node run start for %s", node.id, exc_info=True)
return None
ctx.node_run_ids[node.id] = str(row.get("id") or "")
return row
async def _finish_node_run(
self,
ctx: RunContext,
node_run: dict[str, Any] | None,
*,
status: str,
output: dict[str, Any] | None,
duration_ms: int | None = None,
) -> None:
if self._store is None or node_run is None:
return
try:
await self._store.record_node_run(
{
"run_id": ctx.run_id,
"node_id": node_run["node_id"],
"node_type": node_run["node_type"],
"attempt": node_run["attempt"],
"iteration": node_run["iteration"],
"status": status,
"output": output,
"duration_ms": duration_ms,
"finished_at": datetime.now(UTC),
"is_current": True,
}
)
except Exception: # noqa: BLE001
logger.warning("failed to record node run finish", exc_info=True)
async def _fail_node_run(
self,
ctx: RunContext,
node_run: dict[str, Any] | None,
error: WorkflowError,
started: float,
) -> None:
duration = int((time.monotonic() - started) * 1000)
body = error.to_body().model_dump(by_alias=True)
if self._store is not None and node_run is not None:
try:
await self._store.record_node_run(
{
"run_id": ctx.run_id,
"node_id": node_run["node_id"],
"node_type": node_run["node_type"],
"attempt": node_run["attempt"],
"iteration": node_run["iteration"],
"status": "failed",
"error": body,
"duration_ms": duration,
"finished_at": datetime.now(UTC),
"is_current": True,
}
)
except Exception: # noqa: BLE001
logger.warning("failed to record node run failure", exc_info=True)
await ctx.emit_event(
"node.failed",
data={"error": body, "durationMs": duration},
node_id=error.node_id,
)
def _collect_output(self, nodes: dict[str, WorkflowNode], ctx: RunContext) -> dict[str, Any]:
output: dict[str, Any] = {}
for node_id, node in nodes.items():
if node.type != "output":
continue
result = ctx.node_results.get(node_id)
if isinstance(result, dict) and isinstance(result.get("data"), dict):
output.update(result["data"])
return output
__all__ = ["EngineResult", "WorkflowEngine"]
@@ -0,0 +1,117 @@
"""Persist-then-publish event sink for workflow runs."""
from __future__ import annotations
import logging
from collections.abc import Awaitable, Callable
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.workflows.events import WorkflowEventEnvelope, WorkflowEventType
logger = logging.getLogger(__name__)
PublishFn = Callable[[dict[str, Any]], Awaitable[None]]
class PersistingWorkflowEventSink:
"""Assigns ``seq`` in the database first, then fans out to live listeners.
A persistence failure never aborts the run, but the event is **dropped
rather than published**: SSE clients tail by cursor and rebuild from the
durable log, so a live-only frame with a made-up seq would render once and
then vanish on reconnect (a "ghost event"). Callers can see the drop via
``degraded_count`` / the ``eventPersistenceDegraded`` marker on the returned
envelope; the stream endpoint synthesises a closing frame from the run row
when a terminal event could not be logged.
"""
def __init__(
self,
store: WorkflowEventStore,
*,
run_id: str,
workflow_id: str,
version_id: str,
publish: PublishFn | None = None,
) -> None:
self._store = store
self._run_id = run_id
self._workflow_id = workflow_id
self._version_id = version_id
self._publish = publish
self.degraded_count = 0
async def emit(
self,
event_type: WorkflowEventType,
*,
data: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> WorkflowEventEnvelope:
payload = data or {}
seq = 0
degraded = False
try:
row = await self._store.append(
run_id=self._run_id,
workflow_id=self._workflow_id,
version_id=self._version_id,
event_type=event_type,
payload=payload,
node_id=node_id,
node_run_id=node_run_id,
)
seq = int(row.get("seq") or 0)
except Exception as exc: # noqa: BLE001 - the run must survive a log outage
logger.warning(
"workflow event not persisted (dropped, not published): run=%s event=%s node=%s error=%s:%s",
self._run_id,
event_type,
node_id,
type(exc).__name__,
exc,
)
self.degraded_count += 1
degraded = True
envelope = WorkflowEventEnvelope(
runId=self._run_id,
workflowId=self._workflow_id,
versionId=self._version_id,
seq=seq,
event=event_type,
nodeId=node_id,
nodeRunId=node_run_id,
timestamp=datetime.now(UTC),
data={**payload, "eventPersistenceDegraded": True} if degraded else payload,
)
if degraded:
return envelope
if self._publish is not None:
try:
await self._publish(envelope.to_sse_dict())
except Exception: # noqa: BLE001 - live fan-out is best effort
logger.debug("workflow live publish failed", exc_info=True)
return envelope
def envelope_from_row(row: dict[str, Any]) -> dict[str, Any]:
"""Turn a stored event row into the SSE wire object (replay path)."""
return {
"schemaVersion": "1.0",
"runId": row.get("run_id"),
"workflowId": row.get("workflow_id"),
"versionId": row.get("version_id"),
"seq": int(row.get("seq") or 0),
"event": row.get("event_type"),
"nodeId": row.get("node_id"),
"nodeRunId": row.get("node_run_id"),
"timestamp": row.get("created_at"),
"data": row.get("payload") or {},
}
__all__ = ["PersistingWorkflowEventSink", "envelope_from_row"]
@@ -0,0 +1,107 @@
"""Read-only query execution for the ``sql_read`` node.
Engines are cached per DSN with a tiny pool because a workflow typically hits
the same reporting database many times in one run. Values are bound as
parameters and every cell is truncated before it can reach an event payload.
"""
from __future__ import annotations
import asyncio
import logging
from datetime import date, datetime
from decimal import Decimal
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from deerflow.workflows.errors import WorkflowError
logger = logging.getLogger(__name__)
_engines: dict[str, AsyncEngine] = {}
_lock = asyncio.Lock()
async def _get_engine(dsn: str) -> AsyncEngine:
async with _lock:
engine = _engines.get(dsn)
if engine is None:
engine = create_async_engine(
dsn,
pool_pre_ping=True,
pool_size=2,
max_overflow=2,
pool_recycle=1800,
)
_engines[dsn] = engine
return engine
async def dispose_engines() -> None:
async with _lock:
for engine in _engines.values():
await engine.dispose()
_engines.clear()
def _coerce(value: Any, max_chars: int) -> Any:
if value is None or isinstance(value, (bool, int, float)):
return value
if isinstance(value, Decimal):
return float(value)
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, (bytes, bytearray, memoryview)):
return f"<binary {len(bytes(value))} bytes>"
text_value = str(value)
return text_value if len(text_value) <= max_chars else text_value[:max_chars] + "…"
async def execute_read_query(
*,
dsn: str,
statement: str,
parameters: dict[str, Any],
max_rows: int,
timeout_seconds: int,
max_cell_chars: int = 4000,
) -> dict[str, Any]:
engine = await _get_engine(dsn)
async def _run() -> dict[str, Any]:
async with engine.connect() as conn:
result = await conn.execute(text(statement), parameters)
columns = list(result.keys())
rows = result.fetchmany(max_rows)
truncated = bool(result.fetchone())
return {
"columns": columns,
"rows": [{col: _coerce(val, max_cell_chars) for col, val in zip(columns, row, strict=False)} for row in rows],
"rowCount": len(rows),
"truncated": truncated,
}
try:
return await asyncio.wait_for(_run(), timeout=timeout_seconds)
except TimeoutError as exc:
raise WorkflowError(
"WORKFLOW_TIMEOUT",
f"数据库查询超时({timeout_seconds}s)",
retryable=True,
details={"timeoutSeconds": timeout_seconds},
) from exc
except WorkflowError:
raise
except Exception as exc: # noqa: BLE001 - never leak DSN/credentials upward
logger.warning("workflow sql_read failed", exc_info=True)
raise WorkflowError(
"WORKFLOW_SQL_FAILED",
"数据库查询执行失败",
retryable=True,
details={"reason": type(exc).__name__},
) from exc
__all__ = ["dispose_engines", "execute_read_query"]
@@ -0,0 +1,54 @@
"""Load frozen workflow sample graphs shipped with the harness."""
from __future__ import annotations
import json
from functools import lru_cache
from pathlib import Path
from deerflow.workflows.schemas import WorkflowGraph
_SAMPLES_DIR = Path(__file__).resolve().parent
_NODES_DIR = _SAMPLES_DIR / "nodes"
def samples_dir() -> Path:
return _SAMPLES_DIR
def nodes_dir() -> Path:
return _NODES_DIR
@lru_cache(maxsize=32)
def load_sample_graph(name: str = "report_generation") -> WorkflowGraph:
"""Load and validate a sample graph by stem name (without ``.json``)."""
path = _SAMPLES_DIR / f"{name}.json"
if not path.is_file():
raise FileNotFoundError(f"workflow sample not found: {path}")
raw = json.loads(path.read_text(encoding="utf-8"))
return WorkflowGraph.model_validate(raw)
@lru_cache(maxsize=64)
def load_node_sample(node_type: str) -> dict:
"""Load a minimal single-node fixture (raw dict, not a full graph)."""
path = _NODES_DIR / f"{node_type}.json"
if not path.is_file():
raise FileNotFoundError(f"node sample not found: {path}")
return json.loads(path.read_text(encoding="utf-8"))
def list_node_samples() -> list[str]:
if not _NODES_DIR.is_dir():
return []
return sorted(p.stem for p in _NODES_DIR.glob("*.json"))
__all__ = [
"list_node_samples",
"load_node_sample",
"load_sample_graph",
"nodes_dir",
"samples_dir",
]
@@ -0,0 +1,15 @@
{
"schemaVersion": "1.0",
"runId": "run_01",
"workflowId": "wf_report_generation",
"versionId": "wv_03",
"seq": 42,
"event": "node.output.delta",
"nodeId": "agent_write",
"nodeRunId": "nr_09",
"timestamp": "2026-08-26T10:00:00+08:00",
"data": {
"channel": "answer",
"delta": "第一段内容"
}
}
@@ -0,0 +1,13 @@
{
"id": "agent_1",
"type": "agent",
"name": "智能体",
"config": {
"agentId": "report-writer",
"promptTemplate": "根据材料撰写:{{ nodes.merge_1.data }}",
"responseMode": "text",
"threadMode": "isolated_per_node",
"timeoutSeconds": 600,
"skillNames": []
}
}
@@ -0,0 +1,11 @@
{
"id": "code_1",
"type": "code",
"name": "代码(沙箱)",
"config": {
"language": "python",
"source": "def main(inputs):\n rows = inputs.get(\"rows\") or []\n return {\"count\": len(rows), \"titles\": [r.get(\"title\") for r in rows[:10]]}\n",
"timeoutSeconds": 60,
"allowNetwork": false
}
}
@@ -0,0 +1,18 @@
{
"id": "cond_1",
"type": "condition",
"name": "条件",
"config": {
"branches": [
{
"name": "passed",
"expression": "{{ nodes.agent_review.data.passed }} == true"
},
{
"name": "rejected",
"expression": "{{ nodes.agent_review.data.passed }} == false"
}
],
"defaultBranch": "rejected"
}
}
@@ -0,0 +1,16 @@
{
"id": "http_1",
"type": "http",
"name": "HTTP",
"config": {
"method": "GET",
"url": "https://api.example.internal/items/{{ inputs.taskId }}",
"headers": { "Accept": "application/json" },
"query": {},
"body": null,
"credentialRef": "cred_task_api",
"timeoutSeconds": 30,
"maxResponseBytes": 1048576,
"allowRedirects": false
}
}
@@ -0,0 +1,20 @@
{
"id": "human_1",
"type": "human_input",
"name": "人工介入",
"config": {
"formSchema": {
"type": "object",
"required": ["decision"],
"properties": {
"decision": {
"type": "string",
"enum": ["approve", "revise"]
},
"comment": { "type": "string" }
}
},
"prompt": "请确认是否放行当前报告草稿。",
"actions": ["approve", "revise"]
}
}
@@ -0,0 +1,10 @@
{
"id": "loop_1",
"type": "loop",
"name": "受限循环",
"config": {
"maxIterations": 2,
"continueWhen": "{{ nodes.agent_review.data.passed }} == false",
"bodyEntry": "agent_write"
}
}
@@ -0,0 +1,9 @@
{
"id": "merge_1",
"type": "merge",
"name": "合并",
"config": {
"strategy": "all",
"sources": ["sql_1", "http_1", "agent_collect"]
}
}
@@ -0,0 +1,11 @@
{
"id": "output_1",
"type": "output",
"name": "输出",
"config": {
"mapping": {
"reportMarkdown": "{{ nodes.agent_write.data.answer }}"
},
"llmRepair": false
}
}
@@ -0,0 +1,12 @@
{
"id": "skill_1",
"type": "skill",
"name": "技能(agent_skill)",
"config": {
"mode": "agent_skill",
"agentId": "report-collector",
"skillNames": ["web-research"],
"promptTemplate": "使用指定技能完成:{{ inputs.topic }}",
"timeoutSeconds": 300
}
}
@@ -0,0 +1,13 @@
{
"id": "sql_1",
"type": "sql_read",
"name": "SQL 只读",
"config": {
"dataSourceId": "ds_report_readonly",
"statement": "SELECT id, title FROM report_source WHERE task_id = :task_id",
"parameters": {
"task_id": "{{ inputs.taskId }}"
},
"maxRows": 1000
}
}
@@ -0,0 +1,6 @@
{
"id": "start_1",
"type": "start",
"name": "开始",
"config": {}
}
@@ -0,0 +1,13 @@
{
"id": "sub_1",
"type": "subworkflow",
"name": "子工作流",
"config": {
"workflowId": "wf_collect_pack",
"versionId": "wv_01",
"inputMapping": {
"topic": "{{ inputs.topic }}",
"taskId": "{{ inputs.taskId }}"
}
}
}
@@ -0,0 +1,20 @@
{
"id": "transform_1",
"type": "transform",
"name": "模板/转换",
"config": {
"operations": [
{
"op": "map",
"from": "{{ nodes.sql_1.data.rows }}",
"to": "rows"
},
{
"op": "template",
"template": "# {{ inputs.topic }}\n\n共 {{ nodes.sql_1.metadata.rowCount }} 行。",
"to": "markdown"
}
],
"template": null
}
}
@@ -0,0 +1,260 @@
{
"schemaVersion": "1.0",
"id": "wf_report_generation",
"name": "专题报告生成",
"description": "阶段0验收基线:并行取数 → 合并 → 整理 → 撰写 → 审核 → 条件退回/输出",
"inputSchema": {
"type": "object",
"required": ["topic", "taskId"],
"properties": {
"topic": { "type": "string", "description": "报告主题" },
"taskId": { "type": "string", "description": "业务任务 ID" },
"dateRange": {
"type": "object",
"properties": {
"start": { "type": "string" },
"end": { "type": "string" }
}
}
}
},
"outputSchema": {
"type": "object",
"required": ["reportMarkdown", "passed"],
"properties": {
"reportMarkdown": { "type": "string" },
"passed": { "type": "boolean" },
"artifacts": { "type": "array" },
"issues": { "type": "array" }
}
},
"nodes": [
{
"id": "start_1",
"type": "start",
"name": "开始",
"config": {}
},
{
"id": "sql_1",
"type": "sql_read",
"name": "查询内部结构化数据",
"config": {
"dataSourceId": "ds_report_readonly",
"statement": "SELECT id, title, content, updated_at FROM report_source WHERE task_id = :task_id ORDER BY updated_at DESC",
"parameters": {
"task_id": "{{ inputs.taskId }}"
},
"maxRows": 1000
}
},
{
"id": "http_1",
"type": "http",
"name": "调用只读业务接口",
"config": {
"method": "GET",
"url": "https://api.example.internal/tasks/{{ inputs.taskId }}/summary",
"headers": {
"Accept": "application/json"
},
"credentialRef": "cred_task_api",
"timeoutSeconds": 30,
"maxResponseBytes": 524288,
"allowRedirects": false
}
},
{
"id": "agent_collect",
"type": "agent",
"name": "外部材料采集",
"config": {
"agentId": "report-collector",
"promptTemplate": "围绕主题「{{ inputs.topic }}」搜集外部公开材料,任务 ID:{{ inputs.taskId }}。输出 JSON:{sources:[], notes:string}",
"responseMode": "json",
"responseSchema": {
"type": "object",
"required": ["sources", "notes"],
"properties": {
"sources": { "type": "array" },
"notes": { "type": "string" }
}
},
"threadMode": "isolated_per_node",
"timeoutSeconds": 600,
"skillNames": ["web-research"]
}
},
{
"id": "merge_1",
"type": "merge",
"name": "合并材料并保留来源",
"config": {
"strategy": "all",
"sources": ["sql_1", "http_1", "agent_collect"]
}
},
{
"id": "agent_clean",
"type": "agent",
"name": "数据整理与事实去重",
"config": {
"agentId": "report-cleaner",
"promptTemplate": "整理并去重以下材料,输出严格 JSON:{{ nodes.merge_1.data }}",
"responseMode": "json",
"responseSchema": {
"type": "object",
"required": ["facts", "sources"],
"properties": {
"facts": { "type": "array" },
"sources": { "type": "array" }
}
},
"threadMode": "isolated_per_node",
"timeoutSeconds": 600
}
},
{
"id": "agent_write",
"type": "agent",
"name": "撰写 Markdown 报告",
"config": {
"agentId": "report-writer",
"promptTemplate": "根据整理结果撰写专题报告(Markdown)。主题:{{ inputs.topic }}\n材料:{{ nodes.agent_clean.data }}\n若存在审核意见:{{ nodes.agent_review.data.revisedAdvice }}",
"responseMode": "text",
"threadMode": "isolated_per_node",
"timeoutSeconds": 900
}
},
{
"id": "agent_review",
"type": "agent",
"name": "审核报告",
"config": {
"agentId": "report-reviewer",
"promptTemplate": "审核以下报告,输出 JSON {passed:boolean, issues:array, revisedAdvice:string}:\n{{ nodes.agent_write.data.answer }}",
"responseMode": "json",
"responseSchema": {
"type": "object",
"required": ["passed", "issues", "revisedAdvice"],
"properties": {
"passed": { "type": "boolean" },
"issues": { "type": "array" },
"revisedAdvice": { "type": "string" }
}
},
"threadMode": "isolated_per_node",
"timeoutSeconds": 600
}
},
{
"id": "cond_review",
"type": "condition",
"name": "审核结果分支",
"config": {
"branches": [
{
"name": "passed",
"expression": "{{ nodes.agent_review.data.passed }} == true"
},
{
"name": "rejected",
"expression": "{{ nodes.agent_review.data.passed }} == false"
}
],
"defaultBranch": "rejected"
}
},
{
"id": "loop_revise",
"type": "loop",
"name": "退回修改(最多 2 次)",
"config": {
"maxIterations": 2,
"continueWhen": "{{ nodes.agent_review.data.passed }} == false",
"bodyEntry": "agent_write"
}
},
{
"id": "transform_pack",
"type": "transform",
"name": "组装最终字段",
"config": {
"operations": [
{
"op": "map",
"from": "{{ nodes.agent_write.data.answer }}",
"to": "reportMarkdown"
},
{
"op": "map",
"from": "{{ nodes.agent_review.data.passed }}",
"to": "passed"
},
{
"op": "map",
"from": "{{ nodes.agent_review.data.issues }}",
"to": "issues"
}
]
}
},
{
"id": "output_1",
"type": "output",
"name": "输出",
"config": {
"mapping": {
"reportMarkdown": "{{ nodes.transform_pack.data.reportMarkdown }}",
"passed": "{{ nodes.transform_pack.data.passed }}",
"issues": "{{ nodes.transform_pack.data.issues }}"
},
"llmRepair": false
}
}
],
"edges": [
{ "id": "e_start_sql", "source": "start_1", "target": "sql_1" },
{ "id": "e_start_http", "source": "start_1", "target": "http_1" },
{ "id": "e_start_collect", "source": "start_1", "target": "agent_collect" },
{ "id": "e_sql_merge", "source": "sql_1", "target": "merge_1" },
{ "id": "e_http_merge", "source": "http_1", "target": "merge_1" },
{ "id": "e_collect_merge", "source": "agent_collect", "target": "merge_1" },
{ "id": "e_merge_clean", "source": "merge_1", "target": "agent_clean" },
{ "id": "e_clean_write", "source": "agent_clean", "target": "agent_write" },
{ "id": "e_write_review", "source": "agent_write", "target": "agent_review" },
{ "id": "e_review_cond", "source": "agent_review", "target": "cond_review" },
{
"id": "e_cond_pass",
"source": "cond_review",
"target": "transform_pack",
"sourcePort": "passed"
},
{
"id": "e_cond_reject",
"source": "cond_review",
"target": "loop_revise",
"sourcePort": "rejected"
},
{
"id": "e_loop_write",
"source": "loop_revise",
"target": "agent_write",
"sourcePort": "continue"
},
{
"id": "e_loop_fail",
"source": "loop_revise",
"target": "transform_pack",
"sourcePort": "exhausted"
},
{ "id": "e_pack_out", "source": "transform_pack", "target": "output_1" }
],
"settings": {
"runTimeoutSeconds": 1800,
"nodeTimeoutSeconds": 300,
"maxSteps": 200,
"maxLoopIterations": 5,
"maxParallelism": 4
}
}
@@ -0,0 +1,41 @@
{
"schemaVersion": "1.0",
"id": "wf_start_output",
"name": "空图:开始→输出",
"description": "阶段2运行骨架用的最小可执行图",
"inputSchema": {
"type": "object",
"properties": {
"message": { "type": "string" }
}
},
"outputSchema": {
"type": "object",
"required": ["message"],
"properties": {
"message": { "type": "string" }
}
},
"nodes": [
{ "id": "start_1", "type": "start", "name": "开始", "config": {} },
{
"id": "output_1",
"type": "output",
"name": "输出",
"config": {
"mapping": { "message": "{{ inputs.message }}" },
"llmRepair": false
}
}
],
"edges": [
{ "id": "e1", "source": "start_1", "target": "output_1" }
],
"settings": {
"runTimeoutSeconds": 60,
"nodeTimeoutSeconds": 30,
"maxSteps": 10,
"maxLoopIterations": 1,
"maxParallelism": 1
}
}
@@ -0,0 +1,73 @@
"""Full JSON Schema validation for workflow input/output values.
The engine used to check only ``required`` keys; everything else (wrong type,
unknown enum value, nested ``properties``/``items`` violations) slipped through
until downstream nodes exploded with confusing errors. This module runs the
graph's ``inputSchema`` / ``outputSchema`` through ``Draft202012Validator`` and
returns structured, frontend-friendly errors (path + message) instead of
leaking Python tracebacks.
Kept as pure functions so both the node executors (``WorkflowError``) and the
HTTP router (400 responses) can share the exact same rules.
"""
from __future__ import annotations
from typing import Any
from jsonschema import Draft202012Validator
# Cap the reported errors so a wildly invalid payload cannot produce a
# multi-megabyte error body.
_MAX_ERRORS = 20
def schema_issues(value: Any, schema: dict[str, Any] | None) -> list[dict[str, Any]]:
"""Validate ``value`` against ``schema``.
Returns ``[]`` when the schema is empty/absent (nothing declared = nothing
enforced). Each issue is ``{"path": [...], "message": str}`` with ``path``
pointing at the offending field, e.g. ``["options", 0, "name"]``.
"""
if not isinstance(schema, dict) or not schema:
return []
try:
validator = Draft202012Validator(schema)
# Construction alone does not check the schema — malformed keywords
# (e.g. a non-string ``pattern``) only explode during validation.
errors = sorted(validator.iter_errors(value), key=lambda err: list(err.absolute_path))
except Exception: # noqa: BLE001 - a malformed user schema must not 500
return [{"path": [], "message": "schema 定义本身无效,无法用于校验"}]
out: list[dict[str, Any]] = []
seen: set[tuple[Any, ...]] = set()
for err in errors:
path = [str(p) if not isinstance(p, int) else p for p in err.absolute_path]
key = tuple(path)
if key in seen:
continue
seen.add(key)
out.append({"path": path, "message": _humanize(err)})
if len(out) >= _MAX_ERRORS:
out.append({"path": [], "message": f"……其余问题已省略(共 {len(errors)} 处)"})
break
return out
def _humanize(err: Any) -> str:
"""Map the common jsonschema phrasings to short Chinese messages."""
message = str(err.message)
validator = getattr(err, "validator", "")
path = ".".join(str(p) for p in err.absolute_path) or "输入"
if validator == "required":
missing = message.split("'")[1] if "'" in message else "?"
return f"{path} 缺少必填字段:{missing}"
if validator == "enum":
return f"{path} 的值不在允许范围内:{message}"
if validator == "type":
return f"{path} 类型不正确:{message}"
if validator == "additionalProperties":
return f"{path} 含有未声明的额外字段:{message}"
return f"{path} 不符合 schema:{message}"
__all__ = ["schema_issues"]
@@ -0,0 +1,374 @@
"""Frozen workflow graph schema v1.0 (internal DTO).
Coze canvas DTO is converted into this structure at the Gateway boundary.
The runtime must never depend on Coze-private fields.
Wire JSON uses camelCase aliases; Python code uses snake_case.
"""
from __future__ import annotations
from hashlib import sha256
from typing import Any, Literal
from pydantic import BaseModel, Field, field_validator, model_validator
SCHEMA_VERSION = "1.0"
NodeType = Literal[
"start",
"output",
"agent",
"skill",
"http",
"sql_read",
"code",
"transform",
"condition",
"merge",
"human_input",
"subworkflow",
"loop",
]
ALL_NODE_TYPES: tuple[NodeType, ...] = (
"start",
"output",
"agent",
"skill",
"http",
"sql_read",
"code",
"transform",
"condition",
"merge",
"human_input",
"subworkflow",
"loop",
)
RunStatus = Literal[
"queued",
"running",
"awaiting_input",
"completed",
"failed",
"cancel_requested",
"cancelled",
]
ALL_RUN_STATUSES: tuple[RunStatus, ...] = (
"queued",
"running",
"awaiting_input",
"completed",
"failed",
"cancel_requested",
"cancelled",
)
TERMINAL_RUN_STATUSES: frozenset[RunStatus] = frozenset({"completed", "failed", "cancelled"})
NodeRunStatus = Literal[
"queued",
"running",
"completed",
"failed",
"skipped",
"cancelled",
]
SkillMode = Literal["agent_skill", "callable_skill"]
MergeStrategy = Literal["all", "first_success", "any_success"]
HttpMethod = Literal["GET", "POST", "PUT", "PATCH", "DELETE"]
AgentResponseMode = Literal["text", "json"]
AgentThreadMode = Literal["isolated_per_node", "shared_run_thread"]
ExecutionMode = Literal["normal"]
class WorkflowGraphSettings(BaseModel):
"""Per-graph resource limits (still capped by system limits at publish)."""
run_timeout_seconds: int = Field(default=1800, ge=1, alias="runTimeoutSeconds")
node_timeout_seconds: int = Field(default=300, ge=1, alias="nodeTimeoutSeconds")
max_steps: int = Field(default=200, ge=1, alias="maxSteps")
max_loop_iterations: int = Field(default=5, ge=1, alias="maxLoopIterations")
max_parallelism: int = Field(default=4, ge=1, alias="maxParallelism")
model_config = {"extra": "forbid", "populate_by_name": True}
class WorkflowNode(BaseModel):
"""A single node on the internal standard graph."""
id: str = Field(min_length=1)
type: NodeType
name: str = ""
config: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
@field_validator("id")
@classmethod
def _id_not_blank(cls, value: str) -> str:
stripped = value.strip()
if not stripped:
raise ValueError("node id must be non-empty")
return stripped
class WorkflowEdge(BaseModel):
"""Directed edge. ``source_port`` names a condition / loop outlet when set."""
id: str = Field(min_length=1)
source: str
target: str
source_port: str | None = Field(default=None, alias="sourcePort")
model_config = {"extra": "forbid", "populate_by_name": True}
class WorkflowGraph(BaseModel):
"""Internal standard workflow graph (publish / run snapshot)."""
schema_version: Literal["1.0"] = Field(default=SCHEMA_VERSION, alias="schemaVersion")
id: str = Field(min_length=1)
name: str = ""
description: str = ""
input_schema: dict[str, Any] = Field(default_factory=dict, alias="inputSchema")
output_schema: dict[str, Any] = Field(default_factory=dict, alias="outputSchema")
nodes: list[WorkflowNode] = Field(default_factory=list)
edges: list[WorkflowEdge] = Field(default_factory=list)
settings: WorkflowGraphSettings = Field(default_factory=WorkflowGraphSettings)
model_config = {"extra": "forbid", "populate_by_name": True}
@model_validator(mode="after")
def _unique_ids(self) -> WorkflowGraph:
node_ids = [n.id for n in self.nodes]
if len(node_ids) != len(set(node_ids)):
raise ValueError("duplicate node id")
edge_ids = [e.id for e in self.edges]
if len(edge_ids) != len(set(edge_ids)):
raise ValueError("duplicate edge id")
known = set(node_ids)
for edge in self.edges:
if edge.source not in known:
raise ValueError(f"edge {edge.id} source not found: {edge.source}")
if edge.target not in known:
raise ValueError(f"edge {edge.id} target not found: {edge.target}")
return self
def node_map(self) -> dict[str, WorkflowNode]:
return {node.id: node for node in self.nodes}
def canonical_json(self) -> str:
"""Deterministic JSON used for ``graph_hash`` (sorted keys, no whitespace)."""
return self.model_dump_json(by_alias=True, exclude_none=True)
def graph_hash(self) -> str:
return sha256(self.canonical_json().encode("utf-8")).hexdigest()
class ArtifactRef(BaseModel):
"""Reference to a large/binary result; full bytes live outside events."""
artifact_id: str = Field(alias="artifactId")
name: str
mime_type: str = Field(default="application/octet-stream", alias="mimeType")
path: str = ""
size_bytes: int | None = Field(default=None, alias="sizeBytes")
preview: str | None = None
model_config = {"extra": "forbid", "populate_by_name": True}
class NodeResult(BaseModel):
"""Uniform result returned by every node executor."""
data: dict[str, Any] = Field(default_factory=dict)
messages: list[dict[str, Any]] = Field(default_factory=list)
artifacts: list[ArtifactRef] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
warnings: list[str] = Field(default_factory=list)
model_config = {"extra": "forbid", "populate_by_name": True}
class StartRunRequest(BaseModel):
"""Body for ``POST /api/workflows/{workflow_id}/runs``."""
version_id: str = Field(alias="versionId")
inputs: dict[str, Any] = Field(default_factory=dict)
idempotency_key: str | None = Field(default=None, alias="idempotencyKey")
execution_mode: ExecutionMode = Field(default="normal", alias="executionMode")
model_config = {"extra": "forbid", "populate_by_name": True}
class ResumeRunRequest(BaseModel):
"""Body for ``POST /api/workflows/runs/{run_id}/resume``."""
resume_token: str = Field(alias="resumeToken")
action: str
payload: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
# ── Typed node config helpers (optional; validators may use later) ───────────
class AgentNodeConfig(BaseModel):
agent_id: str = Field(alias="agentId")
prompt_template: str = Field(default="", alias="promptTemplate")
response_mode: AgentResponseMode = Field(default="text", alias="responseMode")
response_schema: dict[str, Any] = Field(default_factory=dict, alias="responseSchema")
thread_mode: AgentThreadMode = Field(default="isolated_per_node", alias="threadMode")
timeout_seconds: int = Field(default=600, ge=1, alias="timeoutSeconds")
skill_names: list[str] = Field(default_factory=list, alias="skillNames")
model_config = {"extra": "forbid", "populate_by_name": True}
class SkillNodeConfig(BaseModel):
mode: SkillMode = "agent_skill"
agent_id: str | None = Field(default=None, alias="agentId")
skill_names: list[str] = Field(default_factory=list, alias="skillNames")
prompt_template: str = Field(default="", alias="promptTemplate")
# callable_skill fields (phase 4+; accepted in schema, not executed in phase 0)
entry: str | None = None
input_schema: dict[str, Any] = Field(default_factory=dict, alias="inputSchema")
output_schema: dict[str, Any] = Field(default_factory=dict, alias="outputSchema")
timeout_seconds: int = Field(default=300, ge=1, alias="timeoutSeconds")
model_config = {"extra": "forbid", "populate_by_name": True}
class HttpNodeConfig(BaseModel):
method: HttpMethod = "GET"
url: str
headers: dict[str, str] = Field(default_factory=dict)
query: dict[str, Any] = Field(default_factory=dict)
body: Any = None
credential_ref: str | None = Field(default=None, alias="credentialRef")
timeout_seconds: int = Field(default=60, ge=1, alias="timeoutSeconds")
max_response_bytes: int = Field(default=1_048_576, ge=1, alias="maxResponseBytes")
allow_redirects: bool = Field(default=False, alias="allowRedirects")
model_config = {"extra": "forbid", "populate_by_name": True}
class SqlReadNodeConfig(BaseModel):
data_source_id: str = Field(alias="dataSourceId")
statement: str
parameters: dict[str, Any] = Field(default_factory=dict)
max_rows: int = Field(default=1000, ge=1, alias="maxRows")
model_config = {"extra": "forbid", "populate_by_name": True}
class CodeNodeConfig(BaseModel):
language: Literal["python"] = "python"
source: str
timeout_seconds: int = Field(default=60, ge=1, alias="timeoutSeconds")
allow_network: bool = Field(default=False, alias="allowNetwork")
model_config = {"extra": "forbid", "populate_by_name": True}
class TransformNodeConfig(BaseModel):
operations: list[dict[str, Any]] = Field(default_factory=list)
template: str | None = None
model_config = {"extra": "forbid", "populate_by_name": True}
class ConditionBranch(BaseModel):
name: str
expression: str
model_config = {"extra": "forbid", "populate_by_name": True}
class ConditionNodeConfig(BaseModel):
branches: list[ConditionBranch] = Field(default_factory=list)
default_branch: str | None = Field(default=None, alias="defaultBranch")
model_config = {"extra": "forbid", "populate_by_name": True}
class MergeNodeConfig(BaseModel):
strategy: MergeStrategy = "all"
sources: list[str] = Field(default_factory=list)
model_config = {"extra": "forbid", "populate_by_name": True}
class HumanInputNodeConfig(BaseModel):
form_schema: dict[str, Any] = Field(default_factory=dict, alias="formSchema")
prompt: str = ""
actions: list[str] = Field(default_factory=lambda: ["submit"])
model_config = {"extra": "forbid", "populate_by_name": True}
class SubworkflowNodeConfig(BaseModel):
workflow_id: str = Field(alias="workflowId")
version_id: str = Field(alias="versionId")
input_mapping: dict[str, str] = Field(default_factory=dict, alias="inputMapping")
model_config = {"extra": "forbid", "populate_by_name": True}
class LoopNodeConfig(BaseModel):
max_iterations: int = Field(default=5, ge=1, alias="maxIterations")
continue_when: str = Field(default="", alias="continueWhen")
body_entry: str = Field(alias="bodyEntry")
model_config = {"extra": "forbid", "populate_by_name": True}
class OutputNodeConfig(BaseModel):
mapping: dict[str, str] = Field(default_factory=dict)
llm_repair: bool = Field(default=False, alias="llmRepair")
model_config = {"extra": "forbid", "populate_by_name": True}
__all__ = [
"SCHEMA_VERSION",
"ALL_NODE_TYPES",
"ALL_RUN_STATUSES",
"TERMINAL_RUN_STATUSES",
"AgentNodeConfig",
"AgentResponseMode",
"AgentThreadMode",
"ArtifactRef",
"CodeNodeConfig",
"ConditionBranch",
"ConditionNodeConfig",
"ExecutionMode",
"HttpMethod",
"HttpNodeConfig",
"HumanInputNodeConfig",
"LoopNodeConfig",
"MergeNodeConfig",
"MergeStrategy",
"NodeResult",
"NodeRunStatus",
"NodeType",
"OutputNodeConfig",
"ResumeRunRequest",
"RunStatus",
"SkillMode",
"SkillNodeConfig",
"SqlReadNodeConfig",
"StartRunRequest",
"SubworkflowNodeConfig",
"TransformNodeConfig",
"WorkflowEdge",
"WorkflowGraph",
"WorkflowGraphSettings",
"WorkflowNode",
]
Loaded 100 of 110 files, more files were not shown because too many files have changed in this diff. Show more