代码提交
This commit is contained in:
1 parent
4ac7bc21cd
commit
17cf849361
110 files changed
+16754
-265
No files matched your search
@@ -119,6 +119,14 @@ offline-backend-20260512/backend/tests/*
|
||||
!offline-backend-20260512/backend/tests/test_enterprise_research_tasks.py
|
||||
# Position-roundtable per-turn write/present cap (weak-model duplicate artifacts).
|
||||
!offline-backend-20260512/backend/tests/test_position_artifact_cap.py
|
||||
# Workflow Studio: schemas / definitions / engine / run-lifecycle regressions
|
||||
# plus the concurrency remediation suite (atomic resume, draft CAS, publish
|
||||
# counter, event seq) — required in CI per docs/WORKFLOW_STUDIO_BACKEND_REMEDIATION_ZH.md P1-06.
|
||||
!offline-backend-20260512/backend/tests/test_workflow_schemas.py
|
||||
!offline-backend-20260512/backend/tests/test_workflow_definitions.py
|
||||
!offline-backend-20260512/backend/tests/test_workflow_engine.py
|
||||
!offline-backend-20260512/backend/tests/test_workflow_runs.py
|
||||
!offline-backend-20260512/backend/tests/test_workflow_concurrency.py
|
||||
deploy
|
||||
.history
|
||||
|
||||
|
||||
@@ -73,6 +73,8 @@ The **Lead Agent** (`deerflow/agents/lead_agent/`) is the single LangGraph graph
|
||||
|
||||
**Harness → App import firewall** is enforced by `tests/test_harness_boundary.py` (runs in CI).
|
||||
|
||||
**Workflow Studio** (`deerflow/workflows/` + `/api/workflows*`) is a second, independent execution path beside the lead agent: a persisted DAG scheduler that runs published workflow graphs, streams a durable event log over SSE, and supports human-input pauses, cancellation, and crash recovery via lease reclaim. Coze Studio supplies the canvas UI only; LangGraph is used only *inside* an `agent`/`skill` node. Master switch and all limits live in `config.yaml` → `workflows`. See the backend `AGENTS.md`/`CLAUDE.md`「Workflow Studio」and `docs/WORKFLOW_STUDIO_BACKEND_DEV_ZH.md`.
|
||||
|
||||
### Frontend
|
||||
|
||||
The frontend (`frontend-web/`) is a React 19 + Vite SPA:
|
||||
|
||||
@@ -73,6 +73,8 @@ The **Lead Agent** (`deerflow/agents/lead_agent/`) is the single LangGraph graph
|
||||
|
||||
**Harness → App import firewall** is enforced by `tests/test_harness_boundary.py` (runs in CI).
|
||||
|
||||
**Workflow Studio** (`deerflow/workflows/` + `/api/workflows*`) is a second, independent execution path beside the lead agent: a persisted DAG scheduler that runs published workflow graphs, streams a durable event log over SSE, and supports human-input pauses, cancellation, and crash recovery via lease reclaim. Coze Studio supplies the canvas UI only; LangGraph is used only *inside* an `agent`/`skill` node. Master switch and all limits live in `config.yaml` → `workflows`. See the backend `CLAUDE.md`「Workflow Studio」and `docs/WORKFLOW_STUDIO_BACKEND_DEV_ZH.md`.
|
||||
|
||||
### Frontend
|
||||
|
||||
The frontend (`frontend-web/`) is a React 19 + Vite SPA:
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
import {
|
||||
collectHumanMessageIds,
|
||||
hasServerEchoedOptimisticHuman,
|
||||
isStaleStreamFinish,
|
||||
mergeStreamDisplayValues,
|
||||
shouldDiscardTerminalOverlay,
|
||||
} from "./stream-display";
|
||||
@@ -190,11 +191,9 @@ function getRunMetadataStorage(): {
|
||||
|
||||
/**
|
||||
* SDK ``useStream`` stores the in-flight run id at ``lg:stream:{threadId}``
|
||||
* but **deletes that key on stream onSuccess**. A premature SSE close
|
||||
* (proxy timeout / false-positive disconnect) looks like success, so the
|
||||
* reconnect key vanishes while the backend run is still occupying the
|
||||
* thread. We keep a second key the SDK never touches and only clear it
|
||||
* after ``runs.get`` confirms a terminal status.
|
||||
* but **deletes that key on stream onSuccess**. We keep a second key the SDK
|
||||
* never touches so a 409 / dropped-SSE exception can still rejoin. Normal
|
||||
* SSE close clears it immediately — do not wait for ``runs.get``.
|
||||
*/
|
||||
const ACTIVE_RUN_STORAGE_PREFIX = "df:active-run:";
|
||||
|
||||
@@ -276,36 +275,18 @@ export async function stopThreadRun(
|
||||
/**
|
||||
* 协助卡提交前:若上一轮还在流式(弱模型出卡后偶发未停干净),先取消服务端 run
|
||||
* 并断开本地 SSE,避免 409「已有活跃 run」或用户点了选项却发不出去。
|
||||
*
|
||||
* 不在发送前 GET /runs —— 协助卡出现时图多半已经结束,为兜底去查会让每次
|
||||
* 点选项都卡住。未在流式则直接发;真遇到 409 再走 confirmOrRejoin。
|
||||
*/
|
||||
export async function stopActiveThreadBeforeClarificationReply(
|
||||
threadId: string | null | undefined,
|
||||
thread: { isLoading?: boolean; stop: () => void | Promise<void> },
|
||||
): Promise<void> {
|
||||
let storedRunId: string | null = null;
|
||||
if (threadId && typeof window !== "undefined") {
|
||||
try {
|
||||
storedRunId =
|
||||
normalizeStoredRunId(
|
||||
window.sessionStorage.getItem(`lg:stream:${threadId}`),
|
||||
) || readActiveRunId(threadId);
|
||||
} catch {
|
||||
storedRunId = null;
|
||||
}
|
||||
}
|
||||
// 协助卡出现时图已经 jump_to=end,run 多半已是 terminal。再 stop()/cancel
|
||||
// 会再触发 onFinish → getState overlay,把刚发出的用户回复冻住、整段攒着出。
|
||||
if (storedRunId && threadId) {
|
||||
try {
|
||||
const run = await getAPIClient().runs.get(threadId, storedRunId);
|
||||
if (run && !isInflightRunStatus(run.status)) {
|
||||
clearActiveRunId(threadId);
|
||||
if (!thread.isLoading) {
|
||||
if (threadId) clearActiveRunId(threadId);
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
/* 404 / 网络:按下面的 best-effort stop 走 */
|
||||
}
|
||||
}
|
||||
if (!thread.isLoading && !storedRunId) return;
|
||||
try {
|
||||
await stopThreadRun(threadId);
|
||||
} catch {
|
||||
@@ -363,12 +344,9 @@ const TERMINAL_RUN_STATUSES = new Set([
|
||||
"interrupted",
|
||||
]);
|
||||
|
||||
// A completed server run normally sends a terminal SSE event. In local
|
||||
// development, however, a Vite hot update, a proxy disconnect, or a brief
|
||||
// network interruption can make the browser miss that final event while the
|
||||
// backend continues (and successfully completes) the run. Do not probe on
|
||||
// every render: only treat the stream as suspicious after it has been quiet
|
||||
// for a while, then reconcile it from the authoritative run/state endpoints.
|
||||
// Only probe /runs after an *exception* path has already set streamHoldOpen
|
||||
// (dropped SSE, 409, React #185). Normal Q&A must not GET /runs on finish or
|
||||
// every 5s — that made every send wait for a status round-trip.
|
||||
const STREAM_RECONCILIATION_IDLE_MS = 12_000;
|
||||
const STREAM_RECONCILIATION_POLL_MS = 5_000;
|
||||
// A run can legitimately be quiet while a long tool call / slow model write is
|
||||
@@ -608,14 +586,9 @@ export function useThreadStream({
|
||||
"terminal" | "rejoined" | "unknown"
|
||||
>
|
||||
>(async () => "unknown");
|
||||
// Keep the send button in "stop" until we confirm the backend run is
|
||||
// terminal. A premature SSE close must not look like the turn ended.
|
||||
const [streamHoldOpen, setStreamHoldOpen] = useState(
|
||||
() =>
|
||||
Boolean(
|
||||
threadId && typeof window !== "undefined" && readActiveRunId(threadId),
|
||||
),
|
||||
);
|
||||
// Held only on exception recovery (dropped SSE / 409). Normal turns rely
|
||||
// on SDK `isLoading` so the send button unlocks as soon as SSE closes.
|
||||
const [streamHoldOpen, setStreamHoldOpen] = useState(false);
|
||||
const sendInFlightRef = useRef(false);
|
||||
const sendEpochRef = useRef(0);
|
||||
// Filled after useStream; onFinish/onError may read it asynchronously.
|
||||
@@ -707,7 +680,6 @@ export function useThreadStream({
|
||||
if (meta.run_id) {
|
||||
activeRunIdRef.current = meta.run_id;
|
||||
persistActiveRunId(meta.thread_id, meta.run_id);
|
||||
setStreamHoldOpen(true);
|
||||
}
|
||||
const metadataAgentId = context.agent_id ?? context.agent_name;
|
||||
const extraMetadata =
|
||||
@@ -944,34 +916,30 @@ export function useThreadStream({
|
||||
}
|
||||
toast.error(message);
|
||||
},
|
||||
onFinish(state) {
|
||||
onFinish(state, runMeta) {
|
||||
lastStreamActivityAtRef.current = Date.now();
|
||||
// Do not trust a clean SSE close as "the run is done". streamResumable
|
||||
// chats keep the backend run alive on disconnect; the SDK still fires
|
||||
// onFinish and would otherwise restore the send button.
|
||||
//
|
||||
// If a newer turn is already streaming (common: 用户协助卡提交后,上一轮
|
||||
// interrupt 的迟到 onFinish 才到), do not freeze/rejoin — that would
|
||||
// steal the live SSE and dump tokens only at the end.
|
||||
if (threadStreamRef.current?.isLoading || sendInFlightRef.current) {
|
||||
// Trust a clean SSE close. Querying /runs here blocked every send for
|
||||
// ~5s to paper over a rare premature disconnect; that case is handled
|
||||
// when the next send hits 409 (or onError for a dropped stream).
|
||||
if (
|
||||
isStaleStreamFinish({
|
||||
finishingRunId: runMeta?.run_id,
|
||||
currentRunId: activeRunIdRef.current,
|
||||
})
|
||||
) {
|
||||
return;
|
||||
}
|
||||
setStreamHoldOpen(true);
|
||||
void confirmOrRejoinRef.current("finish").then((result) => {
|
||||
if (result === "rejoined") {
|
||||
return;
|
||||
}
|
||||
if (threadStreamRef.current?.isLoading || sendInFlightRef.current) {
|
||||
return;
|
||||
}
|
||||
setStreamHoldOpen(false);
|
||||
// run 真正终结(非 rejoin):若深度研究报告流仍 active(tool_finished
|
||||
// 帧丢失/被取消),标记中断,避免进度条永久转圈。
|
||||
const finThreadId = threadIdRef.current ?? onStreamThreadId ?? null;
|
||||
if (finThreadId) finalizeResearchReportStream(finThreadId);
|
||||
if (finThreadId) {
|
||||
clearActiveRunId(finThreadId);
|
||||
finalizeResearchReportStream(finThreadId);
|
||||
}
|
||||
activeRunIdRef.current = null;
|
||||
setStreamHoldOpen(false);
|
||||
if (state?.values) {
|
||||
listeners.current.onFinish?.(state.values);
|
||||
}
|
||||
void queryClient.invalidateQueries({ queryKey: ["threads", "search"] });
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
@@ -1009,14 +977,6 @@ export function useThreadStream({
|
||||
if (streamReconciliationInFlightRef.current && reason === "poll") {
|
||||
return "unknown";
|
||||
}
|
||||
// Stale finish/error from a prior interrupt must not joinStream over a
|
||||
// turn the user already started (协助卡提交后的实时流会被打断并攒批)。
|
||||
if (
|
||||
(reason === "finish" || reason === "error") &&
|
||||
(sendInFlightRef.current || threadStreamRef.current?.isLoading)
|
||||
) {
|
||||
return "rejoined";
|
||||
}
|
||||
streamReconciliationInFlightRef.current = true;
|
||||
const epochAtStart = sendEpochRef.current;
|
||||
try {
|
||||
@@ -1192,11 +1152,10 @@ export function useThreadStream({
|
||||
if (dropped && idleFor < 800) {
|
||||
return;
|
||||
}
|
||||
const stored = activeRunIdRef.current ?? readActiveRunId(activeThreadId);
|
||||
if (!stored && !streamHoldOpen && threadStreamRef.current?.isLoading) {
|
||||
return;
|
||||
}
|
||||
if (!stored && !streamHoldOpen) {
|
||||
// Do not probe just because a run id is stored. Only the exception
|
||||
// path sets streamHoldOpen; polling otherwise made every turn wait
|
||||
// on GET /runs before the user could send again.
|
||||
if (!streamHoldOpen) {
|
||||
return;
|
||||
}
|
||||
await confirmOrRejoin("poll");
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { Message } from "@langchain/langgraph-sdk";
|
||||
import {
|
||||
collectHumanMessageIds,
|
||||
hasServerEchoedOptimisticHuman,
|
||||
isStaleStreamFinish,
|
||||
mergeStreamDisplayValues,
|
||||
shouldDiscardTerminalOverlay,
|
||||
} from "./stream-display";
|
||||
@@ -28,6 +29,30 @@ test("live streaming ignores frozen checkpoint messages so tokens can paint", ()
|
||||
assert.deepEqual(afterStream?.messages, overrides.messages);
|
||||
});
|
||||
|
||||
test("late onFinish of a prior run is stale once a newer run is live", () => {
|
||||
assert.equal(
|
||||
isStaleStreamFinish({
|
||||
finishingRunId: "old-run",
|
||||
currentRunId: "new-run",
|
||||
}),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isStaleStreamFinish({
|
||||
finishingRunId: "run-1",
|
||||
currentRunId: "run-1",
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
isStaleStreamFinish({
|
||||
finishingRunId: "run-1",
|
||||
currentRunId: null,
|
||||
}),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("a later send discards the previous run's getState overlay", () => {
|
||||
assert.equal(
|
||||
shouldDiscardTerminalOverlay({
|
||||
|
||||
@@ -51,6 +51,25 @@ export function shouldDiscardTerminalOverlay(opts: {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when an SSE onFinish belongs to a *previous* run, after the user
|
||||
* already started a newer turn (协助卡提交是典型场景).
|
||||
*
|
||||
* Do **not** use `sendInFlight` / `isLoading` for this: `submit()` stays
|
||||
* pending until the current stream ends, so those flags are still true on a
|
||||
* normal same-turn finish and would skip unlock for every Q&A.
|
||||
*/
|
||||
export function isStaleStreamFinish(opts: {
|
||||
finishingRunId?: string | null;
|
||||
currentRunId?: string | null;
|
||||
}): boolean {
|
||||
return Boolean(
|
||||
opts.finishingRunId &&
|
||||
opts.currentRunId &&
|
||||
opts.finishingRunId !== opts.currentRunId,
|
||||
);
|
||||
}
|
||||
|
||||
export function collectHumanMessageIds(
|
||||
...groups: Array<Message[] | undefined>
|
||||
): Set<string> {
|
||||
|
||||
@@ -1,21 +1,23 @@
|
||||
"use client";
|
||||
|
||||
import {
|
||||
Bot,
|
||||
ChevronLeft,
|
||||
ChevronRight,
|
||||
Compass,
|
||||
Clock,
|
||||
DownloadIcon,
|
||||
FilePenLine,
|
||||
MessageSquare,
|
||||
PenTool,
|
||||
Pin,
|
||||
PinOff,
|
||||
Search,
|
||||
Star,
|
||||
Users2,
|
||||
type LucideIcon,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import { motion } from "motion/react";
|
||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
||||
|
||||
import { Button } from "@/components/ui/button";
|
||||
@@ -48,6 +50,9 @@ import {
|
||||
import { formatTimeAgo } from "@/core/utils/datetime";
|
||||
import { desensitize } from "@/lib/desensitize";
|
||||
import { cn } from "@/lib/utils";
|
||||
import { useAIWritingSessions } from "@/open-canvas/hooks/useAIWritingSessions";
|
||||
import { useRoundtableDrafts } from "@/roundtable-planning/hooks/useRoundtableDrafts";
|
||||
import { useResearchSessions } from "@/strategy-components/api/deep-research";
|
||||
|
||||
import { RoundtableHistoryTab } from "./chats-history/RoundtableHistoryTab";
|
||||
import { WritingHistoryTab } from "./chats-history/WritingHistoryTab";
|
||||
@@ -79,19 +84,19 @@ const HISTORY_TABS: {
|
||||
id: "roundtable",
|
||||
label: "多智能体会商",
|
||||
description: "多角色协作研讨",
|
||||
icon: Bot,
|
||||
icon: Users2,
|
||||
},
|
||||
{
|
||||
id: "writing",
|
||||
label: "AI写作",
|
||||
description: "写作任务与草稿",
|
||||
icon: FilePenLine,
|
||||
icon: PenTool,
|
||||
},
|
||||
{
|
||||
id: "research",
|
||||
label: "深度研究",
|
||||
description: "研究任务与报告",
|
||||
icon: Search,
|
||||
icon: Compass,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -126,11 +131,20 @@ export default function ChatsPage({
|
||||
const [importOpen, setImportOpen] = useState(false);
|
||||
// 非对话 Tab 的记录总数由 Tab 组件回传(客户端过滤后的条数)。
|
||||
const [tabTotal, setTabTotal] = useState<number | null>(null);
|
||||
// 顶部的数字角标与参考页一致,始终显示各类记录的实际总数,而不是当前筛选结果。
|
||||
const { drafts: roundtableTabDrafts, refresh: refreshRoundtableTabDrafts } = useRoundtableDrafts();
|
||||
const { sessions: writingTabSessions, refresh: refreshWritingTabSessions } = useAIWritingSessions();
|
||||
const { data: researchTabData } = useResearchSessions(100);
|
||||
|
||||
useEffect(() => {
|
||||
document.title = `${t.pages.chats} - ${t.pages.appName}`;
|
||||
}, [t.pages.chats, t.pages.appName]);
|
||||
|
||||
useEffect(() => {
|
||||
void refreshRoundtableTabDrafts();
|
||||
void refreshWritingTabSessions();
|
||||
}, [refreshRoundtableTabDrafts, refreshWritingTabSessions]);
|
||||
|
||||
// Debounce the search box so each keystroke doesn't fire a backend query.
|
||||
useEffect(() => {
|
||||
const timer = setTimeout(() => setDebouncedSearch(search), 300);
|
||||
@@ -224,7 +238,7 @@ export default function ChatsPage({
|
||||
"group/chat-row relative flex min-w-0 items-start gap-3 rounded-[10px] border bg-card p-3 transition-[border-color,box-shadow,background-color] duration-150 sm:items-center sm:p-3.5",
|
||||
pinned
|
||||
? "border-red-200 bg-gradient-to-r from-red-50/50 via-white to-white shadow-xs hover:border-red-300 hover:shadow-sm"
|
||||
: "border-neutral-200/80 bg-white hover:border-red-200 hover:bg-red-50/20 hover:shadow-xs",
|
||||
: "border-neutral-200/80 bg-white hover:!border-red-300 hover:!bg-gradient-to-r hover:!from-red-50/50 hover:!via-white hover:!to-white hover:shadow-xs",
|
||||
)}
|
||||
>
|
||||
<Button
|
||||
@@ -248,12 +262,12 @@ export default function ChatsPage({
|
||||
>
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
{pinned && (
|
||||
<span className="inline-flex items-center gap-1 rounded border border-red-200/50 bg-red-100/80 px-1.5 py-0.5 text-[10px] font-semibold text-red-700">
|
||||
<span className="inline-flex items-center gap-1 rounded border !border-red-200 bg-red-100/80 px-1.5 py-0.5 text-[10px] font-semibold text-red-700">
|
||||
<Pin className="size-3 rotate-45 text-red-600" />
|
||||
置顶
|
||||
</span>
|
||||
)}
|
||||
<span className="inline-flex items-center gap-1 rounded-full border border-red-200/70 bg-red-50 px-2 py-0.5 text-[11px] font-semibold text-red-700">
|
||||
<span className="inline-flex items-center gap-1 rounded-full border !border-red-200 bg-red-50 px-2 py-0.5 text-[11px] font-semibold text-red-700">
|
||||
<MessageSquare className="size-3 text-red-600" />
|
||||
对话
|
||||
</span>
|
||||
@@ -298,6 +312,12 @@ export default function ChatsPage({
|
||||
// 头部计数:对话 Tab 用后端总数,其余 Tab 用组件回传的过滤后条数。
|
||||
const displayTotal = activeTab === "chats" ? totalCount : tabTotal;
|
||||
const activeTabConfig = HISTORY_TABS.find((tab) => tab.id === activeTab)!;
|
||||
const historyTabCounts: Record<HistoryTab, number> = {
|
||||
chats: allThreadsCount ?? 0,
|
||||
roundtable: roundtableTabDrafts.length,
|
||||
writing: writingTabSessions.length,
|
||||
research: researchTabData?.total ?? 0,
|
||||
};
|
||||
|
||||
return (
|
||||
<WorkspaceContainer>
|
||||
@@ -315,15 +335,18 @@ export default function ChatsPage({
|
||||
<WorkspaceBody>
|
||||
<div className="flex size-full min-w-0 flex-col">
|
||||
<header className="flex shrink-0 flex-col items-center gap-3 pt-8">
|
||||
<div className="relative flex w-full max-w-5xl items-center justify-center px-4 sm:px-6">
|
||||
<div className="relative w-full max-w-5xl px-4 sm:px-6">
|
||||
<div className="flex w-full flex-col items-center">
|
||||
<div
|
||||
className="inline-flex max-w-full overflow-x-auto rounded-2xl border border-red-100 bg-white p-1.5 shadow-xs backdrop-blur-sm"
|
||||
id="primary-tabs-nav"
|
||||
className="inline-flex rounded-2xl border border-red-100 bg-white p-1.5 shadow-xs backdrop-blur-sm"
|
||||
role="tablist"
|
||||
aria-label="历史记录类型"
|
||||
>
|
||||
{HISTORY_TABS.map((tab) => {
|
||||
const isActive = activeTab === tab.id;
|
||||
const Icon = tab.icon;
|
||||
const count = historyTabCounts[tab.id];
|
||||
|
||||
return (
|
||||
<button
|
||||
@@ -331,7 +354,7 @@ export default function ChatsPage({
|
||||
id={`history-tab-${tab.id}`}
|
||||
type="button"
|
||||
className={cn(
|
||||
"relative flex shrink-0 select-none items-center gap-2 rounded-xl px-4 py-2 text-sm font-medium transition-all duration-200 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-red-500/30",
|
||||
"relative flex items-center gap-2 rounded-xl px-4 py-2 text-sm font-medium transition-all duration-200 select-none focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-red-500/30",
|
||||
isActive
|
||||
? "font-semibold text-red-700 shadow-xs"
|
||||
: "text-neutral-600 hover:bg-red-50/50 hover:text-red-600",
|
||||
@@ -343,24 +366,46 @@ export default function ChatsPage({
|
||||
onClick={() => setActiveTab(tab.id)}
|
||||
>
|
||||
{isActive && (
|
||||
<span
|
||||
className="absolute inset-0 rounded-xl border border-red-200 bg-red-50/90 shadow-xs"
|
||||
<motion.div
|
||||
layoutId="activeTabBadge"
|
||||
className="absolute inset-0 rounded-xl border !border-red-200 bg-red-50/90 shadow-xs"
|
||||
transition={{ type: "spring", stiffness: 450, damping: 35 }}
|
||||
aria-hidden="true"
|
||||
/>
|
||||
)}
|
||||
<span className="relative z-10 flex items-center gap-2">
|
||||
<span
|
||||
className={cn(
|
||||
"relative z-10 flex items-center gap-2",
|
||||
isActive
|
||||
? "font-semibold text-red-700"
|
||||
: "text-neutral-600",
|
||||
)}
|
||||
>
|
||||
<Icon className="size-4" aria-hidden="true" />
|
||||
<span>{tab.label}</span>
|
||||
</span>
|
||||
{count > 0 && (
|
||||
<span
|
||||
className={cn(
|
||||
"relative z-10 rounded-full px-1.5 py-[0.05rem] font-mono text-[11px] transition-colors",
|
||||
isActive
|
||||
? "bg-red-600 font-bold text-white"
|
||||
: "bg-neutral-100 text-neutral-500",
|
||||
)}
|
||||
>
|
||||
{count}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
{activeTab === "chats" && (
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="absolute right-4 sm:right-6"
|
||||
className="absolute right-4 top-1/2 -translate-y-1/2 sm:right-6"
|
||||
onClick={() => setImportOpen(true)}
|
||||
>
|
||||
<DownloadIcon className="size-4" />
|
||||
@@ -376,9 +421,9 @@ export default function ChatsPage({
|
||||
id={`history-panel-${activeTab}`}
|
||||
role="tabpanel"
|
||||
aria-labelledby={`history-tab-${activeTab}`}
|
||||
className="space-y-4 rounded-2xl border border-red-100 bg-white p-4 shadow-xs sm:p-5"
|
||||
className="space-y-4 rounded-2xl border border-red-200 bg-white p-4 shadow-xs sm:p-5"
|
||||
>
|
||||
<div className="w-full space-y-3.5">
|
||||
{activeTab !== "roundtable" && <div className="w-full space-y-3.5">
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
{activeTab === "chats" ? (
|
||||
<div className="flex items-center gap-1.5 overflow-x-auto pb-1 md:pb-0" role="group" aria-label="对话筛选">
|
||||
@@ -387,7 +432,7 @@ export default function ChatsPage({
|
||||
className={cn(
|
||||
"inline-flex shrink-0 items-center gap-1.5 rounded-lg px-3.5 py-1.5 text-xs font-medium transition-all focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-red-500/30",
|
||||
chatFilter === "all"
|
||||
? "bg-red-600 font-semibold text-white shadow-xs"
|
||||
? "border !border-red-600 bg-red-600 font-semibold text-white shadow-xs"
|
||||
: "border border-neutral-200 bg-white text-neutral-600 hover:border-red-200 hover:bg-red-50/60 hover:text-red-700",
|
||||
)}
|
||||
aria-pressed={chatFilter === "all"}
|
||||
@@ -499,7 +544,7 @@ export default function ChatsPage({
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>}
|
||||
{activeTab === "chats" && (
|
||||
<div className="space-y-3 pt-5">
|
||||
{currentPage === 1 && pinnedThreads.map(renderThread)}
|
||||
@@ -543,12 +588,7 @@ export default function ChatsPage({
|
||||
</div>
|
||||
)}
|
||||
{activeTab === "roundtable" && (
|
||||
<div className="pt-5">
|
||||
<RoundtableHistoryTab
|
||||
query={search}
|
||||
onTotalChange={handleTabTotalChange}
|
||||
/>
|
||||
</div>
|
||||
<RoundtableHistoryTab onTotalChange={handleTabTotalChange} />
|
||||
)}
|
||||
{activeTab === "writing" && (
|
||||
<div className="pt-5">
|
||||
|
||||
@@ -3,113 +3,326 @@
|
||||
/**
|
||||
* 「对话历史」页 —— 多智能体会商 Tab。
|
||||
*
|
||||
* 数据与原侧栏「多智能体会商」列表同源(当前用户的个人草稿 roundtable_drafts,
|
||||
* useRoundtableDrafts 不带 taskId);点击行为也一致:暂存 sessionStorage
|
||||
* 「roundtable:pending-draft」后跳转会商页,由页面挂载时消费并加载该草稿。
|
||||
* 任务维度会商(task-draft,按 taskId 全任务共享、不分权)不随个人历史走,
|
||||
* 仍只在对应任务工作区里查看。
|
||||
* 卡片与筛选区按最新「智能工作区管理 (1)」的 ConversationRow /
|
||||
* FilterAndSearchBar 适配。会商步骤、状态、模式、互动数和模型都来自实际草稿快照。
|
||||
*/
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { useNavigate } from "react-router-dom";
|
||||
import { Bot, Share2 } from "lucide-react";
|
||||
import {
|
||||
ArrowUpDown,
|
||||
Clock,
|
||||
Cpu,
|
||||
ExternalLink,
|
||||
Search,
|
||||
Tag,
|
||||
Trash2,
|
||||
Users2,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { toast } from "sonner";
|
||||
|
||||
import { ShareRoundtableDraftDialog } from "@/components/workspace/roundtable-draft-share-dialogs";
|
||||
import { formatRelative } from "@/open-canvas/components/ai-writing/AIWritingHistoryDropdown";
|
||||
import { useRoundtableDrafts } from "@/roundtable-planning/hooks/useRoundtableDrafts";
|
||||
import type { DraftMeta } from "@/roundtable-planning/api/drafts";
|
||||
import { desensitize } from "@/lib/desensitize";
|
||||
|
||||
import {
|
||||
DeleteIcon,
|
||||
HistoryRenameDialog,
|
||||
HistoryRow,
|
||||
HistoryRowIconButton,
|
||||
HistoryStatusPill,
|
||||
HistoryTypeBadge,
|
||||
RenameIcon,
|
||||
} from "./history-tab-shared";
|
||||
getDraft,
|
||||
type DraftMeta,
|
||||
type DraftRecord,
|
||||
} from "@/roundtable-planning/api/drafts";
|
||||
import { useRoundtableDrafts } from "@/roundtable-planning/hooks/useRoundtableDrafts";
|
||||
import { desensitize } from "@/lib/desensitize";
|
||||
|
||||
const ROUNDTABLE_PATH = "/page/workspace/roundtable/planning";
|
||||
|
||||
const ROUNDTABLE_STEP_LABEL: Record<DraftMeta["furthestStep"], string> = {
|
||||
1: "议题梳理",
|
||||
2: "多智能体会商",
|
||||
3: "报告生成",
|
||||
};
|
||||
type SortType = "updated_desc" | "updated_asc" | "messages_desc";
|
||||
|
||||
interface RoundtableItem {
|
||||
draft: DraftMeta;
|
||||
record?: DraftRecord;
|
||||
status: ReturnType<typeof progressOf>;
|
||||
businessChainName: string;
|
||||
messageCount: number;
|
||||
model?: string;
|
||||
tags: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* 列表接口会返回草稿已推进到的工作台步骤以及最新后台作业状态。两者须同时展示:
|
||||
* `furthestStep` 告诉用户做到哪一环,`status` 告诉用户是否已真正完成或被中断。
|
||||
*/
|
||||
function progressOf(draft: DraftMeta): {
|
||||
label: string;
|
||||
className: string;
|
||||
stepLabel: string;
|
||||
tone: "completed" | "active" | "waiting" | "failed" | "cancelled";
|
||||
} {
|
||||
const stepLabel = `第 ${draft.furthestStep} 步 · ${ROUNDTABLE_STEP_LABEL[draft.furthestStep]}`;
|
||||
const stepLabel = `第 ${draft.furthestStep} 步`;
|
||||
|
||||
if (draft.status === "done" || (!draft.status && draft.furthestStep === 3)) {
|
||||
return { label: "已完成", className: "text-emerald-600", stepLabel };
|
||||
return { label: "已完成", tone: "completed", stepLabel };
|
||||
}
|
||||
if (draft.status === "error") {
|
||||
return { label: "执行失败", className: "text-rose-500", stepLabel: `停在${stepLabel}` };
|
||||
return { label: "执行失败", tone: "failed", stepLabel: `停在${stepLabel}` };
|
||||
}
|
||||
if (draft.status === "cancelled") {
|
||||
return { label: "已取消", className: "text-muted-foreground", stepLabel: `停在${stepLabel}` };
|
||||
return { label: "已取消", tone: "cancelled", stepLabel: `停在${stepLabel}` };
|
||||
}
|
||||
if (draft.status === "awaiting_input") {
|
||||
return { label: "等待确认", className: "text-amber-500", stepLabel: `当前${stepLabel}` };
|
||||
return { label: "等待确认", tone: "waiting", stepLabel: `当前${stepLabel}` };
|
||||
}
|
||||
return { label: "进行中", className: "text-primary", stepLabel: `当前${stepLabel}` };
|
||||
return { label: "进行中", tone: "active", stepLabel: `当前${stepLabel}` };
|
||||
}
|
||||
|
||||
function modeOf(record?: DraftRecord): string {
|
||||
switch (record?.step2?.orchestrationMode) {
|
||||
case "chain":
|
||||
return "业务链会商";
|
||||
case "dag":
|
||||
return "DAG 编排";
|
||||
default:
|
||||
return "自由模式";
|
||||
}
|
||||
}
|
||||
|
||||
function businessChainNameOf(draft: DraftMeta, record?: DraftRecord): string {
|
||||
return record?.step2?.chain?.title ?? draft.chainTitle ?? modeOf(record);
|
||||
}
|
||||
|
||||
function interactionCount(record?: DraftRecord): number {
|
||||
if (!record) return 0;
|
||||
return (
|
||||
(record.step1?.interactiveMessages?.length ?? 0) +
|
||||
(record.step2?.step2RoundtableDialogues?.length ?? 0) +
|
||||
(record.step3?.summaryReport?.dialogues?.length ?? 0) +
|
||||
(record.step3?.reportDialogues?.length ?? 0) +
|
||||
(record.step3?.dashboardDialogues?.length ?? 0)
|
||||
);
|
||||
}
|
||||
|
||||
function modelOf(record?: DraftRecord): string | undefined {
|
||||
return record?.step3?.summaryReport?.model ?? record?.step3?.model;
|
||||
}
|
||||
|
||||
function tagsOf(draft: DraftMeta, record?: DraftRecord): string[] {
|
||||
const chain = record?.step2?.chain?.title ?? draft.chainTitle;
|
||||
return chain ? [chain] : [];
|
||||
}
|
||||
|
||||
function timestamp(value: string): number {
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isFinite(parsed) ? parsed : 0;
|
||||
}
|
||||
|
||||
function renderHighlightedTitle(text: string, query: string) {
|
||||
if (!query.trim()) return text;
|
||||
const escaped = query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const parts = text.split(new RegExp(`(${escaped})`, "gi"));
|
||||
return parts.map((part, index) =>
|
||||
part.toLowerCase() === query.toLowerCase() ? (
|
||||
<mark key={index} className="rounded-xs bg-red-100 px-0.5 font-semibold text-red-900">
|
||||
{part}
|
||||
</mark>
|
||||
) : (
|
||||
part
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function RoundtableHistoryRow({
|
||||
item,
|
||||
searchQuery,
|
||||
onDelete,
|
||||
onOpen,
|
||||
}: {
|
||||
item: RoundtableItem;
|
||||
searchQuery: string;
|
||||
onDelete: () => void;
|
||||
onOpen: () => void;
|
||||
}) {
|
||||
const { draft, status } = item;
|
||||
const statusClass = {
|
||||
completed: "border-emerald-200/80 bg-emerald-50 text-emerald-700",
|
||||
active: "border-red-200/80 bg-red-50 text-red-600",
|
||||
waiting: "border-amber-200/80 bg-amber-50 text-amber-700",
|
||||
failed: "border-rose-200/80 bg-rose-50 text-rose-600",
|
||||
cancelled: "border-neutral-200/80 bg-neutral-50 text-neutral-500",
|
||||
}[status.tone];
|
||||
const statusDotClass = {
|
||||
completed: "bg-emerald-500",
|
||||
active: "bg-red-500",
|
||||
waiting: "bg-amber-500",
|
||||
failed: "bg-rose-500",
|
||||
cancelled: "bg-neutral-400",
|
||||
}[status.tone];
|
||||
|
||||
return (
|
||||
<div
|
||||
id={`conversation-item-${draft.id}`}
|
||||
onClick={(event) => {
|
||||
if ((event.target as HTMLElement).closest("button, input")) return;
|
||||
onOpen();
|
||||
}}
|
||||
className="group relative flex cursor-pointer items-start justify-between rounded-xl border border-neutral-200/80 bg-white p-4 transition-all duration-150 hover:!border-red-300 hover:!bg-gradient-to-r hover:!from-red-50/50 hover:!via-white hover:!to-white hover:shadow-xs sm:items-center"
|
||||
>
|
||||
<div className="flex min-w-0 flex-1 items-start gap-3.5 pr-4 sm:items-center">
|
||||
<div className="min-w-0 flex-1 space-y-1.5">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span className="inline-flex items-center gap-1.5 rounded-full border !border-red-200 bg-red-50/90 px-2 py-0.5 text-[11px] font-semibold text-red-700">
|
||||
<span className="relative flex size-1.5">
|
||||
{status.tone === "active" && (
|
||||
<span className="absolute inline-flex size-full animate-ping rounded-full bg-red-400 opacity-75" />
|
||||
)}
|
||||
<span className="relative inline-flex size-1.5 rounded-full bg-red-600" />
|
||||
</span>
|
||||
<Users2 className="size-3 text-red-600" />
|
||||
会商
|
||||
</span>
|
||||
<h3 className="truncate text-sm font-semibold text-neutral-900 transition-colors group-hover:text-red-700">
|
||||
{renderHighlightedTitle(desensitize(draft.title || "未命名会商"), searchQuery)}
|
||||
</h3>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-wrap items-center gap-2 pt-0.5 text-[11px] text-neutral-400">
|
||||
<span className={`inline-flex items-center gap-1 rounded border !border-red-200 px-1.5 py-0.5 text-[10px] font-medium ${statusClass}`}>
|
||||
<span className={`size-1.5 rounded-full ${statusDotClass}`} />
|
||||
{status.label}
|
||||
</span>
|
||||
<span className="inline-flex items-center rounded border border-neutral-200/60 bg-neutral-100 px-1.5 py-0.5 text-[10px] font-medium text-neutral-600">
|
||||
{status.stepLabel}
|
||||
</span>
|
||||
<span className="inline-flex items-center gap-1 rounded border border-neutral-200/40 bg-neutral-50 px-1.5 py-0.5 text-[10px] text-neutral-500">
|
||||
<Cpu className="size-2.5 text-neutral-400" />
|
||||
{item.businessChainName}
|
||||
</span>
|
||||
<span className="inline-flex items-center gap-1 text-neutral-400">
|
||||
<Clock className="size-3 text-neutral-400" />
|
||||
{formatRelative(draft.updatedAt)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex shrink-0 items-center gap-3">
|
||||
<div className="hidden items-center gap-1 opacity-0 transition-opacity group-hover:opacity-100 sm:flex">
|
||||
<button
|
||||
id={`delete-btn-${draft.id}`}
|
||||
type="button"
|
||||
onClick={(event) => {
|
||||
event.stopPropagation();
|
||||
onDelete();
|
||||
}}
|
||||
className="rounded-md p-1.5 text-neutral-400 transition-colors hover:bg-red-50 hover:text-red-600"
|
||||
title="删除会商"
|
||||
aria-label="删除会商"
|
||||
>
|
||||
<Trash2 className="size-3.5" />
|
||||
</button>
|
||||
<button
|
||||
id={`open-preview-btn-${draft.id}`}
|
||||
type="button"
|
||||
onClick={(event) => {
|
||||
event.stopPropagation();
|
||||
onOpen();
|
||||
}}
|
||||
className="rounded-md p-1.5 text-neutral-500 transition-colors hover:bg-red-50 hover:text-red-700"
|
||||
title="打开会商详情"
|
||||
aria-label="打开会商详情"
|
||||
>
|
||||
<ExternalLink className="size-3.5" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function RoundtableHistoryTab({
|
||||
query,
|
||||
onTotalChange,
|
||||
}: {
|
||||
query: string;
|
||||
onTotalChange?: (total: number) => void;
|
||||
}) {
|
||||
const navigate = useNavigate();
|
||||
const { drafts, loading, refresh, rename, remove } = useRoundtableDrafts();
|
||||
const [renameId, setRenameId] = useState<string | null>(null);
|
||||
const [renameTitle, setRenameTitle] = useState("");
|
||||
const [shareId, setShareId] = useState<string | null>(null);
|
||||
const { drafts, loading, refresh, remove } = useRoundtableDrafts();
|
||||
const [records, setRecords] = useState<Record<string, DraftRecord>>({});
|
||||
const [searchQuery, setSearchQuery] = useState("");
|
||||
const [selectedTag, setSelectedTag] = useState<string | null>(null);
|
||||
const [sortType, setSortType] = useState<SortType>("updated_desc");
|
||||
|
||||
useEffect(() => {
|
||||
void refresh();
|
||||
}, [refresh]);
|
||||
|
||||
const keyword = query.trim().toLowerCase();
|
||||
const filtered = keyword
|
||||
? drafts.filter((draft) =>
|
||||
(draft.title || "未命名草稿").toLowerCase().includes(keyword),
|
||||
)
|
||||
: drafts;
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
if (drafts.length === 0) {
|
||||
setRecords({});
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}
|
||||
|
||||
void Promise.all(
|
||||
drafts.map(async (draft) => {
|
||||
try {
|
||||
return await getDraft(draft.id);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
).then((fullDrafts) => {
|
||||
if (cancelled) return;
|
||||
setRecords(
|
||||
Object.fromEntries(
|
||||
fullDrafts
|
||||
.filter((record): record is DraftRecord => record !== null)
|
||||
.map((record) => [record.id, record]),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [drafts]);
|
||||
|
||||
const allItems = useMemo<RoundtableItem[]>(
|
||||
() =>
|
||||
drafts.map((draft) => {
|
||||
const record = records[draft.id];
|
||||
return {
|
||||
draft,
|
||||
record,
|
||||
status: progressOf(draft),
|
||||
businessChainName: businessChainNameOf(draft, record),
|
||||
messageCount: interactionCount(record),
|
||||
model: modelOf(record),
|
||||
tags: tagsOf(draft, record),
|
||||
};
|
||||
}),
|
||||
[drafts, records],
|
||||
);
|
||||
const availableTags = useMemo(
|
||||
() => [...new Set(allItems.flatMap((item) => item.tags))],
|
||||
[allItems],
|
||||
);
|
||||
const filteredItems = useMemo(() => {
|
||||
const keyword = searchQuery.trim().toLowerCase();
|
||||
return allItems
|
||||
.filter((item) => {
|
||||
if (selectedTag && !item.tags.includes(selectedTag)) return false;
|
||||
return !keyword || (item.draft.title || "未命名会商").toLowerCase().includes(keyword);
|
||||
})
|
||||
.sort((left, right) => {
|
||||
if (sortType === "messages_desc") return right.messageCount - left.messageCount;
|
||||
const delta = timestamp(right.draft.updatedAt) - timestamp(left.draft.updatedAt);
|
||||
return sortType === "updated_desc" ? delta : -delta;
|
||||
});
|
||||
}, [allItems, searchQuery, selectedTag, sortType]);
|
||||
|
||||
useEffect(() => {
|
||||
onTotalChange?.(filtered.length);
|
||||
}, [filtered.length, onTotalChange]);
|
||||
onTotalChange?.(filteredItems.length);
|
||||
}, [filteredItems.length, onTotalChange]);
|
||||
|
||||
const openDraft = (draft: DraftMeta) => {
|
||||
sessionStorage.setItem("roundtable:pending-draft", draft.id);
|
||||
navigate(ROUNDTABLE_PATH);
|
||||
};
|
||||
|
||||
const handleRename = async (id: string, title: string) => {
|
||||
try {
|
||||
await rename(id, title);
|
||||
toast.success("已重命名");
|
||||
} catch {
|
||||
toast.error("重命名失败,请重试");
|
||||
} finally {
|
||||
setRenameId(null);
|
||||
}
|
||||
};
|
||||
|
||||
const handleDelete = async (draft: DraftMeta) => {
|
||||
const deleteDraft = async (draft: DraftMeta) => {
|
||||
if (!window.confirm("确定删除这条会商记录吗?此操作不可恢复。")) return;
|
||||
try {
|
||||
await remove(draft.id);
|
||||
@@ -119,90 +332,135 @@ export function RoundtableHistoryTab({
|
||||
}
|
||||
};
|
||||
|
||||
if (loading && drafts.length === 0) {
|
||||
return (
|
||||
<div className="text-muted-foreground py-16 text-center text-sm">
|
||||
加载中…
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (filtered.length === 0) {
|
||||
return (
|
||||
<div className="text-muted-foreground py-16 text-center text-sm">
|
||||
{keyword ? "没有匹配的会商记录" : "暂无会商记录"}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<div>
|
||||
{filtered.map((draft) => {
|
||||
const progress = progressOf(draft);
|
||||
return (
|
||||
<HistoryRow
|
||||
key={draft.id}
|
||||
badge={<HistoryTypeBadge icon={Bot} label="会商" />}
|
||||
title={desensitize(draft.title || "未命名草稿")}
|
||||
meta={
|
||||
<>
|
||||
<HistoryStatusPill
|
||||
label={progress.label}
|
||||
className={progress.className}
|
||||
/>
|
||||
<span className="text-muted-foreground/60">·</span>
|
||||
<span className="max-w-48 truncate" title={progress.stepLabel}>
|
||||
{progress.stepLabel}
|
||||
<div className="w-full space-y-4">
|
||||
<div className="w-full space-y-3.5">
|
||||
<div className="flex flex-col items-stretch justify-between gap-3 md:flex-row md:items-center">
|
||||
<div className="flex items-center gap-1.5 overflow-x-auto pb-1 md:pb-0">
|
||||
<button
|
||||
id="filter-all-btn"
|
||||
type="button"
|
||||
onClick={() => setSelectedTag(null)}
|
||||
className={`inline-flex items-center gap-1.5 rounded-lg px-3.5 py-1.5 text-xs font-medium transition-all ${
|
||||
!selectedTag
|
||||
? "border !border-red-600 bg-red-600 font-semibold text-white shadow-xs"
|
||||
: "border border-neutral-200 bg-white text-neutral-600 hover:border-red-200 hover:bg-red-50/60 hover:text-red-700"
|
||||
}`}
|
||||
>
|
||||
<span>全部</span>
|
||||
<span className={`rounded-full px-1.5 py-0.2 font-mono text-[11px] ${!selectedTag ? "bg-red-700 font-semibold text-white" : "bg-neutral-100 text-neutral-500"}`}>
|
||||
{allItems.length}
|
||||
</span>
|
||||
<span className="text-muted-foreground/60">·</span>
|
||||
<span className="max-w-40 truncate" title={draft.chainTitle || "自由模式"}>
|
||||
{draft.chainTitle || "自由模式"}
|
||||
</span>
|
||||
<span className="text-muted-foreground/60">·</span>
|
||||
<span>{formatRelative(draft.updatedAt)}</span>
|
||||
</>
|
||||
}
|
||||
onOpen={() => openDraft(draft)}
|
||||
</button>
|
||||
{selectedTag && (
|
||||
<div className="inline-flex items-center gap-1 rounded-lg bg-red-600 px-2.5 py-1.5 text-xs font-medium text-white shadow-xs">
|
||||
<Tag className="size-3 text-red-100" />
|
||||
<span>{selectedTag}</span>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setSelectedTag(null)}
|
||||
className="ml-1 hover:text-red-200"
|
||||
title="清除标签筛选"
|
||||
aria-label="清除标签筛选"
|
||||
>
|
||||
<HistoryRowIconButton
|
||||
title="重命名"
|
||||
onClick={() => {
|
||||
setRenameId(draft.id);
|
||||
setRenameTitle(draft.title || "");
|
||||
}}
|
||||
>
|
||||
<RenameIcon />
|
||||
</HistoryRowIconButton>
|
||||
<HistoryRowIconButton title="分享" onClick={() => setShareId(draft.id)}>
|
||||
<Share2 className="text-muted-foreground size-4" />
|
||||
</HistoryRowIconButton>
|
||||
<HistoryRowIconButton
|
||||
title="删除"
|
||||
onClick={() => void handleDelete(draft)}
|
||||
>
|
||||
<DeleteIcon />
|
||||
</HistoryRowIconButton>
|
||||
</HistoryRow>
|
||||
);
|
||||
})}
|
||||
<X className="size-3" />
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<HistoryRenameDialog
|
||||
open={renameId !== null}
|
||||
initialTitle={renameTitle}
|
||||
dialogTitle="重命名会商"
|
||||
onCancel={() => setRenameId(null)}
|
||||
onConfirm={(title) => renameId && void handleRename(renameId, title)}
|
||||
<div className="relative flex-1 md:max-w-md">
|
||||
<Search className="pointer-events-none absolute left-3.5 top-1/2 size-4 -translate-y-1/2 text-neutral-400" />
|
||||
<input
|
||||
id="conversation-search-input"
|
||||
type="search"
|
||||
value={searchQuery}
|
||||
onChange={(event) => setSearchQuery(event.target.value)}
|
||||
placeholder="搜索会商记录…"
|
||||
className="w-full rounded-xl border border-neutral-200 bg-neutral-50/80 py-2 pl-9 pr-16 text-sm text-neutral-900 transition-all placeholder:text-neutral-400 hover:bg-white focus:border-red-500 focus:bg-white focus:outline-hidden focus:ring-2 focus:ring-red-500/20"
|
||||
/>
|
||||
{searchQuery ? (
|
||||
<button
|
||||
id="clear-search-query-btn"
|
||||
type="button"
|
||||
onClick={() => setSearchQuery("")}
|
||||
className="absolute right-3 top-1/2 -translate-y-1/2 p-0.5 text-neutral-400 hover:text-red-600"
|
||||
aria-label="清空搜索"
|
||||
>
|
||||
<X className="size-3.5" />
|
||||
</button>
|
||||
) : (
|
||||
<div className="pointer-events-none absolute right-3 top-1/2 hidden -translate-y-1/2 items-center gap-0.5 rounded bg-neutral-200/60 px-1.5 py-0.5 font-mono text-[10px] text-neutral-400 sm:flex">
|
||||
⌘K
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<ShareRoundtableDraftDialog
|
||||
draftId={shareId}
|
||||
open={shareId !== null}
|
||||
onOpenChange={(o) => {
|
||||
if (!o) setShareId(null);
|
||||
}}
|
||||
<div className="flex flex-wrap items-center justify-between gap-2 border-t border-red-50 pt-1 text-xs text-neutral-500">
|
||||
<div className="flex items-center gap-3 overflow-x-auto">
|
||||
<span className="whitespace-nowrap font-medium text-neutral-500">
|
||||
共 <strong className="font-bold text-red-600">{filteredItems.length}</strong> 条会商记录
|
||||
{searchQuery && <span className="ml-1 text-neutral-400">(匹配 “{searchQuery}”)</span>}
|
||||
</span>
|
||||
{availableTags.length > 0 && (
|
||||
<div className="hidden items-center gap-1.5 text-xs lg:flex">
|
||||
<span className="text-neutral-400">快捷标签:</span>
|
||||
{availableTags.slice(0, 4).map((tag) => (
|
||||
<button
|
||||
key={tag}
|
||||
type="button"
|
||||
onClick={() => setSelectedTag(selectedTag === tag ? null : tag)}
|
||||
className={`rounded-md px-2 py-0.5 transition-colors ${
|
||||
selectedTag === tag
|
||||
? "bg-red-600 font-medium text-white shadow-2xs"
|
||||
: "border !border-red-200 bg-red-50 text-red-700 hover:bg-red-100 hover:text-red-800"
|
||||
}`}
|
||||
>
|
||||
#{tag}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="flex items-center gap-1 rounded-lg border border-neutral-200 bg-white px-2 py-1 shadow-2xs">
|
||||
<ArrowUpDown className="size-3 text-neutral-400" />
|
||||
<select
|
||||
id="sort-select-dropdown"
|
||||
value={sortType}
|
||||
onChange={(event) => setSortType(event.target.value as SortType)}
|
||||
className="cursor-pointer bg-transparent text-xs text-neutral-700 focus:outline-hidden"
|
||||
>
|
||||
<option value="updated_desc">最近活跃优先</option>
|
||||
<option value="updated_asc">最早创建优先</option>
|
||||
<option value="messages_desc">消息数量最多</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{loading && allItems.length === 0 ? (
|
||||
<div className="py-16 text-center text-sm text-muted-foreground">加载中…</div>
|
||||
) : filteredItems.length === 0 ? (
|
||||
<div className="py-16 text-center text-sm text-muted-foreground">
|
||||
{searchQuery || selectedTag ? "没有匹配的会商记录" : "暂无会商记录"}
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-2">
|
||||
{filteredItems.map((item) => (
|
||||
<RoundtableHistoryRow
|
||||
key={item.draft.id}
|
||||
item={item}
|
||||
searchQuery={searchQuery}
|
||||
onDelete={() => void deleteDraft(item.draft)}
|
||||
onOpen={() => openDraft(item.draft)}
|
||||
/>
|
||||
</>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -43,7 +43,7 @@ export function HistoryTypeBadge({
|
||||
label: string;
|
||||
}) {
|
||||
return (
|
||||
<span className="inline-flex shrink-0 items-center gap-1 rounded-full border border-red-200/70 bg-red-50 px-2 py-0.5 text-[11px] font-semibold text-red-700">
|
||||
<span className="inline-flex shrink-0 items-center gap-1 rounded-full border !border-red-200 bg-red-50 px-2 py-0.5 text-[11px] font-semibold text-red-700">
|
||||
<Icon className="size-3 text-red-600" />
|
||||
{label}
|
||||
</span>
|
||||
@@ -68,7 +68,7 @@ export function HistoryRow({
|
||||
children?: ReactNode;
|
||||
}) {
|
||||
return (
|
||||
<div className="group/chat-row relative mb-3 flex min-w-0 rounded-[10px] border border-neutral-200/80 bg-white transition-[border-color,box-shadow,background-color] duration-150 last:mb-0 hover:border-red-200 hover:bg-red-50/20 hover:shadow-xs">
|
||||
<div className="group/chat-row relative mb-3 flex min-w-0 rounded-[10px] border border-neutral-200/80 bg-white transition-[border-color,box-shadow,background-color] duration-150 last:mb-0 hover:!border-red-300 hover:!bg-gradient-to-r hover:!from-red-50/50 hover:!via-white hover:!to-white hover:shadow-xs">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onOpen}
|
||||
|
||||
@@ -199,6 +199,39 @@ The backend is split into two layers with a strict dependency direction:
|
||||
|
||||
**Dependency rule**: App imports deerflow, but deerflow never imports app. This boundary is enforced by `tests/test_harness_boundary.py` which runs in CI.
|
||||
|
||||
### Workflow Studio
|
||||
|
||||
Coze Studio is canvas-only; ChatDev is reference-only. Plan + status per phase: `docs/WORKFLOW_STUDIO_BACKEND_DEV_ZH.md`; decisions: `docs/adr/0001-workflow-studio-runtime.md`; contracts: `docs/WORKFLOW_SCHEMA_V1_ZH.md` + `docs/WORKFLOW_SSE_V1_ZH.md` + `docs/WORKFLOW_OPENAPI_DRAFT_V1.yaml`. Master switch + all limits: `config.yaml` → `workflows` (`deerflow.config.workflow_config.WorkflowConfig`).
|
||||
|
||||
**Execution kernel is a custom DAG scheduler, not a compiled LangGraph.** LangGraph stays the kernel *inside* an `agent`/`skill` node. The reason is single-source-of-truth recovery: a re-leased or resumed run replays completed node outputs from `workflow_node_runs` plus the `workflow_run_events` log, so a second (checkpoint) copy of workflow-level state would have no defensible tie-breaker. See the ADR's phase-2 amendment.
|
||||
|
||||
**Harness** (`packages/harness/deerflow/workflows/`, never imports `app.*`):
|
||||
- `schemas` / `events` / `errors` / `ports` / `validator` / `samples` — frozen v1 contracts.
|
||||
- `expressions.py` — `{{ inputs.x }}` / `{{ nodes.n.data.y }}` templates and conditions resolved by path walking + a small comparator set. **No `eval`.** Roots are restricted to `inputs` / `nodes` / `run` / `loop` / `env`.
|
||||
- `runtime/engine.py` — the scheduler. Every edge is `pending` → `active` (source chose this port) or `pruned`; a node is ready when its non-back incoming edges are all resolved and at least one is `active`, and is skipped when all are `pruned` (skips propagate). That one rule covers fan-out, condition branching, and merge joins with no topological pre-pass. The `loop` node's `continue` back edge is the only permitted cycle; re-entry clears the body's results and invalidates its node runs.
|
||||
- `runtime/{sink,sql_runner,code_runner,context}.py`, `nodes/*` (13 executors), `security/{http_policy,sql_policy,secret_box}.py`.
|
||||
|
||||
**Persistence**: `workflow_definitions` / `workflow_versions` (migration `20260828_01`), `workflow_runs` / `workflow_node_runs` / `workflow_run_events` / `workflow_run_artifacts` (`20260828_02`), `workflow_data_sources` (`20260828_03`). Stores under `deerflow.persistence.workflow_{runs,events,data_sources}` with SQL + in-memory implementations.
|
||||
|
||||
**App layer**: `workflow_dispatcher.py` (lease scan → atomic claim; an expired lease is how a dead worker's run is reclaimed, so every worker can run the loop), `workflow_executor.py` (one lease's lifecycle: replay → engine under a run timeout with heartbeat + cancel watcher → exactly one terminal transition; also runs subworkflows inline, depth ≤ 3), `workflow_retention.py` (hourly sweep of `workflows.retention.event_retention_days` / `run_retention_days`), `workflow_live_hub.py` (in-process SSE fan-out), `workflow_agent_runner.py` (agent/skill nodes → lead agent).
|
||||
|
||||
**Gateway routes** (register static prefixes before `workflows.router`'s `/{workflow_id}` catch-alls): `/api/workflows` CRUD/draft/validate/publish/copy, `/api/workflows/node-types` + `/resources/*`, `/api/workflows/{id}/runs` + `/api/workflows/runs/{run_id}{,/events,/stream,/cancel,/resume,/retry,/artifacts}`, `/api/workflows/data-sources` (+ `/{id}/introspect`, `/{id}/validate-query`), `/api/workflows/embed/{tickets,verify}`, `/api/workflow_api/*` Coze compat.
|
||||
|
||||
**Guarantees worth not breaking**: events are persisted *then* published, and the DB is authoritative for `seq` ordering (live frames at or below the replay cursor are dropped and re-read on the next poll, so a reconnect sees no gap; `Last-Event-ID` wins over `?after=`); a run reaching a terminal status always gets a terminal event, synthesised if necessary, or SSE clients hang; `resumeToken` appears only in the `run.awaiting_input` event, never in the run view; `POST /retry` on a `failed` run copies completed node outputs onto a new run marked `retry_of_run_id` so the engine resumes from the failed node; data-source secrets are write-only (encrypted via `WORKFLOW_SECRET_KEY`, decrypted only inside the executor, reads expose `masked_target`); the `http` node denies private/loopback/link-local targets by default (SSRF), `sql_read` allows one read-only statement with bound parameters, and the `code` node is **off by default** because its runner is a hardened subprocess rather than a container.
|
||||
|
||||
HTTP interface resources persist a non-secret base_url plus an allowed_methods allowlist (migration
|
||||
20260829_01); request headers stay only in the encrypted secret blob. The resource catalog may return
|
||||
the address and methods to the owning user, but must never return headers or any decrypted secret. The
|
||||
executor receives HTTP metadata only through its data-source resolver.
|
||||
|
||||
The Workflow Studio agent/skill resource catalog is a read-only display projection over the existing
|
||||
agent and skill ownership stores. It returns stable `agentId` / `skillId`, an agent's cached skill names,
|
||||
and a `createdBy` display label. Resolve human labels from the immutable owner id at request time and
|
||||
fall back to that id if lookup fails; built-in records must return `系统内置`. Do not create a parallel
|
||||
creator table or accept creator data from the browser.
|
||||
|
||||
Tests: `tests/test_workflow_schemas.py`, `tests/test_workflow_definitions.py`, `tests/test_workflow_engine.py`, `tests/test_workflow_runs.py`.
|
||||
|
||||
**Import conventions**:
|
||||
```python
|
||||
# Harness internal
|
||||
|
||||
@@ -168,6 +168,21 @@ FastAPI application providing REST endpoints for frontend integration:
|
||||
|
||||
The separately deployed TaskCOP Vue app uses `POST /api/v1/auth/login/username` directly (its `username`, `userId`, or `yUserId` URL parameter identifies the user), so this compatibility flow does not depend on the retired Consumer login service. It calls the Gateway address configured as `b1ConsumerUrl` directly; it does not use a Vite `/api` reverse proxy. An empty TaskCOP store remains empty; task data is introduced through normal creation or the JSON import API. The Gateway CORS middleware always merges local Vite origins `http://localhost|127.0.0.1:{5173,5174,3000,8080}`, including when `GATEWAY_CORS_ORIGINS` is set for another frontend. This keeps a Vite server that falls back from 5173 to 5174 (or the reverse) working after a backend restart. For a deployment, set `GATEWAY_CORS_ORIGINS` to the exact additional frontend origins that are allowed to call the Gateway.
|
||||
|
||||
### Workflow Studio interface resources
|
||||
|
||||
The Workflow Studio data-source API also registers reusable HTTP interface resources. Create them through
|
||||
POST /api/workflows/data-sources with kind set to http; provide a non-secret baseUrl, an allowedMethods
|
||||
allowlist, and optional request headers. The service encrypts the headers immediately and never returns
|
||||
them through list, detail, resource-catalog, run-event, or canvas APIs. The canvas resource catalog
|
||||
exposes only the name, description, HTTP address, and allowed methods. HTTP execution continues to
|
||||
enforce the workflow target security policy, including the default SSRF restrictions.
|
||||
|
||||
The same catalog's agent and skill entries are display projections, not a second ownership store:
|
||||
`/api/workflows/resources/agents` returns `agentId`, cached available `skills`, and `createdBy`; the
|
||||
skills endpoint returns `skillId`, direct-call capability, and `createdBy`. Creator names are resolved
|
||||
from the immutable owner id at read time (falling back to the id if an account was removed), while
|
||||
built-in resources explicitly return `系统内置`.
|
||||
|
||||
### Deep Research modes
|
||||
|
||||
Deep Research is a separate Gateway feature rather than part of AI Writing. Its
|
||||
|
||||
@@ -27,10 +27,10 @@ from app.gateway.csrf_middleware import CSRFMiddleware
|
||||
from app.gateway.deps import langgraph_runtime
|
||||
from app.gateway.public_knowledge_cors import PublicKnowledgeCORSMiddleware
|
||||
from app.gateway.routers import (
|
||||
ai_writing,
|
||||
admin_active_runs,
|
||||
admin_users,
|
||||
agents,
|
||||
ai_writing,
|
||||
artifacts,
|
||||
assistants_compat,
|
||||
auth,
|
||||
@@ -39,11 +39,12 @@ from app.gateway.routers import (
|
||||
channels,
|
||||
checkpoint_migration,
|
||||
compact_menu_overrides,
|
||||
dashboard_sessions,
|
||||
deep_research,
|
||||
external_llmwiki,
|
||||
feedback,
|
||||
fixed_questions,
|
||||
dashboard_sessions,
|
||||
deep_research,
|
||||
flow_extract,
|
||||
html_page_favorites,
|
||||
intent,
|
||||
knowledge,
|
||||
@@ -63,19 +64,19 @@ from app.gateway.routers import (
|
||||
open_chat,
|
||||
parallel_agent_tasks,
|
||||
parallel_agents,
|
||||
position_roles,
|
||||
position_roundtable,
|
||||
positions,
|
||||
position_roles,
|
||||
public_config,
|
||||
public_embed,
|
||||
public_knowledge_vector_search,
|
||||
public_leaderboard,
|
||||
public_scheduled_tasks,
|
||||
recommend,
|
||||
flow_extract,
|
||||
recommended_questions,
|
||||
roundtable_chains,
|
||||
report_structures,
|
||||
roundtable_artifact_submissions,
|
||||
roundtable_chains,
|
||||
roundtable_diagnostics,
|
||||
roundtable_draft_shares,
|
||||
roundtable_drafts,
|
||||
@@ -91,16 +92,22 @@ from app.gateway.routers import (
|
||||
system_settings,
|
||||
tags,
|
||||
task_buttons,
|
||||
taskcop_tasks,
|
||||
report_structures,
|
||||
task_reports,
|
||||
taskcop_tasks,
|
||||
thread_runs,
|
||||
tool_metrics,
|
||||
thread_shares,
|
||||
threads,
|
||||
tool_metrics,
|
||||
uploads,
|
||||
user_preferences,
|
||||
user_prompts,
|
||||
workflow_data_sources,
|
||||
workflow_embed,
|
||||
workflow_resources,
|
||||
workflow_runs,
|
||||
workflow_studio,
|
||||
workflows,
|
||||
workflows_coze_compat,
|
||||
writing,
|
||||
)
|
||||
from deerflow.config import app_config as deerflow_app_config
|
||||
@@ -342,10 +349,10 @@ async def _run_db_migrations() -> None:
|
||||
deadline = asyncio.get_running_loop().time() + timeout_s
|
||||
while result_queue.empty():
|
||||
if asyncio.get_running_loop().time() >= deadline:
|
||||
raise asyncio.TimeoutError
|
||||
raise TimeoutError
|
||||
await asyncio.sleep(0.2)
|
||||
result = result_queue.get_nowait()
|
||||
except asyncio.TimeoutError:
|
||||
except TimeoutError:
|
||||
logger.error(
|
||||
"Database migration did not finish within %.0fs; continuing startup on create_all schema. "
|
||||
"Likely cause: MySQL metadata-lock wait or a slow DDL. Set DEER_FLOW_SKIP_ALEMBIC=1 "
|
||||
@@ -1591,6 +1598,18 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Light-app registry (?????) at /api/light-apps
|
||||
app.include_router(light_apps.router)
|
||||
|
||||
# Workflow Studio: definitions / runs / SSE / data sources / embed / Coze compat.
|
||||
# Static prefixes (node-types, resources, runs, data-sources, embed) must be
|
||||
# registered before the /{workflow_id} catch-alls in workflows.router.
|
||||
app.include_router(workflow_resources.router)
|
||||
app.include_router(workflow_data_sources.router)
|
||||
app.include_router(workflow_embed.router)
|
||||
app.include_router(workflow_runs.router)
|
||||
app.include_router(workflow_studio.router)
|
||||
app.include_router(workflows.router)
|
||||
app.include_router(workflows_coze_compat.router)
|
||||
app.include_router(workflows_coze_compat.chrome_router)
|
||||
|
||||
# Sidebar-menu overrides (菜单管理) at /api/menu-overrides
|
||||
app.include_router(menu_overrides.router)
|
||||
|
||||
|
||||
@@ -66,11 +66,30 @@ def should_check_csrf(request: Request) -> bool:
|
||||
# 双提交 Cookie 既无 token 可比、也无会话可被伪造,豁免之。
|
||||
if path.startswith("/api/open/"):
|
||||
return False
|
||||
if is_coze_studio_compat_path(path):
|
||||
return False
|
||||
if is_weknora_embed_proxy_request(request):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
# Coze playground 自己的 axios 不会带 X-CSRF-Token;独立编排页经
|
||||
# localhost:8080 同源代理访问这些接口。Cookie 会话仍由 AuthMiddleware 校验。
|
||||
_COZE_STUDIO_PREFIXES: tuple[str, ...] = (
|
||||
"/api/passport/",
|
||||
"/api/playground_api/",
|
||||
"/api/workflow_api/",
|
||||
"/api/space/",
|
||||
)
|
||||
|
||||
|
||||
def is_coze_studio_compat_path(path: str) -> bool:
|
||||
stripped = path.rstrip("/")
|
||||
if stripped == "/api/bot/get_type_list":
|
||||
return True
|
||||
return any(path.startswith(prefix) for prefix in _COZE_STUDIO_PREFIXES)
|
||||
|
||||
|
||||
def is_weknora_embed_proxy_request(request: Request) -> bool:
|
||||
path = app_relative_path(request)
|
||||
if not is_weknora_embed_proxy_path(path):
|
||||
|
||||
@@ -285,6 +285,18 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
|
||||
app.state.light_app_store = make_light_app_store(sf)
|
||||
|
||||
from deerflow.persistence.workflows import make_workflow_store
|
||||
|
||||
app.state.workflow_store = make_workflow_store(sf)
|
||||
|
||||
from deerflow.persistence.workflow_data_sources import make_workflow_data_source_store
|
||||
from deerflow.persistence.workflow_events import make_workflow_event_store
|
||||
from deerflow.persistence.workflow_runs import make_workflow_run_store
|
||||
|
||||
app.state.workflow_run_store = make_workflow_run_store(sf)
|
||||
app.state.workflow_event_store = make_workflow_event_store(sf, run_store=app.state.workflow_run_store)
|
||||
app.state.workflow_data_source_store = make_workflow_data_source_store(sf)
|
||||
|
||||
from deerflow.persistence.menu_overrides import make_menu_override_store
|
||||
|
||||
app.state.menu_override_store = make_menu_override_store(sf)
|
||||
@@ -555,6 +567,47 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app.state.deep_research_executor = None
|
||||
app.state.deep_research_dispatcher = None
|
||||
|
||||
# ── Workflow Studio runtime: live hub + executor + lease dispatcher ──
|
||||
# Every worker runs a dispatcher; the atomic claim decides ownership and
|
||||
# an expired lease is how a crashed worker's run gets picked back up.
|
||||
workflow_cfg = getattr(config, "workflows", None)
|
||||
if workflow_cfg is not None and workflow_cfg.enabled:
|
||||
from app.gateway.workflow_dispatcher import WorkflowRunDispatcher
|
||||
from app.gateway.workflow_executor import WorkflowRunExecutor
|
||||
from app.gateway.workflow_live_hub import WorkflowLiveHub
|
||||
|
||||
app.state.workflow_live_hub = WorkflowLiveHub()
|
||||
app.state.workflow_executor = WorkflowRunExecutor(
|
||||
app,
|
||||
run_store=app.state.workflow_run_store,
|
||||
event_store=app.state.workflow_event_store,
|
||||
workflow_store=app.state.workflow_store,
|
||||
data_source_store=app.state.workflow_data_source_store,
|
||||
live_publisher=app.state.workflow_live_hub.publish,
|
||||
config=workflow_cfg,
|
||||
)
|
||||
app.state.workflow_dispatcher = WorkflowRunDispatcher(
|
||||
app.state.workflow_run_store,
|
||||
app.state.workflow_executor,
|
||||
event_store=app.state.workflow_event_store,
|
||||
live_publisher=app.state.workflow_live_hub.publish,
|
||||
config=workflow_cfg,
|
||||
)
|
||||
app.state.workflow_dispatcher.start()
|
||||
from app.gateway.workflow_retention import WorkflowRetentionCleaner
|
||||
|
||||
app.state.workflow_retention = WorkflowRetentionCleaner(
|
||||
app.state.workflow_run_store,
|
||||
app.state.workflow_event_store,
|
||||
config=workflow_cfg,
|
||||
)
|
||||
app.state.workflow_retention.start()
|
||||
else:
|
||||
app.state.workflow_live_hub = None
|
||||
app.state.workflow_executor = None
|
||||
app.state.workflow_dispatcher = None
|
||||
app.state.workflow_retention = None
|
||||
|
||||
# Run event store (has its own factory with config-driven backend selection)
|
||||
run_events_config = getattr(config, "run_events", None)
|
||||
app.state.run_event_store = make_run_event_store(run_events_config)
|
||||
@@ -583,6 +636,15 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
dr_dispatcher = getattr(app.state, "deep_research_dispatcher", None)
|
||||
if dr_dispatcher is not None:
|
||||
await dr_dispatcher.stop()
|
||||
wf_dispatcher = getattr(app.state, "workflow_dispatcher", None)
|
||||
if wf_dispatcher is not None:
|
||||
await wf_dispatcher.stop()
|
||||
wf_retention = getattr(app.state, "workflow_retention", None)
|
||||
if wf_retention is not None:
|
||||
await wf_retention.stop()
|
||||
from deerflow.workflows.runtime.sql_runner import dispose_engines
|
||||
|
||||
await dispose_engines()
|
||||
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
|
||||
if queue is not None:
|
||||
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
|
||||
|
||||
@@ -0,0 +1,287 @@
|
||||
"""Workflow data sources and HTTP credentials.
|
||||
|
||||
Write-only secrets: the DSN (or the credential header blob) is accepted on
|
||||
create/update, encrypted immediately, and never returned. Reads expose only a
|
||||
masked target so an operator can tell two sources apart.
|
||||
|
||||
Shared sources (``ownerId`` null) are admin-only; a regular user may manage
|
||||
their own.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_optional_user_from_request
|
||||
from app.gateway.workflow_audit import audit_workflow
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.workflows.security.secret_box import SecretKeyMissingError, secrets_available
|
||||
from deerflow.workflows.security.sql_policy import assert_read_only, assert_tables_allowed
|
||||
|
||||
router = APIRouter(prefix="/api/workflows/data-sources", tags=["workflow-data-sources"])
|
||||
|
||||
_HTTP_METHODS = frozenset({"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"})
|
||||
|
||||
|
||||
class DataSourceCreate(BaseModel):
|
||||
name: str = Field(min_length=1, max_length=191)
|
||||
description: str = ""
|
||||
kind: str = Field(default="sql", pattern="^(sql|http)$")
|
||||
dsn: str | None = None
|
||||
headers: dict[str, str] | None = None
|
||||
base_url: str = Field(default="", max_length=2048, alias="baseUrl")
|
||||
allowed_methods: list[str] = Field(default_factory=list, alias="allowedMethods")
|
||||
max_rows: int = Field(default=1000, ge=1, le=100_000, alias="maxRows")
|
||||
allowed_tables: list[str] = Field(default_factory=list, alias="allowedTables")
|
||||
enabled: bool = True
|
||||
shared: bool = False
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class QueryBody(BaseModel):
|
||||
statement: str = Field(min_length=1)
|
||||
|
||||
|
||||
class DataSourceUpdate(BaseModel):
|
||||
name: str | None = Field(default=None, min_length=1, max_length=191)
|
||||
description: str | None = None
|
||||
dsn: str | None = None
|
||||
headers: dict[str, str] | None = None
|
||||
base_url: str | None = Field(default=None, max_length=2048, alias="baseUrl")
|
||||
allowed_methods: list[str] | None = Field(default=None, alias="allowedMethods")
|
||||
max_rows: int | None = Field(default=None, ge=1, le=100_000, alias="maxRows")
|
||||
allowed_tables: list[str] | None = Field(default=None, alias="allowedTables")
|
||||
enabled: bool | None = None
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
if not get_app_config().workflows.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
def _store(request: Request):
|
||||
store = getattr(request.app.state, "workflow_data_source_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow data source store not available")
|
||||
return store
|
||||
|
||||
|
||||
async def _is_admin(request: Request) -> bool:
|
||||
user = await get_optional_user_from_request(request)
|
||||
return user is not None and getattr(user, "system_role", None) == "admin"
|
||||
|
||||
|
||||
def _normalise_http_metadata(
|
||||
base_url: str,
|
||||
allowed_methods: list[str],
|
||||
) -> tuple[str, list[str]]:
|
||||
target = base_url.strip()
|
||||
try:
|
||||
parsed = urlsplit(target)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail="HTTP 接口地址格式无效") from exc
|
||||
if (
|
||||
parsed.scheme not in {"http", "https"}
|
||||
or not parsed.netloc
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="HTTP 接口地址必须是无凭据、无查询参数的 http(s) 地址",
|
||||
)
|
||||
|
||||
normalised_methods: list[str] = []
|
||||
for raw_method in allowed_methods:
|
||||
method = str(raw_method).strip().upper()
|
||||
if method not in _HTTP_METHODS:
|
||||
raise HTTPException(status_code=400, detail=f"不支持的 HTTP 方法:{method}")
|
||||
if method not in normalised_methods:
|
||||
normalised_methods.append(method)
|
||||
if not normalised_methods:
|
||||
raise HTTPException(status_code=400, detail="至少选择一种 HTTP 方法")
|
||||
return target, normalised_methods
|
||||
|
||||
|
||||
def _normalise_headers(headers: dict[str, str] | None) -> dict[str, str]:
|
||||
normalised: dict[str, str] = {}
|
||||
for raw_name, raw_value in (headers or {}).items():
|
||||
name = str(raw_name).strip()
|
||||
value = str(raw_value)
|
||||
if not name or any(char in name + value for char in "\r\n"):
|
||||
raise HTTPException(status_code=400, detail="HTTP 请求头格式无效")
|
||||
normalised[name] = value
|
||||
return normalised
|
||||
|
||||
|
||||
def _secret_from(kind: str, dsn: str | None, headers: dict[str, str] | None) -> str:
|
||||
if kind == "http":
|
||||
return json.dumps({"headers": _normalise_headers(headers)}, ensure_ascii=False)
|
||||
if not dsn:
|
||||
raise HTTPException(status_code=400, detail="SQL 数据源需要提供 dsn")
|
||||
if "://" not in dsn:
|
||||
raise HTTPException(status_code=400, detail="dsn 格式无效")
|
||||
return dsn
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_data_sources(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
rows = await _store(request).list_sources(owner_id=user_id)
|
||||
return {"dataSources": rows, "secretsEncryptionAvailable": secrets_available()}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def create_data_source(request: Request, body: DataSourceCreate) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
is_admin = await _is_admin(request)
|
||||
if body.shared and not is_admin:
|
||||
raise HTTPException(status_code=403, detail="仅管理员可创建共享数据源")
|
||||
secret = _secret_from(body.kind, body.dsn, body.headers)
|
||||
base_url = ""
|
||||
allowed_methods: list[str] = []
|
||||
if body.kind == "http":
|
||||
base_url, allowed_methods = _normalise_http_metadata(
|
||||
body.base_url,
|
||||
body.allowed_methods,
|
||||
)
|
||||
try:
|
||||
created = await _store(request).create_source(
|
||||
{
|
||||
"name": body.name,
|
||||
"description": body.description,
|
||||
"kind": body.kind,
|
||||
"dsn": secret,
|
||||
"base_url": base_url,
|
||||
"allowed_methods": allowed_methods,
|
||||
"max_rows": body.max_rows,
|
||||
"allowed_tables": body.allowed_tables,
|
||||
"enabled": body.enabled,
|
||||
"owner_id": None if body.shared else user_id,
|
||||
"created_by": user_id,
|
||||
}
|
||||
)
|
||||
except SecretKeyMissingError as exc:
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail={
|
||||
"code": "WORKFLOW_RESOURCE_MISSING",
|
||||
"message": "服务端未配置 WORKFLOW_SECRET_KEY,无法安全保存连接信息",
|
||||
},
|
||||
) from exc
|
||||
audit_workflow("data_source.create", user_id=user_id, dataSourceId=created["id"], kind=body.kind, shared=body.shared)
|
||||
return created
|
||||
|
||||
|
||||
async def _owned(request: Request, source_id: str) -> dict[str, Any]:
|
||||
row = await _store(request).get_source(source_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="数据源不存在")
|
||||
user_id = await get_current_user(request)
|
||||
if row.get("owner_id") == user_id:
|
||||
return row
|
||||
if await _is_admin(request):
|
||||
return row
|
||||
raise HTTPException(status_code=403, detail="无权操作该数据源")
|
||||
|
||||
|
||||
async def _visible(request: Request, source_id: str) -> dict[str, Any]:
|
||||
row = await _store(request).get_source(source_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="数据源不存在")
|
||||
user_id = await get_current_user(request)
|
||||
if row.get("owner_id") in (user_id, None):
|
||||
return row
|
||||
if await _is_admin(request):
|
||||
return row
|
||||
raise HTTPException(status_code=403, detail="无权查看该数据源")
|
||||
|
||||
|
||||
@router.get("/{source_id}")
|
||||
async def get_data_source(request: Request, source_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
return await _visible(request, source_id)
|
||||
|
||||
|
||||
@router.post("/{source_id}/introspect")
|
||||
async def introspect_data_source(request: Request, source_id: str) -> dict[str, Any]:
|
||||
"""Return the configured table allowlist only — never sample rows."""
|
||||
_require_enabled()
|
||||
row = await _visible(request, source_id)
|
||||
tables = list(row.get("allowed_tables") or [])
|
||||
return {
|
||||
"dataSourceId": row["id"],
|
||||
"kind": row.get("kind"),
|
||||
"tables": [{"name": name} for name in tables],
|
||||
"schemas": [],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/validate-query")
|
||||
async def validate_data_source_query(request: Request, source_id: str, body: QueryBody) -> dict[str, Any]:
|
||||
"""Validate SQL against the read-only policy without executing it."""
|
||||
_require_enabled()
|
||||
row = await _visible(request, source_id)
|
||||
try:
|
||||
statement = assert_read_only(body.statement)
|
||||
assert_tables_allowed(statement, list(row.get("allowed_tables") or []))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
if isinstance(exc, WorkflowError):
|
||||
raise HTTPException(status_code=400, detail=exc.to_body().model_dump(by_alias=True)) from exc
|
||||
raise
|
||||
audit_workflow("data_source.validate_query", user_id=await get_current_user(request), dataSourceId=source_id)
|
||||
return {"ok": True, "statement": statement}
|
||||
|
||||
|
||||
@router.put("/{source_id}")
|
||||
async def update_data_source(request: Request, source_id: str, body: DataSourceUpdate) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
row = await _owned(request, source_id)
|
||||
payload = body.model_dump(exclude_unset=True, by_alias=False)
|
||||
kind = str(row.get("kind") or "sql")
|
||||
if body.dsn is not None or body.headers is not None:
|
||||
payload["dsn"] = _secret_from(kind, body.dsn, body.headers)
|
||||
payload.pop("headers", None)
|
||||
if kind == "http":
|
||||
base_url, allowed_methods = _normalise_http_metadata(
|
||||
body.base_url if body.base_url is not None else str(row.get("base_url") or ""),
|
||||
body.allowed_methods
|
||||
if body.allowed_methods is not None
|
||||
else list(row.get("allowed_methods") or []),
|
||||
)
|
||||
payload["base_url"] = base_url
|
||||
payload["allowed_methods"] = allowed_methods
|
||||
else:
|
||||
payload.pop("base_url", None)
|
||||
payload.pop("allowed_methods", None)
|
||||
try:
|
||||
updated = await _store(request).update_source(source_id, payload)
|
||||
except SecretKeyMissingError as exc:
|
||||
raise HTTPException(status_code=503, detail="服务端未配置 WORKFLOW_SECRET_KEY") from exc
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail="数据源不存在")
|
||||
audit_workflow("data_source.update", user_id=await get_current_user(request), dataSourceId=source_id)
|
||||
return updated
|
||||
|
||||
|
||||
@router.delete("/{source_id}")
|
||||
async def delete_data_source(request: Request, source_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _owned(request, source_id)
|
||||
deleted = await _store(request).delete_source(source_id)
|
||||
audit_workflow("data_source.delete", user_id=await get_current_user(request), dataSourceId=source_id)
|
||||
return {"deleted": deleted}
|
||||
@@ -0,0 +1,186 @@
|
||||
"""iframe embedding support: origin allowlist + short-lived signed tickets.
|
||||
|
||||
Scope, stated plainly: a ticket proves *"this workflow may be embedded by this
|
||||
origin, for this user, right now"*. It is **not** an API credential — embedded
|
||||
pages still authenticate their API calls with the normal session, exactly like
|
||||
the rest of the app's iframe surfaces. What the ticket adds is an explicit
|
||||
origin decision that can be audited, plus the ``frame-ancestors`` value the host
|
||||
should send.
|
||||
|
||||
Tickets are stateless HS256 JWTs (so any worker can verify them) with a 60s
|
||||
default TTL. Single-use is enforced best-effort per process via a replay cache;
|
||||
the short TTL is the real protection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import jwt
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user
|
||||
from deerflow.config.app_config import get_app_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/workflows/embed", tags=["workflow-embed"])
|
||||
|
||||
_ALGORITHM = "HS256"
|
||||
_AUDIENCE = "workflow-embed"
|
||||
_replay_cache: dict[str, float] = {}
|
||||
_REPLAY_CACHE_MAX = 4096
|
||||
|
||||
|
||||
class TicketRequest(BaseModel):
|
||||
workflow_id: str = Field(alias="workflowId")
|
||||
origin: str
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class VerifyRequest(BaseModel):
|
||||
ticket: str
|
||||
origin: str | None = None
|
||||
|
||||
|
||||
def _signing_key() -> str:
|
||||
for name in ("WORKFLOW_SECRET_KEY", "DEERFLOW_SECRET_KEY", "JWT_SECRET_KEY"):
|
||||
value = os.environ.get(name)
|
||||
if value and value.strip():
|
||||
return hashlib.sha256(value.strip().encode("utf-8")).hexdigest()
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail={
|
||||
"code": "WORKFLOW_EMBED_TICKET_INVALID",
|
||||
"message": "服务端未配置签名密钥,无法签发嵌入票据",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _normalize_origin(raw: str) -> str:
|
||||
parsed = urlparse(raw.strip())
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_EMBED_ORIGIN_DENIED", "message": "origin 格式无效"},
|
||||
)
|
||||
return f"{parsed.scheme.lower()}://{parsed.netloc.lower()}"
|
||||
|
||||
|
||||
def _assert_origin_allowed(origin: str) -> str:
|
||||
cfg = get_app_config().workflows.embed
|
||||
normalized = _normalize_origin(origin)
|
||||
allowed = {_normalize_origin(o) for o in cfg.allowed_origins if o.strip()}
|
||||
if not allowed:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={
|
||||
"code": "WORKFLOW_EMBED_ORIGIN_DENIED",
|
||||
"message": "未配置任何允许的嵌入来源(workflows.embed.allowed_origins)",
|
||||
},
|
||||
)
|
||||
if normalized not in allowed:
|
||||
logger.warning("workflow embed origin denied: %s", normalized)
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={"code": "WORKFLOW_EMBED_ORIGIN_DENIED", "message": "该来源不允许嵌入"},
|
||||
)
|
||||
return normalized
|
||||
|
||||
|
||||
def _prune_replay_cache(now: float) -> None:
|
||||
for key, expiry in list(_replay_cache.items()):
|
||||
if expiry <= now:
|
||||
_replay_cache.pop(key, None)
|
||||
if len(_replay_cache) > _REPLAY_CACHE_MAX:
|
||||
_replay_cache.clear()
|
||||
|
||||
|
||||
@router.post("/tickets")
|
||||
async def issue_ticket(request: Request, body: TicketRequest) -> dict[str, Any]:
|
||||
cfg = get_app_config().workflows
|
||||
if not cfg.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
user_id = await get_current_user(request)
|
||||
origin = _assert_origin_allowed(body.origin)
|
||||
|
||||
definitions = getattr(request.app.state, "workflow_store", None)
|
||||
if definitions is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
definition = await definitions.get_definition(body.workflow_id, include_draft=False)
|
||||
if definition is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
if definition.get("owner_id") != user_id:
|
||||
raise HTTPException(status_code=403, detail={"code": "WORKFLOW_FORBIDDEN", "message": "无权嵌入该工作流"})
|
||||
|
||||
now = int(time.time())
|
||||
ttl = cfg.embed.ticket_ttl_seconds
|
||||
payload = {
|
||||
"aud": _AUDIENCE,
|
||||
"sub": user_id,
|
||||
"wf": body.workflow_id,
|
||||
"org": origin,
|
||||
"iat": now,
|
||||
"exp": now + ttl,
|
||||
"jti": hashlib.sha256(f"{user_id}{body.workflow_id}{origin}{now}".encode()).hexdigest()[:32],
|
||||
}
|
||||
ticket = jwt.encode(payload, _signing_key(), algorithm=_ALGORITHM)
|
||||
logger.info(
|
||||
"workflow embed ticket issued workflow=%s user=%s origin=%s ttl=%ss",
|
||||
body.workflow_id,
|
||||
user_id,
|
||||
origin,
|
||||
ttl,
|
||||
)
|
||||
return {
|
||||
"ticket": ticket,
|
||||
"expiresIn": ttl,
|
||||
"origin": origin,
|
||||
"frameAncestors": sorted({_normalize_origin(o) for o in cfg.embed.allowed_origins if o.strip()}),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/verify")
|
||||
async def verify_ticket(body: VerifyRequest) -> dict[str, Any]:
|
||||
cfg = get_app_config().workflows
|
||||
if not cfg.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
try:
|
||||
payload = jwt.decode(body.ticket, _signing_key(), algorithms=[_ALGORITHM], audience=_AUDIENCE)
|
||||
except jwt.PyJWTError as exc:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail={"code": "WORKFLOW_EMBED_TICKET_INVALID", "message": "票据无效或已过期"},
|
||||
) from exc
|
||||
|
||||
if body.origin and _normalize_origin(body.origin) != payload.get("org"):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={"code": "WORKFLOW_EMBED_ORIGIN_DENIED", "message": "票据与当前来源不匹配"},
|
||||
)
|
||||
|
||||
now = time.time()
|
||||
_prune_replay_cache(now)
|
||||
jti = str(payload.get("jti") or "")
|
||||
if jti and jti in _replay_cache:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail={"code": "WORKFLOW_EMBED_TICKET_INVALID", "message": "票据已被使用"},
|
||||
)
|
||||
if jti:
|
||||
_replay_cache[jti] = float(payload.get("exp") or now + 60)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"workflowId": payload.get("wf"),
|
||||
"userId": payload.get("sub"),
|
||||
"origin": payload.get("org"),
|
||||
"expiresAt": payload.get("exp"),
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
"""Workflow node-type and resource catalog APIs (phase 1)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
|
||||
from app.gateway.deps import get_current_user, get_local_provider
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.persistence.workflows import WorkflowStore
|
||||
from deerflow.workflows.schemas import ALL_NODE_TYPES
|
||||
|
||||
router = APIRouter(prefix="/api/workflows", tags=["workflow-resources"])
|
||||
|
||||
_NODE_META: dict[str, dict[str, str]] = {
|
||||
"start": {"label": "开始", "category": "control"},
|
||||
"output": {"label": "输出", "category": "control"},
|
||||
"agent": {"label": "智能体", "category": "ai"},
|
||||
"skill": {"label": "技能", "category": "ai"},
|
||||
"http": {"label": "HTTP", "category": "data"},
|
||||
"sql_read": {"label": "SQL 只读", "category": "data"},
|
||||
"code": {"label": "代码", "category": "data"},
|
||||
"transform": {"label": "模板/转换", "category": "data"},
|
||||
"condition": {"label": "条件", "category": "control"},
|
||||
"merge": {"label": "合并", "category": "control"},
|
||||
"human_input": {"label": "人工介入", "category": "control"},
|
||||
"subworkflow": {"label": "子工作流", "category": "control"},
|
||||
"loop": {"label": "循环", "category": "control"},
|
||||
}
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
if not get_app_config().workflows.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
def _get_store(request: Request) -> WorkflowStore:
|
||||
store = getattr(request.app.state, "workflow_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
return store
|
||||
|
||||
|
||||
async def _resolve_creator_names(user_ids: set[str]) -> dict[str, str]:
|
||||
"""Resolve resource-owner ids to short display names, best-effort.
|
||||
|
||||
The workflow catalog is a small display projection over the agent and
|
||||
skill stores. Those stores intentionally retain immutable owner ids for
|
||||
access control; this resolver exposes a friendly creator name without
|
||||
changing that ownership model. If an account has since disappeared, the
|
||||
stable id remains a useful audit fallback.
|
||||
"""
|
||||
if not user_ids:
|
||||
return {}
|
||||
try:
|
||||
provider = get_local_provider()
|
||||
except Exception: # noqa: BLE001
|
||||
return {user_id: user_id for user_id in user_ids}
|
||||
|
||||
names: dict[str, str] = {}
|
||||
for user_id in user_ids:
|
||||
try:
|
||||
user = await provider.get_user(user_id)
|
||||
except Exception: # noqa: BLE001
|
||||
user = None
|
||||
email = getattr(user, "email", None) if user is not None else None
|
||||
if isinstance(email, str) and email:
|
||||
names[user_id] = email.split("@", 1)[0] or user_id
|
||||
else:
|
||||
names[user_id] = user_id
|
||||
return names
|
||||
|
||||
|
||||
def _creator_label(owner_id: object, creator_names: dict[str, str]) -> str:
|
||||
"""Return the catalog label for a user-owned or built-in resource."""
|
||||
if owner_id is None or not str(owner_id).strip():
|
||||
return "系统内置"
|
||||
owner_key = str(owner_id)
|
||||
return creator_names.get(owner_key, owner_key)
|
||||
|
||||
|
||||
@router.get("/node-types")
|
||||
async def list_node_types(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await get_current_user(request)
|
||||
items = []
|
||||
for node_type in ALL_NODE_TYPES:
|
||||
meta = _NODE_META.get(node_type, {"label": node_type, "category": "other"})
|
||||
items.append({"type": node_type, **meta})
|
||||
return {"nodeTypes": items}
|
||||
|
||||
|
||||
@router.get("/resources/agents")
|
||||
async def list_agent_resources(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
agent_store = getattr(request.app.state, "agent_store", None)
|
||||
agents: list[dict[str, Any]] = []
|
||||
if agent_store is not None:
|
||||
try:
|
||||
if hasattr(agent_store, "list_visible"):
|
||||
rows = await agent_store.list_visible(user_id)
|
||||
elif hasattr(agent_store, "list_all"):
|
||||
rows = await agent_store.list_all()
|
||||
else:
|
||||
rows = []
|
||||
creator_names = await _resolve_creator_names({str(row["user_id"]) for row in rows or [] if row.get("user_id") is not None})
|
||||
agent_ids = [str(row.get("id") or row.get("agent_id") or "") for row in rows or []]
|
||||
extras_by_agent: dict[str, dict[str, Any]] = {}
|
||||
if hasattr(agent_store, "get_extras_for"):
|
||||
extras_by_agent = await agent_store.get_extras_for([agent_id for agent_id in agent_ids if agent_id])
|
||||
keyword = request.query_params.get("keyword", "").strip().casefold()
|
||||
for row in rows or []:
|
||||
agent_id = str(row.get("id") or row.get("agent_id") or "")
|
||||
name = str(row.get("name") or agent_id)
|
||||
description = str(row.get("description") or "")
|
||||
if keyword and keyword not in f"{agent_id} {name} {description}".casefold():
|
||||
continue
|
||||
cached_skills = (extras_by_agent.get(agent_id) or {}).get("skills") or []
|
||||
agents.append(
|
||||
{
|
||||
"agentId": agent_id,
|
||||
"name": name,
|
||||
"description": description,
|
||||
"skills": [str(skill) for skill in cached_skills],
|
||||
"createdBy": _creator_label(row.get("user_id"), creator_names),
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
agents = []
|
||||
return {"agents": agents}
|
||||
|
||||
|
||||
@router.get("/resources/skills")
|
||||
async def list_skill_resources(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
skills: list[dict[str, Any]] = []
|
||||
try:
|
||||
from deerflow.skills import get_or_new_skill_storage
|
||||
|
||||
skill_store = getattr(request.app.state, "skill_store", None)
|
||||
ownership_by_name: dict[str, dict[str, Any]] = {}
|
||||
if skill_store is not None and hasattr(skill_store, "list_visible"):
|
||||
records = await skill_store.list_visible(user_id)
|
||||
ownership_by_name = {str(record["name"]): record for record in records if record.get("name")}
|
||||
creator_names = await _resolve_creator_names({str(record["owner_user_id"]) for record in ownership_by_name.values() if record.get("owner_user_id") is not None})
|
||||
storage = get_or_new_skill_storage()
|
||||
for skill in storage.load_skills(enabled_only=True) or []:
|
||||
name = getattr(skill, "name", None)
|
||||
if not name:
|
||||
continue
|
||||
record = ownership_by_name.get(str(name), {})
|
||||
skills.append(
|
||||
{
|
||||
"skillId": name,
|
||||
"name": name,
|
||||
"description": getattr(skill, "description", "") or "",
|
||||
# Workflow direct-skill execution remains opt-in. Older
|
||||
# Skill definitions do not advertise this capability and
|
||||
# are therefore correctly shown as agent-only resources.
|
||||
"callable": bool(getattr(skill, "callable", False)),
|
||||
"createdBy": _creator_label(record.get("owner_user_id"), creator_names),
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
skills = []
|
||||
return {"skills": skills}
|
||||
|
||||
|
||||
@router.get("/resources/data-sources")
|
||||
async def list_data_source_resources(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
store = getattr(request.app.state, "workflow_data_source_store", None)
|
||||
if store is None:
|
||||
return {"dataSources": []}
|
||||
rows = await store.list_sources(owner_id=user_id)
|
||||
return {
|
||||
"dataSources": [
|
||||
{
|
||||
"id": row["id"],
|
||||
"dataSourceId": row["id"],
|
||||
"name": row["name"],
|
||||
"description": row.get("description") or "",
|
||||
"kind": row.get("kind"),
|
||||
"engine": row.get("driver") or "",
|
||||
"baseUrl": row.get("base_url") or "",
|
||||
"maskedTarget": row.get("masked_target"),
|
||||
"enabled": row.get("enabled"),
|
||||
"readOnly": row.get("kind") == "sql",
|
||||
"allowedMethods": row.get("allowed_methods") or [],
|
||||
"allowedTables": row.get("allowed_tables") or [],
|
||||
}
|
||||
for row in rows
|
||||
if row.get("enabled", True)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.get("/resources/credentials")
|
||||
async def list_credential_resources(request: Request) -> dict[str, Any]:
|
||||
"""Read-only credential directory: ids and labels only.
|
||||
|
||||
HTTP credentials are ``http``-kind data sources; the encrypted header blob
|
||||
is resolved exclusively by the run executor. This endpoint must never
|
||||
return headers, tokens, DSNs or ciphertext — a 404-free catalog so the
|
||||
canvas credential picker does not fail the whole resource panel.
|
||||
"""
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
store = getattr(request.app.state, "workflow_data_source_store", None)
|
||||
if store is None:
|
||||
return {"credentials": []}
|
||||
rows = await store.list_sources(owner_id=user_id, include_shared=True)
|
||||
return {
|
||||
"credentials": [
|
||||
{
|
||||
"credentialId": row["id"],
|
||||
"name": row["name"],
|
||||
"type": "http_headers",
|
||||
"sourceKind": "http",
|
||||
}
|
||||
for row in rows
|
||||
if str(row.get("kind") or "") == "http" and row.get("enabled", True)
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@router.get("/resources/subworkflows")
|
||||
async def list_subworkflow_resources(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await get_current_user(request)
|
||||
store = _get_store(request)
|
||||
return {"subworkflows": await store.list_published_summaries()}
|
||||
@@ -0,0 +1,700 @@
|
||||
"""Workflow run APIs: start / inspect / cancel / resume / retry / events / SSE stream."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_optional_user_from_request
|
||||
from app.gateway.workflow_audit import audit_workflow
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
from deerflow.workflows.runtime.sink import envelope_from_row
|
||||
from deerflow.workflows.schema_validation import schema_issues
|
||||
from deerflow.workflows.schemas import WorkflowGraph
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/workflows", tags=["workflow-runs"])
|
||||
|
||||
_TERMINAL_STATUSES = ("completed", "failed", "cancelled")
|
||||
_TERMINAL_EVENTS = ("run.completed", "run.failed", "run.cancelled")
|
||||
_REPLAY_PAGE = 200
|
||||
_DB_POLL_SECONDS = 0.8
|
||||
_LIVE_WAIT_SECONDS = 0.25
|
||||
_HEARTBEAT_SECONDS = 15.0
|
||||
_STREAM_DEADLINE_SECONDS = 60 * 60
|
||||
|
||||
|
||||
class StartRunBody(BaseModel):
|
||||
version_id: str | None = Field(default=None, alias="versionId")
|
||||
inputs: dict[str, Any] = Field(default_factory=dict)
|
||||
# 独立 Coze 工作流页早期使用 input/target;保留兼容入口,内部仍只使用 inputs。
|
||||
input: dict[str, Any] | None = None
|
||||
target: str | None = None
|
||||
idempotency_key: str | None = Field(default=None, alias="idempotencyKey", max_length=128)
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
def resolved_inputs(self) -> dict[str, Any]:
|
||||
return self.inputs if self.inputs or self.input is None else self.input
|
||||
|
||||
|
||||
class ResumeRunBody(BaseModel):
|
||||
resume_token: str = Field(alias="resumeToken")
|
||||
node_id: str | None = Field(default=None, alias="nodeId")
|
||||
action: str = "submit"
|
||||
values: dict[str, Any] = Field(default_factory=dict)
|
||||
# 兼容独立页的 ResumeRunRequest:payload/message。
|
||||
payload: dict[str, Any] | None = None
|
||||
message: str | None = None
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
def resolved_values(self) -> dict[str, Any]:
|
||||
return self.values if self.values or self.payload is None else self.payload
|
||||
|
||||
|
||||
# ── shared plumbing ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _cfg():
|
||||
return get_app_config().workflows
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
if not _cfg().enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
def _runs(request: Request) -> WorkflowRunStore:
|
||||
store = getattr(request.app.state, "workflow_run_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow run store not available")
|
||||
return store
|
||||
|
||||
|
||||
def _events(request: Request) -> WorkflowEventStore:
|
||||
store = getattr(request.app.state, "workflow_event_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow event store not available")
|
||||
return store
|
||||
|
||||
|
||||
def _definitions(request: Request):
|
||||
store = getattr(request.app.state, "workflow_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
return store
|
||||
|
||||
|
||||
def _nudge(request: Request) -> None:
|
||||
dispatcher = getattr(request.app.state, "workflow_dispatcher", None)
|
||||
if dispatcher is not None:
|
||||
dispatcher.nudge()
|
||||
|
||||
|
||||
async def _is_admin(request: Request) -> bool:
|
||||
user = await get_optional_user_from_request(request)
|
||||
return user is not None and getattr(user, "system_role", None) == "admin"
|
||||
|
||||
|
||||
async def _authorized_run(request: Request, run_id: str) -> dict[str, Any]:
|
||||
run = await _runs(request).get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail={"code": "WORKFLOW_RUN_NOT_FOUND", "message": "运行不存在"})
|
||||
user_id = await get_current_user(request)
|
||||
if run.get("owner_id") != user_id and not await _is_admin(request):
|
||||
raise HTTPException(status_code=403, detail={"code": "WORKFLOW_FORBIDDEN", "message": "无权访问该运行"})
|
||||
return run
|
||||
|
||||
|
||||
def _public_run(run: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Run view for HTTP clients: no lease internals, no resume token."""
|
||||
return {
|
||||
"id": run["id"],
|
||||
"runId": run["id"],
|
||||
"workflowId": run["workflow_id"],
|
||||
"versionId": run["workflow_version_id"],
|
||||
"status": run["status"],
|
||||
"ownerId": run["owner_id"],
|
||||
"input": run.get("input") or {},
|
||||
"output": run.get("output"),
|
||||
"error": run.get("error"),
|
||||
"pendingInput": run.get("pending_input"),
|
||||
"attempt": run.get("attempt"),
|
||||
"retryOfRunId": run.get("retry_of_run_id"),
|
||||
"startedAt": run.get("started_at"),
|
||||
"finishedAt": run.get("finished_at"),
|
||||
"createdAt": run.get("created_at"),
|
||||
"updatedAt": run.get("updated_at"),
|
||||
# The run row's counter holds the last seq handed out by the event log
|
||||
# (the event store bumps it in the insert transaction); 0 = no events.
|
||||
"lastSeq": int(run.get("next_event_seq") or 0),
|
||||
}
|
||||
|
||||
|
||||
# ── start / list / get / cancel / resume ────────────────────────────────
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/runs")
|
||||
async def start_run(request: Request, workflow_id: str, body: StartRunBody) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
if str(body.target or "").strip().lower() == "draft":
|
||||
# Never silently fall back to the published version when the client
|
||||
# thinks it is exercising the draft — draft snapshots need the explicit
|
||||
# test-run API (not built yet).
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_RUN_UNSUPPORTED",
|
||||
"message": "暂不支持运行草稿,请先发布或指定 versionId",
|
||||
},
|
||||
)
|
||||
definitions = _definitions(request)
|
||||
definition = await definitions.get_definition(workflow_id, include_draft=False)
|
||||
if definition is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
if definition.get("owner_id") != user_id and not await _is_admin(request):
|
||||
raise HTTPException(status_code=403, detail={"code": "WORKFLOW_FORBIDDEN", "message": "无权运行该工作流"})
|
||||
|
||||
version_id = body.version_id
|
||||
if not version_id:
|
||||
versions = await definitions.list_versions(workflow_id)
|
||||
if not versions:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_VERSION_NOT_FOUND", "message": "工作流尚未发布任何版本"},
|
||||
)
|
||||
version_id = versions[0]["id"]
|
||||
version = await definitions.get_version(version_id)
|
||||
if version is None or version.get("workflow_id") != workflow_id:
|
||||
raise HTTPException(status_code=404, detail={"code": "WORKFLOW_VERSION_NOT_FOUND", "message": "版本不存在"})
|
||||
|
||||
graph = WorkflowGraph.model_validate(version.get("graph") or {})
|
||||
inputs = body.resolved_inputs()
|
||||
missing = [key for key in (graph.input_schema.get("required") or []) if inputs.get(str(key)) in (None, "")]
|
||||
if missing:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_INPUT_INVALID",
|
||||
"message": f"缺少必填输入:{'、'.join(str(m) for m in missing)}",
|
||||
"details": {"missing": missing},
|
||||
},
|
||||
)
|
||||
# Full JSON Schema pass so type/enum/nested mistakes fail here with a
|
||||
# structured 400 instead of surfacing mid-run as a node error.
|
||||
schema_errors = schema_issues(inputs, graph.input_schema or {})
|
||||
if schema_errors:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_INPUT_INVALID",
|
||||
"message": "输入不符合 schema:" + ";".join(e["message"] for e in schema_errors[:3]),
|
||||
"details": {"errors": schema_errors},
|
||||
},
|
||||
)
|
||||
|
||||
runs = _runs(request)
|
||||
active = [r for r in await runs.list_runs(owner_id=user_id, limit=200) if r["status"] in ("queued", "running")]
|
||||
if len(active) >= _cfg().max_concurrent_runs_per_user:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail={
|
||||
"code": "WORKFLOW_LIMIT_EXCEEDED",
|
||||
"message": f"同时运行的工作流已达上限({_cfg().max_concurrent_runs_per_user})",
|
||||
},
|
||||
)
|
||||
|
||||
# Scope the idempotency key per workflow so two workflows can share a client key.
|
||||
key = f"{workflow_id}:{body.idempotency_key}" if body.idempotency_key else None
|
||||
run, created = await runs.create_run(
|
||||
{
|
||||
"workflow_id": workflow_id,
|
||||
"workflow_version_id": version_id,
|
||||
"owner_id": user_id,
|
||||
"input": inputs,
|
||||
"idempotency_key": key,
|
||||
"context": {"depth": 0},
|
||||
}
|
||||
)
|
||||
if created:
|
||||
await _events(request).append(
|
||||
run_id=run["id"],
|
||||
workflow_id=workflow_id,
|
||||
version_id=version_id,
|
||||
event_type="run.created",
|
||||
payload={"inputs": inputs},
|
||||
)
|
||||
audit_workflow("run.start", user_id=user_id, workflowId=workflow_id, runId=run["id"], versionId=version_id)
|
||||
_nudge(request)
|
||||
return {**_public_run(run), "created": created}
|
||||
|
||||
|
||||
@router.get("/{workflow_id}/runs")
|
||||
async def list_workflow_runs(
|
||||
request: Request,
|
||||
workflow_id: str,
|
||||
status: str | None = None,
|
||||
limit: int = Query(default=50, ge=1, le=200),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
owner = None if await _is_admin(request) else user_id
|
||||
rows = await _runs(request).list_runs(workflow_id=workflow_id, owner_id=owner, status=status, limit=limit, offset=offset)
|
||||
return {"runs": [_public_run(r) for r in rows], "hasMore": len(rows) >= limit}
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}")
|
||||
async def get_run(request: Request, run_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
run = await _authorized_run(request, run_id)
|
||||
node_runs = await _runs(request).list_node_runs(run_id, current_only=True)
|
||||
return {
|
||||
**_public_run(run),
|
||||
"nodeRuns": [
|
||||
{
|
||||
"nodeId": r["node_id"],
|
||||
"nodeType": r["node_type"],
|
||||
"status": r["status"],
|
||||
"attempt": r["attempt"],
|
||||
"iteration": r["iteration"],
|
||||
"durationMs": r["duration_ms"],
|
||||
"error": r["error"],
|
||||
"startedAt": r["started_at"],
|
||||
"finishedAt": r["finished_at"],
|
||||
}
|
||||
for r in node_runs
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/artifacts")
|
||||
async def list_run_artifacts(request: Request, run_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _authorized_run(request, run_id)
|
||||
rows = await _runs(request).list_artifacts(run_id)
|
||||
return {
|
||||
"artifacts": [
|
||||
{
|
||||
"artifactId": r["id"],
|
||||
"nodeId": r["node_id"],
|
||||
"name": r["name"],
|
||||
"mimeType": r["mime_type"],
|
||||
"sizeBytes": r["size_bytes"],
|
||||
"preview": r["preview"],
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# ── artifact content / download ─────────────────────────────────────────
|
||||
|
||||
_PREVIEW_MAX_BYTES = 256 * 1024
|
||||
_TEXT_MIME_PREFIXES = ("text/", "application/json", "application/javascript", "application/x-yaml", "application/xml", "application/csv")
|
||||
|
||||
|
||||
def _is_textual(mime_type: str) -> bool:
|
||||
return (mime_type or "").lower().startswith(_TEXT_MIME_PREFIXES)
|
||||
|
||||
|
||||
def _read_prefix(path: str, limit: int) -> tuple[bytes, bool]:
|
||||
"""Read up to ``limit + 1`` bytes; the extra byte detects truncation."""
|
||||
with open(path, "rb") as handle:
|
||||
data = handle.read(limit + 1)
|
||||
return data[:limit], len(data) > limit
|
||||
|
||||
|
||||
def _file_exists(path: str) -> bool:
|
||||
import os
|
||||
|
||||
return bool(path) and os.path.isfile(path)
|
||||
|
||||
|
||||
async def _authorized_artifact(request: Request, run_id: str, artifact_id: str) -> dict[str, Any]:
|
||||
"""Run-level authorization first, then artifact-must-belong-to-run.
|
||||
|
||||
The file path is resolved exclusively from the registered artifact row —
|
||||
clients address artifacts by id and can never pass a sandbox path in.
|
||||
"""
|
||||
await _authorized_run(request, run_id)
|
||||
artifact = await _runs(request).get_artifact(run_id, artifact_id)
|
||||
if artifact is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"code": "WORKFLOW_ARTIFACT_NOT_FOUND", "message": "制品不存在"},
|
||||
)
|
||||
return artifact
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/artifacts/{artifact_id}")
|
||||
async def get_run_artifact_content(request: Request, run_id: str, artifact_id: str) -> dict[str, Any]:
|
||||
"""Size-bounded text preview for file cards; binaries stay download-only."""
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from deerflow.workflows.events import artifact_kind
|
||||
|
||||
_require_enabled()
|
||||
artifact = await _authorized_artifact(request, run_id, artifact_id)
|
||||
mime = str(artifact.get("mime_type") or "application/octet-stream")
|
||||
kind = artifact_kind(mime)
|
||||
base = {
|
||||
"artifactId": artifact["id"],
|
||||
"name": artifact["name"],
|
||||
"kind": kind,
|
||||
"mimeType": mime,
|
||||
"sizeBytes": artifact.get("size_bytes"),
|
||||
}
|
||||
|
||||
path = str(artifact.get("path") or "")
|
||||
file_ok = bool(path) and await run_in_threadpool(_file_exists, path)
|
||||
if file_ok:
|
||||
raw, truncated = (b"", False)
|
||||
if _is_textual(mime):
|
||||
raw, truncated = await run_in_threadpool(_read_prefix, path, _PREVIEW_MAX_BYTES)
|
||||
return {
|
||||
**base,
|
||||
"text": raw.decode("utf-8", errors="replace"),
|
||||
"truncated": truncated,
|
||||
"source": "file",
|
||||
}
|
||||
return {
|
||||
**base,
|
||||
"binary": True,
|
||||
"downloadUrl": f"/api/workflows/runs/{run_id}/artifacts/{artifact_id}/download",
|
||||
"source": "file",
|
||||
}
|
||||
|
||||
preview = artifact.get("preview")
|
||||
if preview:
|
||||
return {**base, "text": str(preview), "truncated": True, "source": "preview"}
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"code": "WORKFLOW_ARTIFACT_CONTENT_UNAVAILABLE", "message": "制品内容不可用(文件已过期)"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/artifacts/{artifact_id}/download")
|
||||
async def download_run_artifact(request: Request, run_id: str, artifact_id: str):
|
||||
from urllib.parse import quote
|
||||
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.responses import FileResponse, Response
|
||||
|
||||
_require_enabled()
|
||||
artifact = await _authorized_artifact(request, run_id, artifact_id)
|
||||
mime = str(artifact.get("mime_type") or "application/octet-stream")
|
||||
name = str(artifact.get("name") or artifact_id)
|
||||
disposition = f"attachment; filename*=UTF-8''{quote(name)}"
|
||||
path = str(artifact.get("path") or "")
|
||||
|
||||
if path and await run_in_threadpool(_file_exists, path):
|
||||
return FileResponse(path, media_type=mime, headers={"Content-Disposition": disposition})
|
||||
preview = artifact.get("preview")
|
||||
if preview:
|
||||
return Response(
|
||||
content=str(preview),
|
||||
media_type=mime if _is_textual(mime) else "text/plain",
|
||||
headers={"Content-Disposition": disposition},
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail={"code": "WORKFLOW_ARTIFACT_CONTENT_UNAVAILABLE", "message": "制品文件已过期,无法下载"},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/cancel")
|
||||
async def cancel_run(request: Request, run_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
run = await _authorized_run(request, run_id)
|
||||
if run["status"] in _TERMINAL_STATUSES:
|
||||
return {**_public_run(run), "cancelled": False}
|
||||
updated = await _runs(request).request_cancel(run_id)
|
||||
if updated is None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "WORKFLOW_RUN_CONFLICT", "message": "当前状态无法取消"},
|
||||
)
|
||||
await _events(request).append(
|
||||
run_id=run_id,
|
||||
workflow_id=run["workflow_id"],
|
||||
version_id=run["workflow_version_id"],
|
||||
event_type="run.cancel_requested",
|
||||
payload={},
|
||||
)
|
||||
_nudge(request)
|
||||
audit_workflow("run.cancel", user_id=await get_current_user(request), runId=run_id, workflowId=run["workflow_id"])
|
||||
return {**_public_run(updated), "cancelled": True}
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/resume")
|
||||
async def resume_run(request: Request, run_id: str, body: ResumeRunBody) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
run = await _authorized_run(request, run_id)
|
||||
if run["status"] != "awaiting_input":
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "WORKFLOW_RUN_NOT_RESUMABLE", "message": "运行当前不在等待人工输入"},
|
||||
)
|
||||
pending = run.get("pending_input") or {}
|
||||
node_id = body.node_id or str(pending.get("nodeId") or "")
|
||||
if not node_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_HUMAN_INPUT_INVALID", "message": "缺少 nodeId"},
|
||||
)
|
||||
|
||||
runs = _runs(request)
|
||||
# Single conditional update: token validation + payload write + token
|
||||
# consumption + status flip all in one statement, so two concurrent resume
|
||||
# requests with the same token cannot interleave — exactly one succeeds and
|
||||
# the loser gets 409 without ever touching the run.
|
||||
resumed = await runs.resume_with_payload(
|
||||
run_id,
|
||||
body.resume_token,
|
||||
{
|
||||
"nodeId": node_id,
|
||||
"action": body.action,
|
||||
"values": body.resolved_values(),
|
||||
**({"message": body.message} if body.message else {}),
|
||||
},
|
||||
context=dict(run.get("context") or {}),
|
||||
)
|
||||
if resumed is None:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "WORKFLOW_RESUME_TOKEN_INVALID", "message": "恢复令牌无效或已被使用"},
|
||||
)
|
||||
await _events(request).append(
|
||||
run_id=run_id,
|
||||
workflow_id=run["workflow_id"],
|
||||
version_id=run["workflow_version_id"],
|
||||
event_type="run.resumed",
|
||||
payload={"nodeId": node_id, "action": body.action},
|
||||
node_id=node_id,
|
||||
)
|
||||
_nudge(request)
|
||||
audit_workflow("run.resume", user_id=await get_current_user(request), runId=run_id, nodeId=node_id)
|
||||
return _public_run(resumed)
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/retry", status_code=201)
|
||||
async def retry_run(request: Request, run_id: str) -> dict[str, Any]:
|
||||
"""Start a new run from a failed one.
|
||||
|
||||
Completed node outputs are copied onto the new run so the engine resumes
|
||||
from the first unfinished node. If there is nothing to replay, the new run
|
||||
is a full replay of the same version and inputs, still marked
|
||||
``retry_of_run_id``.
|
||||
"""
|
||||
_require_enabled()
|
||||
run = await _authorized_run(request, run_id)
|
||||
if run["status"] != "failed":
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "WORKFLOW_RUN_NOT_RETRYABLE", "message": "只有失败的运行可以重试"},
|
||||
)
|
||||
|
||||
user_id = await get_current_user(request)
|
||||
runs = _runs(request)
|
||||
active = [r for r in await runs.list_runs(owner_id=user_id, limit=200) if r["status"] in ("queued", "running")]
|
||||
if len(active) >= _cfg().max_concurrent_runs_per_user:
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail={
|
||||
"code": "WORKFLOW_LIMIT_EXCEEDED",
|
||||
"message": f"同时运行的工作流已达上限({_cfg().max_concurrent_runs_per_user})",
|
||||
},
|
||||
)
|
||||
|
||||
context = dict(run.get("context") or {})
|
||||
context.pop("resume_payload", None)
|
||||
context["retry_of_run_id"] = run_id
|
||||
new_run, _created = await runs.create_run(
|
||||
{
|
||||
"workflow_id": run["workflow_id"],
|
||||
"workflow_version_id": run["workflow_version_id"],
|
||||
"owner_id": user_id,
|
||||
"input": run.get("input") or {},
|
||||
"retry_of_run_id": run_id,
|
||||
"context": context,
|
||||
}
|
||||
)
|
||||
for row in await runs.list_node_runs(run_id, current_only=True):
|
||||
if row.get("status") != "completed":
|
||||
continue
|
||||
await runs.record_node_run(
|
||||
{
|
||||
"run_id": new_run["id"],
|
||||
"node_id": row["node_id"],
|
||||
"node_type": row.get("node_type") or "unknown",
|
||||
"status": "completed",
|
||||
"attempt": 1,
|
||||
"iteration": int(row.get("iteration") or 0),
|
||||
"output": row.get("output"),
|
||||
"duration_ms": row.get("duration_ms"),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
await _events(request).append(
|
||||
run_id=new_run["id"],
|
||||
workflow_id=new_run["workflow_id"],
|
||||
version_id=new_run["workflow_version_id"],
|
||||
event_type="run.created",
|
||||
payload={"retryOfRunId": run_id, "inputs": new_run.get("input") or {}},
|
||||
)
|
||||
_nudge(request)
|
||||
audit_workflow("run.retry", user_id=user_id, runId=new_run["id"], retryOfRunId=run_id)
|
||||
return {**_public_run(new_run), "created": True}
|
||||
|
||||
|
||||
# ── events / SSE ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/events")
|
||||
async def list_run_events(
|
||||
request: Request,
|
||||
run_id: str,
|
||||
after: int = Query(default=0, ge=0),
|
||||
after_seq: int | None = Query(default=None, alias="after_seq", ge=0),
|
||||
limit: int = Query(default=_REPLAY_PAGE, ge=1, le=1000),
|
||||
) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _authorized_run(request, run_id)
|
||||
cursor = after_seq if after_seq is not None else after
|
||||
rows = await _events(request).list_after(run_id, after_seq=cursor, limit=limit)
|
||||
return {"events": [envelope_from_row(r) for r in rows], "lastSeq": rows[-1]["seq"] if rows else cursor}
|
||||
|
||||
|
||||
def _frame(payload: dict[str, Any]) -> str:
|
||||
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
return f"id: {payload.get('seq', 0)}\nevent: {payload.get('event', 'message')}\ndata: {body}\n\n"
|
||||
|
||||
|
||||
def _replay_cursor(request: Request, after: int) -> int:
|
||||
"""``Last-Event-ID`` wins over ``?after=``; invalid headers fall back."""
|
||||
raw = request.headers.get("last-event-id")
|
||||
if raw:
|
||||
try:
|
||||
return max(0, int(str(raw).strip()))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return after
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}/stream")
|
||||
async def stream_run(
|
||||
request: Request,
|
||||
run_id: str,
|
||||
after: int = Query(default=0, ge=0),
|
||||
after_seq: int | None = Query(default=None, alias="after_seq", ge=0),
|
||||
) -> StreamingResponse:
|
||||
"""Replay from ``after`` then tail live.
|
||||
|
||||
The database is authoritative for ordering: live frames whose ``seq`` we have
|
||||
not yet replayed are held back and picked up by the next DB poll, so a
|
||||
reconnecting client never sees a gap or an out-of-order ``seq``.
|
||||
``Last-Event-ID`` takes priority over ``after``.
|
||||
"""
|
||||
_require_enabled()
|
||||
run = await _authorized_run(request, run_id)
|
||||
events = _events(request)
|
||||
runs = _runs(request)
|
||||
hub = getattr(request.app.state, "workflow_live_hub", None)
|
||||
cursor_start = _replay_cursor(request, after_seq if after_seq is not None else after)
|
||||
|
||||
async def generator():
|
||||
cursor = cursor_start
|
||||
live_queue = hub.subscribe(run_id) if hub is not None else None
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + _STREAM_DEADLINE_SECONDS
|
||||
next_poll = 0.0
|
||||
last_beat = loop.time()
|
||||
try:
|
||||
while loop.time() < deadline:
|
||||
if await request.is_disconnected():
|
||||
return
|
||||
|
||||
if loop.time() >= next_poll:
|
||||
while True:
|
||||
rows = await events.list_after(run_id, after_seq=cursor, limit=_REPLAY_PAGE)
|
||||
if not rows:
|
||||
break
|
||||
for row in rows:
|
||||
yield _frame(envelope_from_row(row))
|
||||
cursor = max(cursor, int(row["seq"]))
|
||||
last_beat = loop.time()
|
||||
if row["event_type"] in _TERMINAL_EVENTS:
|
||||
return
|
||||
if len(rows) < _REPLAY_PAGE:
|
||||
break
|
||||
current = await runs.get_run(run_id)
|
||||
if current is None:
|
||||
return
|
||||
if current["status"] in _TERMINAL_STATUSES:
|
||||
# Terminal state with no terminal event (e.g. a crash
|
||||
# before the event landed): synthesise a closing frame
|
||||
# so the client is never left hanging.
|
||||
yield _frame(
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"runId": run_id,
|
||||
"workflowId": current["workflow_id"],
|
||||
"versionId": current["workflow_version_id"],
|
||||
"seq": cursor,
|
||||
"event": f"run.{current['status']}",
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"data": {
|
||||
"status": current["status"],
|
||||
"output": current.get("output"),
|
||||
"error": current.get("error"),
|
||||
"synthesised": True,
|
||||
},
|
||||
}
|
||||
)
|
||||
return
|
||||
next_poll = loop.time() + _DB_POLL_SECONDS
|
||||
|
||||
if live_queue is not None:
|
||||
try:
|
||||
event = await asyncio.wait_for(live_queue.get(), timeout=_LIVE_WAIT_SECONDS)
|
||||
except TimeoutError:
|
||||
event = None
|
||||
if event is not None and int(event.get("seq") or 0) > cursor:
|
||||
yield _frame(event)
|
||||
cursor = int(event["seq"])
|
||||
last_beat = loop.time()
|
||||
if event.get("event") in _TERMINAL_EVENTS:
|
||||
return
|
||||
else:
|
||||
await asyncio.sleep(_LIVE_WAIT_SECONDS)
|
||||
|
||||
if loop.time() - last_beat >= _HEARTBEAT_SECONDS:
|
||||
yield ": heartbeat\n\n"
|
||||
last_beat = loop.time()
|
||||
finally:
|
||||
if live_queue is not None and hub is not None:
|
||||
hub.unsubscribe(run_id, live_queue)
|
||||
|
||||
return StreamingResponse(
|
||||
generator(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache, no-transform",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
"X-Workflow-Run-Status": run["status"],
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,351 @@
|
||||
"""Studio facade: camelCase DTOs for the standalone Coze canvas page.
|
||||
|
||||
Four endpoints in front of the standard workflow store:
|
||||
|
||||
GET /api/workflows/{id}/studio-document
|
||||
PUT /api/workflows/{id}/studio-draft
|
||||
POST /api/workflows/{id}/studio-validate
|
||||
POST /api/workflows/{id}/studio-publish
|
||||
|
||||
Design contract (docs/WORKFLOW_STUDIO_COZE_FRONTEND_ADAPTATION_ZH.md §3-§4):
|
||||
|
||||
- One workflow, two documents. ``canvasSchemaJson`` is an opaque Coze canvas
|
||||
saved for editing; ``executionGraph`` is the WorkflowGraph v1.0 the runtime
|
||||
executes. Both land in the SAME CAS update, so they can never disagree on
|
||||
revision.
|
||||
- The facade only maps DTO naming and delegates to the standard store /
|
||||
validator / publish service — it never forks versioning logic.
|
||||
- Publishing a workflow whose draft has no valid execution graph is a 400, not
|
||||
a silent empty-graph publish.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user
|
||||
from app.gateway.workflow_audit import audit_workflow
|
||||
from app.gateway.workflow_resource_checks import build_resource_catalog, make_resource_checker
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.persistence.workflows import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
from deerflow.workflows.schemas import WorkflowGraph
|
||||
from deerflow.workflows.validator import validate_workflow_graph
|
||||
|
||||
router = APIRouter(prefix="/api/workflows", tags=["workflow-studio"])
|
||||
|
||||
# Canvas document guards: it is stored and replayed as an opaque document, so
|
||||
# the only checks are size / JSON shape / depth / obvious credential keys.
|
||||
_CANVAS_MAX_CHARS = 2_000_000
|
||||
_CANVAS_MAX_DEPTH = 64
|
||||
_CANVAS_SECRET_KEYS = frozenset(
|
||||
{
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
"cookie",
|
||||
"x-api-key",
|
||||
"apikey",
|
||||
"api_key",
|
||||
"secret",
|
||||
"client_secret",
|
||||
"password",
|
||||
"dsn",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"resumetoken",
|
||||
"resume_token",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class StudioDraftBody(BaseModel):
|
||||
expected_revision: int = Field(alias="expectedRevision", ge=0)
|
||||
# Canvas may arrive as a JSON string or an embedded object; both are
|
||||
# canonicalised to compact JSON text before storage.
|
||||
canvas_schema_json: Any = Field(default=None, alias="canvasSchemaJson")
|
||||
execution_graph: dict[str, Any] = Field(alias="executionGraph")
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class StudioValidateBody(BaseModel):
|
||||
execution_graph: dict[str, Any] | None = Field(default=None, alias="executionGraph")
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class StudioPublishBody(BaseModel):
|
||||
expected_revision: int | None = Field(default=None, alias="expectedRevision", ge=0)
|
||||
change_note: str = Field(default="", alias="changeNote", max_length=2000)
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
if not get_app_config().workflows.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
def _get_store(request: Request) -> WorkflowStore:
|
||||
store = getattr(request.app.state, "workflow_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
return store
|
||||
|
||||
|
||||
async def _load_owned(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
"""Owner or admin only — the studio edits the draft, so read == write."""
|
||||
from app.gateway.routers.workflows import _load_owned as _standard_load_owned
|
||||
|
||||
return await _standard_load_owned(request, workflow_id, write=True)
|
||||
|
||||
|
||||
def _limits() -> dict[str, int]:
|
||||
cfg = get_app_config().workflows
|
||||
return {
|
||||
"system_max_steps": cfg.max_steps,
|
||||
"system_max_loop_iterations": cfg.max_loop_iterations,
|
||||
"system_max_parallelism": cfg.max_parallelism,
|
||||
}
|
||||
|
||||
|
||||
def _parse_canvas(raw: Any) -> str:
|
||||
"""Validate + canonicalise the canvas document into compact JSON text."""
|
||||
if raw is None or raw == "":
|
||||
return "{}"
|
||||
if isinstance(raw, str):
|
||||
text = raw
|
||||
try:
|
||||
document = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_CANVAS_INVALID", "message": "画布 JSON 无法解析"},
|
||||
) from exc
|
||||
elif isinstance(raw, dict):
|
||||
document = raw
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_CANVAS_INVALID", "message": "画布必须是 JSON 对象或 JSON 字符串"},
|
||||
)
|
||||
text = json.dumps(document, ensure_ascii=False, separators=(",", ":"))
|
||||
if len(text) > _CANVAS_MAX_CHARS:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_CANVAS_INVALID", "message": f"画布文档超过 {_CANVAS_MAX_CHARS} 字符上限"},
|
||||
)
|
||||
offending = _find_secret_key(document, 0)
|
||||
if offending:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_CANVAS_INVALID",
|
||||
"message": f"画布中出现凭据类字段 {offending},请改用凭据引用",
|
||||
"field": offending,
|
||||
},
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def _find_secret_key(node: Any, depth: int) -> str | None:
|
||||
if depth > _CANVAS_MAX_DEPTH:
|
||||
return "<max-depth-exceeded>"
|
||||
if isinstance(node, dict):
|
||||
for key, value in node.items():
|
||||
name = str(key).strip().lower()
|
||||
if name in _CANVAS_SECRET_KEYS and value not in (None, "", {}, []):
|
||||
return str(key)
|
||||
found = _find_secret_key(value, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
found = _find_secret_key(item, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _studio_issues(issues: list[Any]) -> list[dict[str, Any]]:
|
||||
out: list[dict[str, Any]] = []
|
||||
for issue in issues:
|
||||
body = issue.model_dump(by_alias=True)
|
||||
details = body.get("details") or {}
|
||||
out.append(
|
||||
{
|
||||
"code": body.get("code"),
|
||||
"nodeId": body.get("nodeId"),
|
||||
"field": details.get("field") if isinstance(details, dict) else None,
|
||||
"message": body.get("message"),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
async def _validate_with_resources(request: Request, workflow_id: str, raw_graph: dict[str, Any], user_id: str) -> list[Any]:
|
||||
catalog = await build_resource_catalog(request, user_id=user_id)
|
||||
checker = make_resource_checker(catalog, config=get_app_config().workflows, current_workflow_id=workflow_id)
|
||||
return validate_workflow_graph(raw_graph, resource_checker=checker, **_limits())
|
||||
|
||||
|
||||
@router.get("/{workflow_id}/studio-document")
|
||||
async def get_studio_document(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id)
|
||||
store = _get_store(request)
|
||||
graph = definition.get("draft_graph") or {}
|
||||
versions = await store.list_versions(workflow_id)
|
||||
return {
|
||||
"workflowId": definition["id"],
|
||||
"name": definition.get("name"),
|
||||
"description": definition.get("description") or "",
|
||||
"status": definition.get("status"),
|
||||
"draftRevision": int(definition.get("draft_revision") or 0),
|
||||
"publishedVersionId": versions[0]["id"] if versions else None,
|
||||
# Canvas stays an opaque JSON document (string); never null.
|
||||
"canvasSchemaJson": definition.get("draft_canvas_schema") or "{}",
|
||||
"executionGraph": graph,
|
||||
"inputSchema": graph.get("inputSchema") or graph.get("input_schema") or {"type": "object", "properties": {}},
|
||||
"outputSchema": graph.get("outputSchema") or graph.get("output_schema") or {"type": "object", "properties": {}},
|
||||
"updatedAt": definition.get("updated_at"),
|
||||
}
|
||||
|
||||
|
||||
@router.put("/{workflow_id}/studio-draft")
|
||||
async def put_studio_draft(request: Request, workflow_id: str, body: StudioDraftBody) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id)
|
||||
store = _get_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
|
||||
try:
|
||||
WorkflowGraph.model_validate(body.execution_graph)
|
||||
except Exception as exc: # noqa: BLE001 - surface pydantic as 400
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_SCHEMA_INVALID", "message": f"执行图不符合 WorkflowGraph v1.0: {exc}"},
|
||||
) from exc
|
||||
|
||||
canvas_text = _parse_canvas(body.canvas_schema_json)
|
||||
try:
|
||||
saved = await store.save_studio_draft(
|
||||
workflow_id,
|
||||
expected_revision=body.expected_revision,
|
||||
graph=body.execution_graph,
|
||||
canvas_schema=canvas_text,
|
||||
updated_by=user_id,
|
||||
)
|
||||
except WorkflowDraftConflictError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"message": "草稿版本冲突,请刷新后重试或另存副本",
|
||||
"currentRevision": exc.current_revision,
|
||||
},
|
||||
) from exc
|
||||
except WorkflowNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在") from exc
|
||||
audit_workflow("workflow.studio_draft", user_id=user_id, workflowId=workflow_id, revision=saved.get("draft_revision"))
|
||||
return {"revision": int(saved.get("draft_revision") or 0), "updatedAt": saved.get("updated_at")}
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/studio-validate")
|
||||
async def studio_validate(request: Request, workflow_id: str, body: StudioValidateBody | None = None) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id)
|
||||
user_id = await get_current_user(request)
|
||||
graph = (body.execution_graph if body and body.execution_graph is not None else definition.get("draft_graph")) or {}
|
||||
try:
|
||||
issues = await _validate_with_resources(request, workflow_id, graph, user_id)
|
||||
except Exception as exc: # noqa: BLE001 - unparsable graph is a validation result, not a 500
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_SCHEMA_INVALID", "message": f"图结构无效: {exc}"},
|
||||
) from exc
|
||||
valid = not issues
|
||||
graph_hash = None
|
||||
if valid:
|
||||
try:
|
||||
graph_hash = WorkflowGraph.model_validate(graph).graph_hash()
|
||||
except Exception: # noqa: BLE001
|
||||
valid = False
|
||||
return {"valid": valid, "issues": _studio_issues(issues), "graphHash": graph_hash}
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/studio-publish")
|
||||
async def studio_publish(request: Request, workflow_id: str, body: StudioPublishBody | None = None) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id)
|
||||
store = _get_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
body = body or StudioPublishBody()
|
||||
|
||||
graph_raw = definition.get("draft_graph") or {}
|
||||
if not graph_raw or not (graph_raw.get("nodes") or []):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_SCHEMA_INVALID",
|
||||
"message": "草稿没有可发布的执行图(只有画布文档不能发布)",
|
||||
},
|
||||
)
|
||||
try:
|
||||
issues = await _validate_with_resources(request, workflow_id, graph_raw, user_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"code": "WORKFLOW_SCHEMA_INVALID", "message": f"图结构无效: {exc}"},
|
||||
) from exc
|
||||
if issues:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_SCHEMA_INVALID",
|
||||
"message": "发布校验未通过",
|
||||
"issues": _studio_issues(issues),
|
||||
},
|
||||
)
|
||||
|
||||
if body.expected_revision is not None and body.expected_revision != definition.get("draft_revision"):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"message": "发布前草稿版本已变化",
|
||||
"currentRevision": definition.get("draft_revision"),
|
||||
},
|
||||
)
|
||||
|
||||
graph = WorkflowGraph.model_validate(graph_raw)
|
||||
canvas_text = str(definition.get("draft_canvas_schema") or "{}")
|
||||
canvas_hash = sha256(canvas_text.encode("utf-8")).hexdigest()
|
||||
version = await store.publish_version(
|
||||
workflow_id,
|
||||
graph=graph.model_dump(by_alias=True),
|
||||
graph_hash=graph.graph_hash(),
|
||||
published_by=user_id,
|
||||
change_note=body.change_note,
|
||||
canvas_schema_hash=canvas_hash,
|
||||
)
|
||||
audit_workflow(
|
||||
"workflow.studio_publish",
|
||||
user_id=user_id,
|
||||
workflowId=workflow_id,
|
||||
versionId=version.get("id"),
|
||||
graphHash=graph.graph_hash(),
|
||||
)
|
||||
return {
|
||||
"versionId": version.get("id"),
|
||||
"versionNumber": version.get("version_number"),
|
||||
"publishedAt": version.get("published_at"),
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Workflow definition / draft / publish API (phase 1 — no execution yet)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_optional_user_from_request
|
||||
from app.gateway.workflow_audit import audit_workflow
|
||||
from app.gateway.workflow_resource_checks import build_resource_catalog, make_resource_checker
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.persistence.workflows import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
from deerflow.workflows.schemas import WorkflowGraph
|
||||
from deerflow.workflows.validator import validate_workflow_graph
|
||||
|
||||
router = APIRouter(prefix="/api/workflows", tags=["workflows"])
|
||||
|
||||
|
||||
class WorkflowCreateRequest(BaseModel):
|
||||
name: str = Field(default="未命名工作流", min_length=1, max_length=255)
|
||||
description: str = ""
|
||||
graph: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class WorkflowPatchRequest(BaseModel):
|
||||
name: str | None = Field(default=None, min_length=1, max_length=255)
|
||||
description: str | None = None
|
||||
|
||||
|
||||
class DraftPutRequest(BaseModel):
|
||||
expected_revision: int = Field(alias="expectedRevision", ge=0)
|
||||
graph: dict[str, Any]
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class PublishRequest(BaseModel):
|
||||
change_note: str = Field(default="", alias="changeNote")
|
||||
expected_revision: int | None = Field(default=None, alias="expectedRevision")
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
|
||||
class ValidateRequest(BaseModel):
|
||||
graph: dict[str, Any] | None = None
|
||||
|
||||
|
||||
def _get_store(request: Request) -> WorkflowStore:
|
||||
store = getattr(request.app.state, "workflow_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
return store
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
cfg = get_app_config().workflows
|
||||
if not cfg.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
async def _current_user_id(request: Request) -> str:
|
||||
return await get_current_user(request)
|
||||
|
||||
|
||||
def _can_access(definition: dict[str, Any], user_id: str, *, write: bool = False) -> bool:
|
||||
# Phase 1: owner-only. Shared/org ACLs land later.
|
||||
owner = definition.get("owner_id")
|
||||
if owner == user_id:
|
||||
return True
|
||||
# Auth disabled environments often use "default" / anonymous — still owner-scoped.
|
||||
if write:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
async def _load_owned(request: Request, workflow_id: str, *, write: bool = False) -> dict[str, Any]:
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
definition = await store.get_definition(workflow_id, include_draft=True)
|
||||
if definition is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
if not _can_access(definition, user_id, write=write):
|
||||
# Soft admin bypass for list/read when system_role=admin
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is not None and getattr(user, "system_role", None) == "admin" and not write:
|
||||
return definition
|
||||
if user is not None and getattr(user, "system_role", None) == "admin" and write:
|
||||
return definition
|
||||
raise HTTPException(status_code=403, detail="无权访问该工作流")
|
||||
return definition
|
||||
|
||||
|
||||
def _limits() -> dict[str, int]:
|
||||
cfg = get_app_config().workflows
|
||||
return {
|
||||
"system_max_steps": cfg.max_steps,
|
||||
"system_max_loop_iterations": cfg.max_loop_iterations,
|
||||
"system_max_parallelism": cfg.max_parallelism,
|
||||
}
|
||||
|
||||
|
||||
@router.get("")
|
||||
async def list_workflows(
|
||||
request: Request,
|
||||
include_archived: bool = Query(False, alias="includeArchived"),
|
||||
status: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
user = await get_optional_user_from_request(request)
|
||||
is_admin = user is not None and getattr(user, "system_role", None) == "admin"
|
||||
rows = await store.list_definitions(
|
||||
owner_id=None if is_admin else user_id,
|
||||
status=status,
|
||||
include_archived=include_archived,
|
||||
)
|
||||
if is_admin:
|
||||
# Still hide other users' drafts from non-explicit admin listing? Phase 1: admin sees all.
|
||||
pass
|
||||
else:
|
||||
rows = [r for r in rows if r.get("owner_id") == user_id]
|
||||
return {"workflows": rows}
|
||||
|
||||
|
||||
@router.post("")
|
||||
async def create_workflow(request: Request, body: WorkflowCreateRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
created = await store.create_definition(
|
||||
{
|
||||
"name": body.name,
|
||||
"description": body.description,
|
||||
"owner_id": user_id,
|
||||
"draft_graph": body.graph,
|
||||
}
|
||||
)
|
||||
audit_workflow("workflow.create", user_id=user_id, workflowId=created["id"])
|
||||
return created
|
||||
|
||||
|
||||
@router.get("/{workflow_id}")
|
||||
async def get_workflow(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
return await _load_owned(request, workflow_id, write=False)
|
||||
|
||||
|
||||
@router.patch("/{workflow_id}")
|
||||
async def patch_workflow(request: Request, workflow_id: str, body: WorkflowPatchRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id, write=True)
|
||||
store = _get_store(request)
|
||||
updated = await store.update_definition(
|
||||
workflow_id,
|
||||
body.model_dump(exclude_unset=True),
|
||||
)
|
||||
if updated is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
return updated
|
||||
|
||||
|
||||
@router.delete("/{workflow_id}")
|
||||
async def archive_workflow(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id, write=True)
|
||||
store = _get_store(request)
|
||||
archived = await store.archive_definition(workflow_id)
|
||||
if archived is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
audit_workflow("workflow.archive", user_id=await _current_user_id(request), workflowId=workflow_id)
|
||||
return archived
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/copy")
|
||||
async def copy_workflow(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id, write=False)
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
created = await store.create_definition(
|
||||
{
|
||||
"name": f"{definition.get('name') or '未命名工作流'} 副本",
|
||||
"description": definition.get("description") or "",
|
||||
"owner_id": user_id,
|
||||
"draft_graph": definition.get("draft_graph"),
|
||||
}
|
||||
)
|
||||
audit_workflow("workflow.copy", user_id=user_id, workflowId=created["id"], sourceWorkflowId=workflow_id)
|
||||
return created
|
||||
|
||||
|
||||
@router.put("/{workflow_id}/draft")
|
||||
async def put_draft(request: Request, workflow_id: str, body: DraftPutRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id, write=True)
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
try:
|
||||
# Light structural parse so obviously broken drafts fail early; full publish
|
||||
# checks run on /validate and /publish.
|
||||
WorkflowGraph.model_validate(body.graph)
|
||||
except Exception as exc: # noqa: BLE001 — surface pydantic as 400
|
||||
raise HTTPException(status_code=400, detail=f"图结构无效: {exc}") from exc
|
||||
try:
|
||||
return await store.save_draft(
|
||||
workflow_id,
|
||||
expected_revision=body.expected_revision,
|
||||
graph=body.graph,
|
||||
updated_by=user_id,
|
||||
)
|
||||
except WorkflowDraftConflictError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"message": "草稿版本冲突,请刷新后重试或另存副本",
|
||||
"currentRevision": exc.current_revision,
|
||||
},
|
||||
) from exc
|
||||
except WorkflowNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在") from exc
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/validate")
|
||||
async def validate_workflow(request: Request, workflow_id: str, body: ValidateRequest | None = None) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id, write=False)
|
||||
graph = (body.graph if body and body.graph is not None else definition.get("draft_graph")) or {}
|
||||
issues = validate_workflow_graph(graph, **_limits())
|
||||
return {
|
||||
"ok": len(issues) == 0,
|
||||
"issues": [i.model_dump(by_alias=True) for i in issues],
|
||||
"graphHash": WorkflowGraph.model_validate(graph).graph_hash() if not issues else None,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/publish")
|
||||
async def publish_workflow(request: Request, workflow_id: str, body: PublishRequest | None = None) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
definition = await _load_owned(request, workflow_id, write=True)
|
||||
body = body or PublishRequest()
|
||||
if body.expected_revision is not None and body.expected_revision != definition.get("draft_revision"):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"message": "发布前草稿版本已变化",
|
||||
"currentRevision": definition.get("draft_revision"),
|
||||
},
|
||||
)
|
||||
graph_raw = definition.get("draft_graph") or {}
|
||||
# Same resource/permission checks as the studio facade (§5.3): the standard
|
||||
# publish path must not accept graphs referencing invisible agents or
|
||||
# forbidden resources.
|
||||
catalog = await build_resource_catalog(request, user_id=await _current_user_id(request))
|
||||
checker = make_resource_checker(catalog, config=get_app_config().workflows, current_workflow_id=workflow_id)
|
||||
issues = validate_workflow_graph(graph_raw, resource_checker=checker, **_limits())
|
||||
if issues:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"code": "WORKFLOW_SCHEMA_INVALID",
|
||||
"message": "发布校验未通过",
|
||||
"issues": [i.model_dump(by_alias=True) for i in issues],
|
||||
},
|
||||
)
|
||||
graph = WorkflowGraph.model_validate(graph_raw)
|
||||
store = _get_store(request)
|
||||
user_id = await _current_user_id(request)
|
||||
canvas_text = str(definition.get("draft_canvas_schema") or "{}")
|
||||
version = await store.publish_version(
|
||||
workflow_id,
|
||||
graph=graph.model_dump(by_alias=True),
|
||||
graph_hash=graph.graph_hash(),
|
||||
published_by=user_id,
|
||||
change_note=body.change_note,
|
||||
canvas_schema_hash=sha256(canvas_text.encode("utf-8")).hexdigest(),
|
||||
)
|
||||
audit_workflow(
|
||||
"workflow.publish",
|
||||
user_id=user_id,
|
||||
workflowId=workflow_id,
|
||||
versionId=version.get("id"),
|
||||
graphHash=graph.graph_hash(),
|
||||
)
|
||||
return version
|
||||
|
||||
|
||||
@router.get("/{workflow_id}/versions")
|
||||
async def list_versions(request: Request, workflow_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id, write=False)
|
||||
store = _get_store(request)
|
||||
return {"versions": await store.list_versions(workflow_id)}
|
||||
|
||||
|
||||
@router.get("/{workflow_id}/versions/{version_id}")
|
||||
async def get_version(request: Request, workflow_id: str, version_id: str) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await _load_owned(request, workflow_id, write=False)
|
||||
store = _get_store(request)
|
||||
version = await store.get_version(version_id)
|
||||
if version is None or version.get("workflow_id") != workflow_id:
|
||||
raise HTTPException(status_code=404, detail="版本不存在")
|
||||
return version
|
||||
@@ -0,0 +1,368 @@
|
||||
"""Minimal Coze Studio workflow_api compatibility layer (phase 1).
|
||||
|
||||
Internal services always use DeerFlow draft DTOs. This module only translates
|
||||
request/response shapes for the standalone canvas page: ``/canvas`` replies
|
||||
with the Coze playground envelope and ``/save`` persists the Coze canvas into
|
||||
``save_studio_draft`` without letting it anywhere near ``WorkflowGraph``
|
||||
validation (the canvas is an opaque editing document; the executable graph is
|
||||
compiled client-side and saved via the studio facade).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_current_user, get_optional_user_from_request
|
||||
from app.gateway.routers.workflow_studio import _parse_canvas
|
||||
from deerflow.config.app_config import get_app_config
|
||||
from deerflow.persistence.workflows import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/workflow_api", tags=["workflow-coze-compat"])
|
||||
# Playground 启动还会打这些 Coze 壳接口;缺了它们画布和节点资源都不会挂载。
|
||||
chrome_router = APIRouter(tags=["workflow-coze-compat"])
|
||||
|
||||
_DEFAULT_SPACE: dict[str, Any] = {
|
||||
"id": "1",
|
||||
"space_id": 1,
|
||||
"name": "默认空间",
|
||||
"description": "",
|
||||
# SpaceType.Personal = 1;Playground 找不到个人空间会去调 space/save。
|
||||
"space_type": 1,
|
||||
"space_mode": 0,
|
||||
"icon_uri": "",
|
||||
"icon_url": "",
|
||||
"owner_id": "1",
|
||||
"owner_user_id": "1",
|
||||
"role_type": 1,
|
||||
"has_permission": True,
|
||||
}
|
||||
|
||||
# Coze playground node_template_list payload. The standalone canvas maps
|
||||
# DeerFlow node kinds onto these StandardNodeType ids (see the frontend's
|
||||
# node-catalog.ts); entries keep the Coze field names (node_type / icon_url /
|
||||
# color) so the playground's template list renders without adaptation.
|
||||
_COZE_NODE_TEMPLATES: list[dict[str, Any]] = [
|
||||
{"node_type": "1", "name": "开始", "icon_url": "", "color": "#5C62FF", "desc": "工作流入口,定义输入参数"},
|
||||
{"node_type": "2", "name": "结束", "icon_url": "", "color": "#5C62FF", "desc": "工作流出口,汇总输出"},
|
||||
{"node_type": "3", "name": "大模型", "icon_url": "", "color": "#5C62FF", "desc": "调用智能体处理任务", "support_batch": 1},
|
||||
{"node_type": "5", "name": "代码", "icon_url": "", "color": "#00B2B2", "desc": "在沙箱中执行代码", "support_batch": 0},
|
||||
{"node_type": "8", "name": "选择器", "icon_url": "", "color": "#00B2B2", "desc": "按条件分支执行", "support_batch": 0},
|
||||
{"node_type": "9", "name": "工作流", "icon_url": "", "color": "#00B83E", "desc": "调用已发布子工作流", "support_batch": 1},
|
||||
{"node_type": "13", "name": "输出", "icon_url": "", "color": "#5C62FF", "desc": "定义输出字段", "support_batch": 0},
|
||||
{"node_type": "15", "name": "文本处理", "icon_url": "", "color": "#3071F2", "desc": "模板/文本转换", "support_batch": 0},
|
||||
{"node_type": "18", "name": "问答", "icon_url": "", "color": "#3071F2", "desc": "人工介入确认", "support_batch": 0},
|
||||
{"node_type": "21", "name": "循环", "icon_url": "", "color": "#00B2B2", "desc": "循环执行子图", "support_batch": 0},
|
||||
{"node_type": "32", "name": "变量聚合", "icon_url": "", "color": "#00B2B2", "desc": "合并多分支变量", "support_batch": 0},
|
||||
{"node_type": "43", "name": "查询数据", "icon_url": "", "color": "#F2B600", "desc": "SQL 只读查询", "support_batch": 0},
|
||||
{"node_type": "45", "name": "HTTP 请求", "icon_url": "", "color": "#3071F2", "desc": "调用外部接口", "support_batch": 0},
|
||||
]
|
||||
|
||||
_COZE_NODE_CATEGORIES: list[dict[str, Any]] = [
|
||||
{"name": "", "node_type_list": ["3", "9"]},
|
||||
{"name": "业务逻辑", "node_type_list": ["5", "8", "21", "32"]},
|
||||
{"name": "输入&输出", "node_type_list": ["1", "2", "13"]},
|
||||
{"name": "组件", "node_type_list": ["15", "18", "45"]},
|
||||
{"name": "数据库", "node_type_list": ["43"]},
|
||||
]
|
||||
|
||||
# Mirrors the standalone page's built-in empty canvas (start → end), used when
|
||||
# a workflow has never been opened in the Coze editor.
|
||||
_DEFAULT_CANVAS_SCHEMA = json.dumps(
|
||||
{
|
||||
"nodes": [
|
||||
{
|
||||
"id": "100001",
|
||||
"type": "1",
|
||||
"meta": {"position": {"x": 0, "y": 0}},
|
||||
"data": {"outputs": [{"type": "string", "name": "", "required": True}]},
|
||||
},
|
||||
{
|
||||
"id": "900001",
|
||||
"type": "2",
|
||||
"meta": {"position": {"x": 1000, "y": 0}},
|
||||
},
|
||||
],
|
||||
"edges": [{"sourceNodeID": "100001", "targetNodeID": "900001"}],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
|
||||
class CozeCreateRequest(BaseModel):
|
||||
name: str = Field(default="未命名工作流")
|
||||
description: str = ""
|
||||
graph: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class CozeCanvasRequest(BaseModel):
|
||||
workflow_id: str | None = Field(default=None, alias="workflowId")
|
||||
workflow_id_snake: str | None = Field(default=None, alias="workflow_id")
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
def resolved_id(self) -> str | None:
|
||||
return self.workflow_id or self.workflow_id_snake
|
||||
|
||||
|
||||
class CozeSaveRequest(BaseModel):
|
||||
workflow_id: str | None = Field(default=None, alias="workflowId")
|
||||
workflow_id_snake: str | None = Field(default=None, alias="workflow_id")
|
||||
expected_revision: int | None = Field(default=None, alias="expectedRevision")
|
||||
expected_revision_snake: int | None = Field(default=None, alias="expected_revision")
|
||||
# Coze posts the canvas as a JSON string under ``schema``.
|
||||
schema_: str | dict[str, Any] | None = Field(default=None, alias="schema")
|
||||
graph: str | dict[str, Any] | None = None
|
||||
data: dict[str, Any] | None = None
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
def resolved_id(self) -> str | None:
|
||||
return self.workflow_id or self.workflow_id_snake
|
||||
|
||||
def resolved_revision(self) -> int | None:
|
||||
if self.expected_revision is not None:
|
||||
return self.expected_revision
|
||||
return self.expected_revision_snake
|
||||
|
||||
def resolved_canvas(self) -> str | dict[str, Any] | None:
|
||||
if self.schema_ is not None:
|
||||
return self.schema_
|
||||
if isinstance(self.graph, (str, dict)):
|
||||
return self.graph
|
||||
if self.data is not None:
|
||||
for key in ("schema", "graph"):
|
||||
value = self.data.get(key)
|
||||
if isinstance(value, (str, dict)):
|
||||
return value
|
||||
return self.data
|
||||
return None
|
||||
|
||||
|
||||
def _require_enabled() -> None:
|
||||
if not get_app_config().workflows.enabled:
|
||||
raise HTTPException(status_code=503, detail="Workflow Studio is disabled")
|
||||
|
||||
|
||||
def _get_store(request: Request) -> WorkflowStore:
|
||||
store = getattr(request.app.state, "workflow_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="Workflow store not available")
|
||||
return store
|
||||
|
||||
|
||||
def _canvas_schema_text(definition: dict[str, Any]) -> str:
|
||||
raw = definition.get("draft_canvas_schema")
|
||||
if isinstance(raw, str) and raw.strip() not in ("", "{}", "null"):
|
||||
return raw
|
||||
if isinstance(raw, dict) and raw:
|
||||
return json.dumps(raw, ensure_ascii=False)
|
||||
return _DEFAULT_CANVAS_SCHEMA
|
||||
|
||||
|
||||
def _to_coze_canvas(definition: dict[str, Any], *, is_published: bool = False) -> dict[str, Any]:
|
||||
owner = str(definition.get("owner_id") or "1")
|
||||
return {
|
||||
"code": 0,
|
||||
"msg": "success",
|
||||
"data": {
|
||||
"workflow": {
|
||||
"workflow_id": definition["id"],
|
||||
"space_id": "1",
|
||||
"name": definition.get("name"),
|
||||
"desc": definition.get("description") or "",
|
||||
"schema_json": _canvas_schema_text(definition),
|
||||
"status": 1,
|
||||
"collaborator_mode": "Open",
|
||||
"creator": {"user_id": owner, "nickname": owner},
|
||||
"is_published": is_published,
|
||||
},
|
||||
"vcs_data": {"can_edit": True},
|
||||
"db_data": {"status": 1},
|
||||
"operation_info": {},
|
||||
"is_bind_agent": False,
|
||||
"workflow_version": "1",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def _load_owned_definition(request: Request, store: WorkflowStore, workflow_id: str, user_id: str) -> dict[str, Any]:
|
||||
definition = await store.get_definition(workflow_id, include_draft=True)
|
||||
if definition is None:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在")
|
||||
if definition.get("owner_id") == user_id:
|
||||
return definition
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is not None and getattr(user, "system_role", None) == "admin":
|
||||
return definition
|
||||
raise HTTPException(status_code=403, detail="无权访问该工作流")
|
||||
|
||||
|
||||
def _coze_ok(data: dict[str, Any]) -> dict[str, Any]:
|
||||
return {"code": 0, "msg": "success", "data": data}
|
||||
|
||||
|
||||
def _account_payload(user: Any, user_id: str) -> dict[str, Any]:
|
||||
email = str(getattr(user, "email", "") or "")
|
||||
nickname = email.split("@", 1)[0] if email else user_id
|
||||
return {
|
||||
"user_id_str": user_id,
|
||||
"user_id": user_id,
|
||||
"nickname": nickname,
|
||||
"name": nickname,
|
||||
"email": email,
|
||||
"avatar_url": "",
|
||||
"login_status": "logined",
|
||||
}
|
||||
|
||||
|
||||
async def _require_studio_user(request: Request) -> tuple[str, Any]:
|
||||
_require_enabled()
|
||||
user_id = await get_current_user(request)
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=401, detail="未登录")
|
||||
user = await get_optional_user_from_request(request)
|
||||
return user_id, user
|
||||
|
||||
|
||||
def _space_list_payload() -> dict[str, Any]:
|
||||
return {
|
||||
"has_personal_space": True,
|
||||
"bot_space_list": [_DEFAULT_SPACE],
|
||||
"team_space_num": 0,
|
||||
"max_team_space_num": 10,
|
||||
"recently_used_space_list": [],
|
||||
}
|
||||
|
||||
|
||||
@chrome_router.api_route("/api/passport/account/info/v2/", methods=["GET", "POST"])
|
||||
async def coze_account_info(request: Request) -> dict[str, Any]:
|
||||
user_id, user = await _require_studio_user(request)
|
||||
return _coze_ok(_account_payload(user, user_id))
|
||||
|
||||
|
||||
@chrome_router.api_route("/api/space/list", methods=["GET", "POST"])
|
||||
async def coze_space_list(request: Request) -> dict[str, Any]:
|
||||
await _require_studio_user(request)
|
||||
payload = _space_list_payload()
|
||||
# developer_api.SpaceList 把列表放在顶层;playground v2 读 data。
|
||||
return {"code": 0, "msg": "success", "data": payload, **payload}
|
||||
|
||||
|
||||
@chrome_router.api_route(
|
||||
"/api/playground_api/space/list",
|
||||
methods=["GET", "POST", "PUT", "PATCH"],
|
||||
)
|
||||
@chrome_router.api_route(
|
||||
"/api/playground_api/space/list/",
|
||||
methods=["GET", "POST", "PUT", "PATCH"],
|
||||
)
|
||||
async def coze_playground_space_list(request: Request) -> dict[str, Any]:
|
||||
await _require_studio_user(request)
|
||||
return _coze_ok(_space_list_payload())
|
||||
|
||||
|
||||
@chrome_router.api_route(
|
||||
"/api/playground_api/space/save",
|
||||
methods=["GET", "POST"],
|
||||
)
|
||||
async def coze_playground_space_save(request: Request) -> dict[str, Any]:
|
||||
await _require_studio_user(request)
|
||||
return _coze_ok({"id": _DEFAULT_SPACE["id"], "check_not_pass": False})
|
||||
|
||||
|
||||
@chrome_router.api_route("/api/bot/get_type_list", methods=["GET", "POST"])
|
||||
async def coze_model_type_list(request: Request) -> dict[str, Any]:
|
||||
await _require_studio_user(request)
|
||||
return _coze_ok({"model_list": []})
|
||||
|
||||
|
||||
@router.post("/create")
|
||||
async def coze_create(request: Request, body: CozeCreateRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
store = _get_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
created = await store.create_definition(
|
||||
{
|
||||
"name": body.name,
|
||||
"description": body.description,
|
||||
"owner_id": user_id,
|
||||
}
|
||||
)
|
||||
return {"code": 0, "msg": "success", "data": {"workflow_id": created["id"]}}
|
||||
|
||||
|
||||
@router.post("/canvas")
|
||||
async def coze_canvas(request: Request, body: CozeCanvasRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
workflow_id = body.resolved_id()
|
||||
if not workflow_id:
|
||||
raise HTTPException(status_code=400, detail="workflow_id required")
|
||||
store = _get_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
definition = await _load_owned_definition(request, store, workflow_id, user_id)
|
||||
versions = await store.list_versions(workflow_id)
|
||||
return _to_coze_canvas(definition, is_published=bool(versions))
|
||||
|
||||
|
||||
@router.post("/save")
|
||||
async def coze_save(request: Request, body: CozeSaveRequest) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
workflow_id = body.resolved_id()
|
||||
if not workflow_id:
|
||||
raise HTTPException(status_code=400, detail="workflow_id required")
|
||||
canvas_raw = body.resolved_canvas()
|
||||
if canvas_raw is None or canvas_raw == "" or canvas_raw == {}:
|
||||
raise HTTPException(status_code=400, detail="schema required")
|
||||
canvas_text = _parse_canvas(canvas_raw)
|
||||
store = _get_store(request)
|
||||
user_id = await get_current_user(request)
|
||||
definition = await _load_owned_definition(request, store, workflow_id, user_id)
|
||||
expected = body.resolved_revision()
|
||||
if expected is None:
|
||||
expected = int(definition.get("draft_revision") or 0)
|
||||
try:
|
||||
await store.save_studio_draft(
|
||||
workflow_id,
|
||||
expected_revision=expected,
|
||||
# The execution graph is owned by the studio facade; the Coze save
|
||||
# only persists the canvas, so the current graph passes through.
|
||||
graph=definition.get("draft_graph") or {},
|
||||
canvas_schema=canvas_text,
|
||||
updated_by=user_id,
|
||||
)
|
||||
except WorkflowDraftConflictError as exc:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"currentRevision": exc.current_revision,
|
||||
},
|
||||
) from exc
|
||||
except WorkflowNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="工作流不存在") from exc
|
||||
return {"code": 0, "msg": "success", "data": {}}
|
||||
|
||||
|
||||
@router.post("/node_template_list")
|
||||
async def coze_node_template_list(request: Request) -> dict[str, Any]:
|
||||
_require_enabled()
|
||||
await get_current_user(request)
|
||||
# Coze playground envelope: data.template_list / cate_list (§ workflow-playground-context).
|
||||
return {
|
||||
"code": 0,
|
||||
"msg": "success",
|
||||
"data": {
|
||||
"template_list": _COZE_NODE_TEMPLATES,
|
||||
"cate_list": _COZE_NODE_CATEGORIES,
|
||||
"plugin_api_list": [],
|
||||
"plugin_category_list": [],
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Runs a DeerFlow agent for one workflow ``agent`` / ``skill`` node.
|
||||
|
||||
This is the single place where the workflow runtime touches LangGraph. We build
|
||||
the lead agent with :func:`make_lead_agent` (so the node inherits the same
|
||||
middleware chain, SOUL, skills and model resolution as a normal chat turn) and
|
||||
stream ``stream_mode=["values", "messages"]`` so the node can forward token
|
||||
deltas into the workflow event log.
|
||||
|
||||
Each node gets its own thread id (``wf-{run_id}-{node_id}``) unless the node
|
||||
asks for ``shared_run_thread``, which keeps the whole run in one conversation.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _extract_text(content: Any) -> str:
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict) and block.get("type") == "text":
|
||||
parts.append(str(block.get("text") or ""))
|
||||
return "".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def _thread_id(run_id: str, node_id: str, thread_mode: str) -> str:
|
||||
if thread_mode == "shared_run_thread":
|
||||
return f"wf-{run_id}"
|
||||
return f"wf-{run_id}-{node_id}"
|
||||
|
||||
|
||||
class WorkflowAgentRunner:
|
||||
def __init__(self, app: Any) -> None:
|
||||
self._app = app
|
||||
|
||||
async def run_agent(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
node_id: str,
|
||||
owner_id: str,
|
||||
agent_id: str,
|
||||
prompt: str,
|
||||
skill_names: list[str] | None = None,
|
||||
thread_mode: str = "isolated_per_node",
|
||||
cancel: Any = None,
|
||||
on_delta: Callable[[str], Awaitable[None]] | None = None,
|
||||
on_tool: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
|
||||
model_name: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
from deerflow.agents.lead_agent.agent import make_lead_agent
|
||||
|
||||
thread_id = _thread_id(run_id, node_id, thread_mode)
|
||||
configurable: dict[str, Any] = {
|
||||
"thread_id": thread_id,
|
||||
"agent_id": agent_id,
|
||||
"agent_name": agent_id,
|
||||
"user_id": owner_id,
|
||||
"model_name": model_name or "",
|
||||
# Workflow nodes are non-interactive: no plan mode, no clarification.
|
||||
"is_plan_mode": False,
|
||||
"workflow_run_id": run_id,
|
||||
"workflow_node_id": node_id,
|
||||
}
|
||||
if skill_names:
|
||||
configurable["skill_names"] = list(skill_names)
|
||||
config = {"configurable": configurable, "recursion_limit": 60}
|
||||
|
||||
try:
|
||||
agent = make_lead_agent(config)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("failed to build agent %s for workflow node %s", agent_id, node_id, exc_info=True)
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"引用的智能体不存在或配置无效",
|
||||
node_id=node_id,
|
||||
details={"agentId": agent_id},
|
||||
) from exc
|
||||
|
||||
checkpointer = getattr(self._app.state, "checkpointer", None)
|
||||
if checkpointer is not None:
|
||||
agent.checkpointer = checkpointer
|
||||
store = getattr(self._app.state, "store", None)
|
||||
if store is not None:
|
||||
agent.store = store
|
||||
|
||||
seen_tool_calls: set[str] = set()
|
||||
finished_tool_calls: set[str] = set()
|
||||
tool_names: dict[str, str] = {}
|
||||
final_state: dict[str, Any] | None = None
|
||||
streamed: dict[str, list[str]] = {}
|
||||
last_message_id = ""
|
||||
|
||||
async for mode, chunk in agent.astream(
|
||||
{"messages": [{"role": "user", "content": prompt}]},
|
||||
config=config,
|
||||
stream_mode=["values", "messages"],
|
||||
):
|
||||
if cancel is not None and getattr(cancel, "cancelled", False):
|
||||
raise WorkflowError("WORKFLOW_CANCELLED", "运行已被取消", node_id=node_id)
|
||||
if mode == "values":
|
||||
if isinstance(chunk, dict):
|
||||
final_state = chunk
|
||||
continue
|
||||
if mode != "messages":
|
||||
continue
|
||||
message = chunk[0] if isinstance(chunk, tuple) and chunk else chunk
|
||||
if isinstance(message, ToolMessage):
|
||||
# Tool result for an earlier call: close its step in the event log.
|
||||
call_id = str(getattr(message, "tool_call_id", "") or "")
|
||||
if call_id and call_id not in finished_tool_calls:
|
||||
finished_tool_calls.add(call_id)
|
||||
if on_tool is not None:
|
||||
content = getattr(message, "content", "")
|
||||
if not isinstance(content, str):
|
||||
content = str(content)
|
||||
await on_tool(
|
||||
{
|
||||
"phase": "finished",
|
||||
"id": call_id,
|
||||
"name": tool_names.get(call_id) or getattr(message, "name", "") or "",
|
||||
"ok": getattr(message, "status", None) != "error",
|
||||
"summary": content[:200],
|
||||
}
|
||||
)
|
||||
continue
|
||||
if not isinstance(message, AIMessage):
|
||||
continue
|
||||
text = _extract_text(getattr(message, "content", ""))
|
||||
if text:
|
||||
message_id = str(getattr(message, "id", "") or "")
|
||||
streamed.setdefault(message_id, []).append(text)
|
||||
last_message_id = message_id
|
||||
if on_delta is not None:
|
||||
await on_delta(text)
|
||||
for call in getattr(message, "tool_calls", None) or []:
|
||||
call_id = str(call.get("id") or "")
|
||||
if not call_id or call_id in seen_tool_calls:
|
||||
continue
|
||||
seen_tool_calls.add(call_id)
|
||||
tool_names[call_id] = str(call.get("name") or "")
|
||||
if on_tool is not None:
|
||||
await on_tool(
|
||||
{
|
||||
"phase": "started",
|
||||
"id": call_id,
|
||||
"name": tool_names[call_id],
|
||||
"args": call.get("args"),
|
||||
}
|
||||
)
|
||||
|
||||
final_text = "".join(streamed.get(last_message_id, ())) or self._final_text(final_state)
|
||||
if not final_text.strip():
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_AGENT_FAILED",
|
||||
"智能体没有返回任何内容",
|
||||
retryable=True,
|
||||
node_id=node_id,
|
||||
)
|
||||
return {
|
||||
"text": final_text,
|
||||
"thread_id": thread_id,
|
||||
"messages": [],
|
||||
"artifacts": self._artifacts(final_state),
|
||||
"usage": {},
|
||||
}
|
||||
|
||||
def _final_text(self, state: dict[str, Any] | None) -> str:
|
||||
for message in reversed((state or {}).get("messages") or []):
|
||||
if isinstance(message, AIMessage):
|
||||
text = _extract_text(getattr(message, "content", "")).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
def _artifacts(self, state: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
raw = (state or {}).get("artifacts")
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
for item in raw:
|
||||
if isinstance(item, dict) and item.get("name"):
|
||||
out.append(
|
||||
{
|
||||
"name": str(item.get("name")),
|
||||
"path": str(item.get("path") or ""),
|
||||
"mime_type": str(item.get("mime_type") or item.get("mimeType") or ""),
|
||||
"size_bytes": item.get("size_bytes") or item.get("sizeBytes"),
|
||||
"preview": item.get("preview"),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
__all__ = ["WorkflowAgentRunner"]
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Structured audit lines for Workflow Studio mutating APIs.
|
||||
|
||||
These are operator logs, not a separate audit table: every line is a single
|
||||
JSON object with a stable ``action`` plus the ids needed to reconstruct who
|
||||
did what. Secrets never go in.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger("deerflow.workflows.audit")
|
||||
|
||||
|
||||
def audit_workflow(action: str, *, user_id: str, **fields: Any) -> None:
|
||||
payload = {"action": action, "userId": user_id, **fields}
|
||||
logger.info("workflow.audit %s", json.dumps(payload, ensure_ascii=False, default=str))
|
||||
|
||||
|
||||
__all__ = ["audit_workflow"]
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Lease-based dispatcher for workflow runs.
|
||||
|
||||
A single background loop scans for claimable runs (``queued``, or ``running``
|
||||
with an expired lease) and hands each successfully claimed run to the executor.
|
||||
Because the claim is an atomic conditional UPDATE, several workers can run this
|
||||
loop concurrently and a run that dies with its worker is picked up by whoever
|
||||
notices the expired lease first — that is the restart-recovery story.
|
||||
|
||||
``nudge()`` is called by the start/resume routes so a fresh run does not wait
|
||||
for the next poll tick.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from app.gateway.workflow_executor import LeaseParams, WorkflowRunExecutor
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
from deerflow.workflows.runtime import PersistingWorkflowEventSink
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_POLL_INTERVAL_SECONDS = 2.0
|
||||
_CLAIM_BATCH = 5
|
||||
|
||||
|
||||
def _worker_id() -> str:
|
||||
return f"{socket.gethostname()[:24]}-{os.getpid()}-{uuid.uuid4().hex[:6]}"
|
||||
|
||||
|
||||
class WorkflowRunDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
store: WorkflowRunStore,
|
||||
executor: WorkflowRunExecutor,
|
||||
*,
|
||||
event_store: WorkflowEventStore | None = None,
|
||||
live_publisher: Any = None,
|
||||
config: WorkflowConfig | None = None,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._executor = executor
|
||||
self._events = event_store
|
||||
self._publish = live_publisher
|
||||
self._config = config or WorkflowConfig()
|
||||
self._worker_id = _worker_id()
|
||||
self._wake = asyncio.Event()
|
||||
self._task: asyncio.Task | None = None
|
||||
self._stopping = False
|
||||
|
||||
@property
|
||||
def worker_id(self) -> str:
|
||||
return self._worker_id
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is not None and not self._task.done():
|
||||
return
|
||||
self._stopping = False
|
||||
self._task = asyncio.create_task(self._loop(), name="workflow-run-dispatcher")
|
||||
logger.info("workflow dispatcher started (worker=%s)", self._worker_id)
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._stopping = True
|
||||
self._wake.set()
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
self._task = None
|
||||
await self._executor.drain()
|
||||
|
||||
def nudge(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
async def _loop(self) -> None:
|
||||
await self._safe_dispatch()
|
||||
while not self._stopping:
|
||||
try:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=_POLL_INTERVAL_SECONDS)
|
||||
except TimeoutError:
|
||||
pass
|
||||
self._wake.clear()
|
||||
if self._stopping:
|
||||
return
|
||||
await self._safe_dispatch()
|
||||
|
||||
async def _safe_dispatch(self) -> None:
|
||||
try:
|
||||
await self.dispatch_once()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - the loop must survive a bad round
|
||||
logger.exception("workflow dispatch round failed")
|
||||
|
||||
async def dispatch_once(self) -> int:
|
||||
now = datetime.now(UTC)
|
||||
await self._reap_cancel_requested()
|
||||
candidates = await self._store.list_claimable(now=now, limit=_CLAIM_BATCH)
|
||||
if not candidates:
|
||||
return 0
|
||||
ttl = max(30, self._config.lease_ttl_seconds)
|
||||
started = 0
|
||||
for run_id in candidates:
|
||||
row = await self._store.claim_run(
|
||||
run_id,
|
||||
lease_owner=self._worker_id,
|
||||
lease_until=now + timedelta(seconds=ttl),
|
||||
)
|
||||
if row is None:
|
||||
continue
|
||||
if self._executor.start(LeaseParams(run_id=run_id, lease_owner=self._worker_id, attempt=int(row.get("attempt") or 1))):
|
||||
started += 1
|
||||
return started
|
||||
|
||||
async def _reap_cancel_requested(self) -> None:
|
||||
"""Finalize cancels whose worker is gone (no live lease to notice them)."""
|
||||
rows = await self._store.list_runs(status="cancel_requested", limit=20)
|
||||
now = datetime.now(UTC)
|
||||
for row in rows:
|
||||
lease_until = row.get("lease_until")
|
||||
if lease_until:
|
||||
try:
|
||||
if datetime.fromisoformat(str(lease_until)) > now:
|
||||
continue # its worker is alive and will handle the cancel
|
||||
except ValueError:
|
||||
pass
|
||||
finalized = await self._store.finalize_cancel(row["id"])
|
||||
if finalized is not None and self._events is not None:
|
||||
# Nobody owns this run, so the dispatcher owes the stream its
|
||||
# terminal frame — otherwise open SSE connections hang.
|
||||
sink = PersistingWorkflowEventSink(
|
||||
self._events,
|
||||
run_id=row["id"],
|
||||
workflow_id=row["workflow_id"],
|
||||
version_id=row["workflow_version_id"],
|
||||
publish=self._publish,
|
||||
)
|
||||
await sink.emit("run.cancelled", data={"reapedBy": self._worker_id})
|
||||
|
||||
|
||||
__all__ = ["WorkflowRunDispatcher"]
|
||||
@@ -0,0 +1,444 @@
|
||||
"""Executes leased workflow runs and owns every terminal state transition.
|
||||
|
||||
Lifecycle for one lease:
|
||||
|
||||
1. resolve the published version → immutable graph snapshot,
|
||||
2. replay completed node results from ``workflow_node_runs`` (so a re-lease or a
|
||||
resume never re-runs finished work),
|
||||
3. drive :class:`WorkflowEngine` under a run-level timeout with a heartbeat
|
||||
renewing the lease and a watcher polling for ``cancel_requested``,
|
||||
4. write exactly one terminal outcome: ``completed`` / ``failed`` /
|
||||
``cancelled`` / ``awaiting_input``.
|
||||
|
||||
Every exit path emits a matching terminal event, because the SSE endpoint uses
|
||||
those events to decide when to close a stream.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.nodes import build_default_registry
|
||||
from deerflow.workflows.runtime import (
|
||||
CancelToken,
|
||||
PersistingWorkflowEventSink,
|
||||
RunContext,
|
||||
WorkflowEngine,
|
||||
WorkflowPaused,
|
||||
WorkflowRuntimeDeps,
|
||||
)
|
||||
from deerflow.workflows.schemas import WorkflowGraph
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_HEARTBEAT_INTERVAL_SECONDS = 15
|
||||
_CANCEL_POLL_SECONDS = 2.0
|
||||
_MAX_SUBWORKFLOW_DEPTH = 3
|
||||
|
||||
|
||||
@dataclass
|
||||
class LeaseParams:
|
||||
run_id: str
|
||||
lease_owner: str
|
||||
attempt: int = 1
|
||||
|
||||
|
||||
class WorkflowRunExecutor:
|
||||
def __init__(
|
||||
self,
|
||||
app: Any,
|
||||
*,
|
||||
run_store: WorkflowRunStore,
|
||||
event_store: WorkflowEventStore,
|
||||
workflow_store: Any,
|
||||
data_source_store: Any = None,
|
||||
live_publisher: Any = None,
|
||||
config: WorkflowConfig | None = None,
|
||||
) -> None:
|
||||
self._app = app
|
||||
self._runs = run_store
|
||||
self._events = event_store
|
||||
self._workflows = workflow_store
|
||||
self._data_sources = data_source_store
|
||||
self._publish = live_publisher
|
||||
self._config = config or WorkflowConfig()
|
||||
self._registry = build_default_registry(self._config)
|
||||
self._tasks: dict[str, asyncio.Task] = {}
|
||||
|
||||
# ── public API ──────────────────────────────────────────────────────
|
||||
def start(self, params: LeaseParams) -> bool:
|
||||
"""Launch a background task for a leased run. Idempotent per run id."""
|
||||
existing = self._tasks.get(params.run_id)
|
||||
if existing is not None and not existing.done():
|
||||
return False
|
||||
task = asyncio.create_task(self._guarded(params), name=f"workflow-run-{params.run_id}")
|
||||
self._tasks[params.run_id] = task
|
||||
task.add_done_callback(lambda _t, rid=params.run_id: self._tasks.pop(rid, None))
|
||||
return True
|
||||
|
||||
async def drain(self, timeout: float = 5.0) -> None:
|
||||
tasks = [t for t in self._tasks.values() if not t.done()]
|
||||
if not tasks:
|
||||
return
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await asyncio.wait(tasks, timeout=timeout)
|
||||
|
||||
# ── run driver ──────────────────────────────────────────────────────
|
||||
async def _guarded(self, params: LeaseParams) -> None:
|
||||
try:
|
||||
await self._execute_lease(params)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - never let a run kill the dispatcher
|
||||
logger.exception("workflow run %s crashed outside the engine", params.run_id)
|
||||
await self._fail(
|
||||
params,
|
||||
WorkflowError("WORKFLOW_INTERNAL", "运行发生内部错误,请查看服务端日志"),
|
||||
)
|
||||
|
||||
async def _execute_lease(self, params: LeaseParams) -> None:
|
||||
run = await self._runs.get_run(params.run_id)
|
||||
if run is None:
|
||||
return
|
||||
sink = self._sink(run)
|
||||
graph = await self._load_graph(run)
|
||||
if graph is None:
|
||||
await self._fail(
|
||||
params,
|
||||
WorkflowError("WORKFLOW_VERSION_NOT_FOUND", "运行引用的工作流版本不存在"),
|
||||
sink=sink,
|
||||
)
|
||||
return
|
||||
|
||||
cancel = CancelToken()
|
||||
resume_payload = (run.get("context") or {}).get("resume_payload")
|
||||
preloaded, loop_state = await self._replay_state(params.run_id)
|
||||
|
||||
ctx = RunContext(
|
||||
run_id=run["id"],
|
||||
workflow_id=run["workflow_id"],
|
||||
version_id=run["workflow_version_id"],
|
||||
owner_id=run["owner_id"],
|
||||
graph=graph,
|
||||
inputs=run.get("input") or {},
|
||||
deps=self._deps(depth=int((run.get("context") or {}).get("depth") or 0)),
|
||||
cancel=cancel,
|
||||
emit=sink.emit,
|
||||
resume_payload=resume_payload if isinstance(resume_payload, dict) else None,
|
||||
)
|
||||
|
||||
await sink.emit(
|
||||
"run.started" if not preloaded else "run.resumed",
|
||||
data={"attempt": params.attempt, "replayedNodes": sorted(preloaded)},
|
||||
)
|
||||
|
||||
engine = WorkflowEngine(self._registry, run_store=self._runs)
|
||||
main = asyncio.create_task(
|
||||
asyncio.wait_for(
|
||||
engine.run(ctx, preloaded=preloaded, loop_state=loop_state, attempt=params.attempt),
|
||||
timeout=min(graph.settings.run_timeout_seconds, self._config.run_timeout_seconds),
|
||||
)
|
||||
)
|
||||
heartbeat = asyncio.create_task(self._heartbeat(params, main))
|
||||
watcher = asyncio.create_task(self._watch_cancel(params, cancel, main))
|
||||
try:
|
||||
result = await main
|
||||
except WorkflowPaused as paused:
|
||||
await self._pause(params, paused, ctx, sink)
|
||||
return
|
||||
except TimeoutError:
|
||||
await self._fail(
|
||||
params,
|
||||
WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"运行超时({graph.settings.run_timeout_seconds}s)",
|
||||
details={"timeoutSeconds": graph.settings.run_timeout_seconds},
|
||||
),
|
||||
sink=sink,
|
||||
)
|
||||
return
|
||||
except WorkflowError as exc:
|
||||
if exc.code == "WORKFLOW_CANCELLED" or cancel.cancelled:
|
||||
await self._cancel(params, sink)
|
||||
else:
|
||||
await self._fail(params, exc, sink=sink)
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
# Lease lost or process shutting down: leave the run claimable again.
|
||||
await self._runs.update_run(params.run_id, lease_owner=params.lease_owner, status="queued")
|
||||
raise
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
watcher.cancel()
|
||||
for task in (heartbeat, watcher):
|
||||
try:
|
||||
await task
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
|
||||
await self._complete(params, result, sink)
|
||||
|
||||
# ── terminal transitions ────────────────────────────────────────────
|
||||
async def _complete(self, params: LeaseParams, result: Any, sink: Any) -> None:
|
||||
await self._runs.update_run(
|
||||
params.run_id,
|
||||
lease_owner=params.lease_owner,
|
||||
status="completed",
|
||||
output_json=result.output,
|
||||
context_json={"loop_state": result.loop_state, "steps": result.steps},
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
await sink.emit("run.completed", data={"output": result.output, "steps": result.steps})
|
||||
|
||||
async def _fail(self, params: LeaseParams, error: WorkflowError, *, sink: Any = None) -> None:
|
||||
run = await self._runs.get_run(params.run_id)
|
||||
if run is None:
|
||||
return
|
||||
body = error.to_body().model_dump(by_alias=True)
|
||||
retry = error.retryable and int(run.get("attempt") or 1) < int(run.get("max_attempts") or 1)
|
||||
sink = sink or self._sink(run)
|
||||
if retry:
|
||||
await self._runs.update_run(
|
||||
params.run_id,
|
||||
lease_owner=params.lease_owner,
|
||||
status="queued",
|
||||
error_json=body,
|
||||
)
|
||||
await sink.emit("run.queued", data={"retryAfterError": body, "attempt": run.get("attempt")})
|
||||
return
|
||||
await self._runs.update_run(
|
||||
params.run_id,
|
||||
lease_owner=params.lease_owner,
|
||||
status="failed",
|
||||
error_json=body,
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
await sink.emit("run.failed", data={"error": body})
|
||||
|
||||
async def _cancel(self, params: LeaseParams, sink: Any) -> None:
|
||||
await self._runs.request_cancel(params.run_id)
|
||||
finalized = await self._runs.finalize_cancel(params.run_id)
|
||||
if finalized is None:
|
||||
await self._runs.update_run(params.run_id, lease_owner=params.lease_owner, status="cancelled")
|
||||
await sink.emit("run.cancelled", data={})
|
||||
|
||||
async def _pause(self, params: LeaseParams, paused: WorkflowPaused, ctx: RunContext, sink: Any) -> None:
|
||||
token = secrets.token_urlsafe(24)
|
||||
# One conditional update writes status + token + pending descriptor +
|
||||
# run context together; a separate follow-up context write could clobber
|
||||
# a resume that lands in between.
|
||||
updated = await self._runs.set_awaiting_input(
|
||||
params.run_id,
|
||||
lease_owner=params.lease_owner,
|
||||
pending_input=paused.pending_input,
|
||||
resume_token=token,
|
||||
context={"loop_state": dict(ctx.loop_state), "resume_payload": None},
|
||||
)
|
||||
if updated is None:
|
||||
# Lost the lease (cancel or steal) — do not fabricate a pause.
|
||||
logger.info("workflow run %s could not enter awaiting_input", params.run_id)
|
||||
return
|
||||
await sink.emit(
|
||||
"run.awaiting_input",
|
||||
data={
|
||||
# Flattened wire contract: the full descriptor keeps living in
|
||||
# the run row (pending_input), the event exposes exactly what a
|
||||
# client needs to render the intervention card.
|
||||
"resumeToken": token,
|
||||
"prompt": paused.pending_input.get("prompt") or "",
|
||||
"formSchema": paused.pending_input.get("formSchema") or {},
|
||||
"actions": paused.pending_input.get("actions") or ["submit"],
|
||||
},
|
||||
node_id=paused.node_id,
|
||||
)
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────
|
||||
def _sink(self, run: dict[str, Any]) -> PersistingWorkflowEventSink:
|
||||
return PersistingWorkflowEventSink(
|
||||
self._events,
|
||||
run_id=run["id"],
|
||||
workflow_id=run["workflow_id"],
|
||||
version_id=run["workflow_version_id"],
|
||||
publish=self._publish,
|
||||
)
|
||||
|
||||
async def _load_graph(self, run: dict[str, Any]) -> WorkflowGraph | None:
|
||||
version = await self._workflows.get_version(run["workflow_version_id"])
|
||||
if version is None or version.get("workflow_id") != run["workflow_id"]:
|
||||
return None
|
||||
try:
|
||||
return WorkflowGraph.model_validate(version.get("graph") or {})
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("workflow version %s has an invalid graph", run["workflow_version_id"], exc_info=True)
|
||||
return None
|
||||
|
||||
async def _replay_state(self, run_id: str) -> tuple[dict[str, Any], dict[str, int]]:
|
||||
run = await self._runs.get_run(run_id)
|
||||
loop_state = {}
|
||||
raw_loop = (run or {}).get("context", {}).get("loop_state")
|
||||
if isinstance(raw_loop, dict):
|
||||
loop_state = {str(k): int(v) for k, v in raw_loop.items() if isinstance(v, int)}
|
||||
completed: dict[str, Any] = {}
|
||||
for row in await self._runs.list_node_runs(run_id, current_only=True):
|
||||
if row.get("status") != "completed":
|
||||
continue
|
||||
output = row.get("output")
|
||||
if isinstance(output, dict):
|
||||
completed[str(row["node_id"])] = output
|
||||
return completed, loop_state
|
||||
|
||||
def _deps(self, *, depth: int = 0) -> WorkflowRuntimeDeps:
|
||||
from app.gateway.workflow_agent_runner import WorkflowAgentRunner
|
||||
|
||||
agent_runner = WorkflowAgentRunner(self._app)
|
||||
|
||||
async def save_artifact(**kwargs: Any) -> dict[str, Any]:
|
||||
return await self._runs.add_artifact(kwargs)
|
||||
|
||||
async def resolve_data_source(source_id: str) -> dict[str, Any] | None:
|
||||
if self._data_sources is None:
|
||||
return None
|
||||
return await self._data_sources.resolve_dsn(source_id)
|
||||
|
||||
async def resolve_credential(ref: str) -> dict[str, Any] | None:
|
||||
"""HTTP credentials are data-source rows of kind ``http`` whose
|
||||
secret is a JSON blob of headers to inject."""
|
||||
if self._data_sources is None:
|
||||
return None
|
||||
resolved = await self._data_sources.resolve_dsn(ref)
|
||||
if not resolved:
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(resolved.get("dsn") or "{}")
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
headers = payload.get("headers") if isinstance(payload, dict) else None
|
||||
return {"headers": headers} if isinstance(headers, dict) else None
|
||||
|
||||
async def run_subworkflow(**kwargs: Any) -> dict[str, Any]:
|
||||
if depth + 1 > _MAX_SUBWORKFLOW_DEPTH:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_LIMIT_EXCEEDED",
|
||||
f"子工作流嵌套超过 {_MAX_SUBWORKFLOW_DEPTH} 层",
|
||||
details={"depth": depth + 1},
|
||||
)
|
||||
return await self._run_child(depth=depth + 1, **kwargs)
|
||||
|
||||
return WorkflowRuntimeDeps(
|
||||
run_agent=agent_runner.run_agent,
|
||||
save_artifact=save_artifact,
|
||||
resolve_data_source=resolve_data_source,
|
||||
resolve_credential=resolve_credential,
|
||||
run_subworkflow=run_subworkflow,
|
||||
)
|
||||
|
||||
async def _run_child(
|
||||
self,
|
||||
*,
|
||||
depth: int,
|
||||
parent_run_id: str,
|
||||
node_id: str,
|
||||
owner_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
inputs: dict[str, Any],
|
||||
cancel: CancelToken | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run a child workflow inline: same process, its own run row and events."""
|
||||
child, _ = await self._runs.create_run(
|
||||
{
|
||||
"workflow_id": workflow_id,
|
||||
"workflow_version_id": version_id,
|
||||
"owner_id": owner_id,
|
||||
"input": inputs,
|
||||
"status": "running",
|
||||
"context": {
|
||||
"depth": depth,
|
||||
"parent_run_id": parent_run_id,
|
||||
"parent_node_id": node_id,
|
||||
},
|
||||
}
|
||||
)
|
||||
graph = await self._load_graph(child)
|
||||
if graph is None:
|
||||
raise WorkflowError("WORKFLOW_VERSION_NOT_FOUND", "子工作流版本不存在", node_id=node_id)
|
||||
sink = self._sink(child)
|
||||
ctx = RunContext(
|
||||
run_id=child["id"],
|
||||
workflow_id=workflow_id,
|
||||
version_id=version_id,
|
||||
owner_id=owner_id,
|
||||
graph=graph,
|
||||
inputs=inputs,
|
||||
deps=self._deps(depth=depth),
|
||||
cancel=cancel or CancelToken(),
|
||||
emit=sink.emit,
|
||||
)
|
||||
await sink.emit("run.started", data={"parentRunId": parent_run_id, "depth": depth})
|
||||
engine = WorkflowEngine(self._registry, run_store=self._runs)
|
||||
try:
|
||||
result = await engine.run(ctx)
|
||||
except WorkflowPaused as exc:
|
||||
await self._runs.update_run(child["id"], status="failed")
|
||||
await sink.emit(
|
||||
"run.failed",
|
||||
data={
|
||||
"error": {
|
||||
"code": "WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
"message": "子工作流不支持人工节点",
|
||||
}
|
||||
},
|
||||
)
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
"子工作流不支持人工输入节点",
|
||||
node_id=node_id,
|
||||
details={"childRunId": child["id"], "pausedAt": exc.node_id},
|
||||
) from exc
|
||||
except WorkflowError as exc:
|
||||
await self._runs.update_run(child["id"], status="failed", error_json=exc.to_body().model_dump(by_alias=True))
|
||||
await sink.emit("run.failed", data={"error": exc.to_body().model_dump(by_alias=True)})
|
||||
raise
|
||||
await self._runs.update_run(child["id"], status="completed", output_json=result.output)
|
||||
await sink.emit("run.completed", data={"output": result.output})
|
||||
return {"run_id": child["id"], "status": "completed", "output": result.output}
|
||||
|
||||
async def _heartbeat(self, params: LeaseParams, main: asyncio.Task) -> None:
|
||||
ttl = max(30, self._config.lease_ttl_seconds)
|
||||
while not main.done():
|
||||
await asyncio.sleep(_HEARTBEAT_INTERVAL_SECONDS)
|
||||
if main.done():
|
||||
return
|
||||
ok = await self._runs.renew_lease(
|
||||
params.run_id,
|
||||
lease_owner=params.lease_owner,
|
||||
lease_until=datetime.now(UTC) + timedelta(seconds=ttl),
|
||||
)
|
||||
if not ok:
|
||||
logger.warning("workflow run %s lost its lease; aborting", params.run_id)
|
||||
main.cancel()
|
||||
return
|
||||
|
||||
async def _watch_cancel(self, params: LeaseParams, cancel: CancelToken, main: asyncio.Task) -> None:
|
||||
while not main.done():
|
||||
await asyncio.sleep(_CANCEL_POLL_SECONDS)
|
||||
run = await self._runs.get_run(params.run_id)
|
||||
if run is None:
|
||||
return
|
||||
if run.get("status") in ("cancel_requested", "cancelled"):
|
||||
cancel.cancel()
|
||||
return
|
||||
|
||||
|
||||
__all__ = ["LeaseParams", "WorkflowRunExecutor"]
|
||||
@@ -0,0 +1,59 @@
|
||||
"""In-process fan-out of workflow events to connected SSE clients.
|
||||
|
||||
The durable log in ``workflow_run_events`` is the source of truth; this hub only
|
||||
shortens latency for the tail. Slow consumers lose the *oldest* queued frame
|
||||
rather than blocking the executor, and the SSE endpoint reconciles any gap from
|
||||
the database using the ``seq`` cursor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WorkflowLiveHub:
|
||||
def __init__(self, *, queue_size: int = 512) -> None:
|
||||
self._subscribers: dict[str, set[asyncio.Queue[dict[str, Any]]]] = defaultdict(set)
|
||||
self._queue_size = queue_size
|
||||
|
||||
def subscribe(self, run_id: str) -> asyncio.Queue[dict[str, Any]]:
|
||||
queue: asyncio.Queue[dict[str, Any]] = asyncio.Queue(maxsize=self._queue_size)
|
||||
self._subscribers[run_id].add(queue)
|
||||
return queue
|
||||
|
||||
def unsubscribe(self, run_id: str, queue: asyncio.Queue[dict[str, Any]]) -> None:
|
||||
subscribers = self._subscribers.get(run_id)
|
||||
if not subscribers:
|
||||
return
|
||||
subscribers.discard(queue)
|
||||
if not subscribers:
|
||||
self._subscribers.pop(run_id, None)
|
||||
|
||||
def subscriber_count(self, run_id: str) -> int:
|
||||
return len(self._subscribers.get(run_id) or ())
|
||||
|
||||
async def publish(self, event: Mapping[str, Any]) -> None:
|
||||
run_id = str(event.get("runId") or "")
|
||||
subscribers = self._subscribers.get(run_id)
|
||||
if not run_id or not subscribers:
|
||||
return
|
||||
payload = dict(event)
|
||||
for queue in list(subscribers):
|
||||
if queue.full():
|
||||
try:
|
||||
queue.get_nowait()
|
||||
except asyncio.QueueEmpty: # pragma: no cover - race with a reader
|
||||
pass
|
||||
try:
|
||||
queue.put_nowait(payload)
|
||||
except asyncio.QueueFull: # pragma: no cover - race with a reader
|
||||
logger.debug("workflow live queue still full for run %s", run_id)
|
||||
|
||||
|
||||
__all__ = ["WorkflowLiveHub"]
|
||||
@@ -0,0 +1,303 @@
|
||||
"""App-layer publish-time resource checks for workflow graphs.
|
||||
|
||||
The harness validator stays store-agnostic via its ``resource_checker`` hook;
|
||||
this module is where the Gateway binds that hook to the real agent / skill /
|
||||
data-source / published-version stores. It pre-fetches the catalogs the caller
|
||||
is allowed to see, then exposes a **sync** checker so it can be passed straight
|
||||
into ``validate_workflow_graph``.
|
||||
|
||||
Never imports from ``packages/harness`` internals beyond the public workflow
|
||||
API, and the harness never imports this module.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.security.sql_policy import assert_read_only, assert_tables_allowed
|
||||
|
||||
# Header-ish keys that must never be configured as literal values on the canvas:
|
||||
# credentials belong in an http data source referenced by ``credentialRef``.
|
||||
_LITERAL_SECRET_KEYS = frozenset(
|
||||
{
|
||||
"authorization",
|
||||
"proxy-authorization",
|
||||
"cookie",
|
||||
"x-api-key",
|
||||
"apikey",
|
||||
"api_key",
|
||||
"secret",
|
||||
"client_secret",
|
||||
"password",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
}
|
||||
)
|
||||
|
||||
_HTTP_METHODS = frozenset({"GET", "POST", "PUT", "PATCH", "DELETE"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResourceCatalog:
|
||||
"""Everything the checker is allowed to reference, prefetched."""
|
||||
|
||||
agent_ids: set[str] = field(default_factory=set)
|
||||
# agent_id -> cached skill whitelist from the agent store (empty = no
|
||||
# restriction recorded).
|
||||
agent_skills: dict[str, set[str]] = field(default_factory=dict)
|
||||
enabled_skills: set[str] = field(default_factory=set)
|
||||
# data source id -> public row (kind / enabled / allowed_* / max_rows).
|
||||
data_sources: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
# workflow_id -> set of published version ids.
|
||||
published_subworkflows: dict[str, set[str]] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _issue(message: str, field_path: str) -> dict[str, Any]:
|
||||
return {"message": message, "field": field_path}
|
||||
|
||||
|
||||
async def build_resource_catalog(request: Request, *, user_id: str) -> ResourceCatalog:
|
||||
catalog = ResourceCatalog()
|
||||
|
||||
agent_store = getattr(request.app.state, "agent_store", None)
|
||||
if agent_store is not None:
|
||||
try:
|
||||
if hasattr(agent_store, "list_visible"):
|
||||
rows = await agent_store.list_visible(user_id)
|
||||
elif hasattr(agent_store, "list_all"):
|
||||
rows = await agent_store.list_all()
|
||||
else:
|
||||
rows = []
|
||||
catalog.agent_ids = {
|
||||
str(row.get("id") or row.get("agent_id") or "") for row in rows or [] if row.get("id") or row.get("agent_id")
|
||||
}
|
||||
if hasattr(agent_store, "get_extras_for"):
|
||||
extras = await agent_store.get_extras_for([aid for aid in catalog.agent_ids if aid])
|
||||
for agent_id, extra in (extras or {}).items():
|
||||
skills = (extra or {}).get("skills") or []
|
||||
if skills:
|
||||
catalog.agent_skills[str(agent_id)] = {str(s) for s in skills}
|
||||
except Exception: # noqa: BLE001 - publish checks degrade to strict-by-empty
|
||||
catalog.agent_ids = set()
|
||||
|
||||
try:
|
||||
from deerflow.skills import get_or_new_skill_storage
|
||||
|
||||
catalog.enabled_skills = {
|
||||
str(getattr(skill, "name"))
|
||||
for skill in get_or_new_skill_storage().load_skills(enabled_only=True) or []
|
||||
if getattr(skill, "name", None)
|
||||
}
|
||||
except Exception: # noqa: BLE001
|
||||
catalog.enabled_skills = set()
|
||||
|
||||
ds_store = getattr(request.app.state, "workflow_data_source_store", None)
|
||||
if ds_store is not None:
|
||||
try:
|
||||
for row in await ds_store.list_sources(owner_id=user_id, include_shared=True) or []:
|
||||
catalog.data_sources[str(row.get("id"))] = row
|
||||
except Exception: # noqa: BLE001
|
||||
catalog.data_sources = {}
|
||||
|
||||
wf_store = getattr(request.app.state, "workflow_store", None)
|
||||
if wf_store is not None:
|
||||
try:
|
||||
for row in await wf_store.list_published_summaries() or []:
|
||||
versions = catalog.published_subworkflows.setdefault(str(row.get("workflow_id")), set())
|
||||
versions.add(str(row.get("version_id")))
|
||||
except Exception: # noqa: BLE001
|
||||
catalog.published_subworkflows = {}
|
||||
|
||||
return catalog
|
||||
|
||||
|
||||
def _check_agent(config: dict[str, Any], catalog: ResourceCatalog) -> dict[str, Any] | None:
|
||||
agent_id = str(config.get("agentId") or config.get("agent_id") or "")
|
||||
if not agent_id:
|
||||
return _issue("未选择智能体", "config.agentId")
|
||||
if agent_id not in catalog.agent_ids:
|
||||
return _issue("未选择可访问的智能体", "config.agentId")
|
||||
if not str(config.get("promptTemplate") or config.get("prompt_template") or "").strip():
|
||||
return _issue("智能体节点的提示词不能为空", "config.promptTemplate")
|
||||
mode = str(config.get("responseMode") or config.get("response_mode") or "text")
|
||||
schema = config.get("responseSchema") or config.get("response_schema")
|
||||
if mode == "json":
|
||||
if not isinstance(schema, dict) or not schema:
|
||||
return _issue("JSON 输出模式必须配置 responseSchema 对象", "config.responseSchema")
|
||||
if schema.get("type") not in (None, "object"):
|
||||
return _issue("responseSchema 目前仅支持 object 类型", "config.responseSchema")
|
||||
return None
|
||||
|
||||
|
||||
def _check_skill(config: dict[str, Any], catalog: ResourceCatalog) -> dict[str, Any] | None:
|
||||
mode = str(config.get("mode") or "agent_skill")
|
||||
if mode != "agent_skill":
|
||||
return _issue("callable_skill 模式暂未开放执行,请改用 agent_skill", "config.mode")
|
||||
agent_id = str(config.get("agentId") or config.get("agent_id") or "")
|
||||
if not agent_id:
|
||||
return _issue("技能节点未选择智能体", "config.agentId")
|
||||
if agent_id not in catalog.agent_ids:
|
||||
return _issue("技能节点引用的智能体不可访问", "config.agentId")
|
||||
skill_names = [str(s) for s in (config.get("skillNames") or config.get("skill_names") or [])]
|
||||
if not skill_names:
|
||||
return _issue("技能节点至少需要选择一个技能", "config.skillNames")
|
||||
for name in skill_names:
|
||||
if name not in catalog.enabled_skills:
|
||||
return _issue(f"技能不存在或未启用:{name}", "config.skillNames")
|
||||
whitelist = catalog.agent_skills.get(agent_id)
|
||||
if whitelist:
|
||||
missing = [name for name in skill_names if name not in whitelist]
|
||||
if missing:
|
||||
return _issue(f"技能不在智能体白名单内:{'、'.join(missing)}", "config.skillNames")
|
||||
return None
|
||||
|
||||
|
||||
def _check_sql(config: dict[str, Any], catalog: ResourceCatalog, cfg: WorkflowConfig) -> dict[str, Any] | None:
|
||||
source_id = str(config.get("dataSourceId") or config.get("data_source_id") or "")
|
||||
if not source_id:
|
||||
return _issue("SQL 节点未选择数据源", "config.dataSourceId")
|
||||
source = catalog.data_sources.get(source_id)
|
||||
if source is None:
|
||||
return _issue("数据源不存在或无权访问", "config.dataSourceId")
|
||||
if not source.get("enabled", True):
|
||||
return _issue("数据源已被禁用", "config.dataSourceId")
|
||||
if str(source.get("kind") or "sql") != "sql":
|
||||
return _issue("该资源不是 SQL 数据源", "config.dataSourceId")
|
||||
statement = str(config.get("statement") or config.get("queryTemplate") or config.get("query_template") or "")
|
||||
if not statement.strip():
|
||||
return _issue("SQL 节点缺少查询语句", "config.statement")
|
||||
try:
|
||||
assert_read_only(statement)
|
||||
assert_tables_allowed(statement, list(source.get("allowed_tables") or []))
|
||||
except WorkflowError as exc:
|
||||
return _issue(str(exc.message if hasattr(exc, "message") else exc), "config.statement")
|
||||
parameters = config.get("parameters")
|
||||
if parameters is not None and not isinstance(parameters, dict):
|
||||
return _issue("SQL 参数必须是对象", "config.parameters")
|
||||
max_rows = config.get("maxRows") or config.get("max_rows")
|
||||
if max_rows is not None:
|
||||
try:
|
||||
rows = int(max_rows)
|
||||
except (TypeError, ValueError):
|
||||
return _issue("maxRows 必须是整数", "config.maxRows")
|
||||
ceiling = min(int(cfg.sql.max_rows), int(source.get("max_rows") or cfg.sql.max_rows))
|
||||
if rows < 1 or rows > ceiling:
|
||||
return _issue(f"maxRows 超过上限 {ceiling}", "config.maxRows")
|
||||
return None
|
||||
|
||||
|
||||
def _check_http(config: dict[str, Any], catalog: ResourceCatalog) -> dict[str, Any] | None:
|
||||
method = str(config.get("method") or "GET").upper()
|
||||
if method not in _HTTP_METHODS:
|
||||
return _issue(f"不支持的 HTTP 方法:{method}", "config.method")
|
||||
headers = config.get("headers")
|
||||
if headers is not None:
|
||||
if not isinstance(headers, dict):
|
||||
return _issue("请求头必须是字符串到字符串的映射", "config.headers")
|
||||
for name in headers:
|
||||
if str(name).strip().lower() in _LITERAL_SECRET_KEYS:
|
||||
return _issue(f"请勿在节点上明文配置 {name},改用凭据引用", "config.headers")
|
||||
url = str(config.get("url") or "").strip()
|
||||
credential_ref = str(config.get("credentialRef") or config.get("credential_ref") or "").strip()
|
||||
if not url and not credential_ref:
|
||||
return _issue("HTTP 节点缺少 URL 或凭据引用", "config.url")
|
||||
if credential_ref:
|
||||
source = catalog.data_sources.get(credential_ref)
|
||||
if source is None:
|
||||
return _issue("引用的凭据(HTTP 数据源)不存在或无权访问", "config.credentialRef")
|
||||
if not source.get("enabled", True):
|
||||
return _issue("引用的凭据已被禁用", "config.credentialRef")
|
||||
if str(source.get("kind") or "") != "http":
|
||||
return _issue("引用的凭据不是 HTTP 类型数据源", "config.credentialRef")
|
||||
allowed = {str(m).upper() for m in (source.get("allowed_methods") or [])}
|
||||
if allowed and method not in allowed:
|
||||
return _issue(f"方法 {method} 不在凭据资源允许的方法列表内", "config.method")
|
||||
return None
|
||||
|
||||
|
||||
def _check_code(config: dict[str, Any], cfg: WorkflowConfig) -> dict[str, Any] | None:
|
||||
if not cfg.code.enabled:
|
||||
return _issue("代码节点未启用(需管理员在 config.yaml 中开启 workflows.code.enabled)", "")
|
||||
language = config.get("language")
|
||||
if language is not None and str(language) != "python":
|
||||
return _issue("代码节点目前仅支持 python", "config.language")
|
||||
source = str(config.get("source") or "")
|
||||
if not source.strip():
|
||||
return _issue("代码节点没有源代码", "config.source")
|
||||
timeout = config.get("timeoutSeconds") or config.get("timeout_seconds")
|
||||
if timeout is not None:
|
||||
try:
|
||||
seconds = int(timeout)
|
||||
except (TypeError, ValueError):
|
||||
return _issue("timeoutSeconds 必须是整数", "config.timeoutSeconds")
|
||||
if seconds < 1 or seconds > int(cfg.code.timeout_seconds):
|
||||
return _issue(f"timeoutSeconds 超过上限 {cfg.code.timeout_seconds}", "config.timeoutSeconds")
|
||||
return None
|
||||
|
||||
|
||||
def _check_output(config: dict[str, Any]) -> dict[str, Any] | None:
|
||||
mapping = config.get("mapping")
|
||||
if mapping is None:
|
||||
return None
|
||||
if not isinstance(mapping, dict):
|
||||
return _issue("输出映射必须是对象", "config.mapping")
|
||||
for key, template in mapping.items():
|
||||
if not isinstance(template, str) or not template.strip():
|
||||
return _issue(f"输出字段 {key} 的映射模板为空", f"config.mapping.{key}")
|
||||
return None
|
||||
|
||||
|
||||
def _check_subworkflow(config: dict[str, Any], catalog: ResourceCatalog, *, current_workflow_id: str) -> dict[str, Any] | None:
|
||||
workflow_id = str(config.get("workflowId") or config.get("workflow_id") or "")
|
||||
if not workflow_id:
|
||||
return _issue("子工作流节点缺少 workflowId", "config.workflowId")
|
||||
version_id = str(config.get("versionId") or config.get("version_id") or "")
|
||||
if not version_id:
|
||||
return _issue("子工作流节点缺少 versionId", "config.versionId")
|
||||
if workflow_id == current_workflow_id:
|
||||
return _issue("子工作流不能引用自身", "config.workflowId")
|
||||
versions = catalog.published_subworkflows.get(workflow_id)
|
||||
if not versions:
|
||||
return _issue("子工作流未发布或无权访问", "config.workflowId")
|
||||
if version_id not in versions:
|
||||
return _issue("子工作流版本不存在或已下线", "config.versionId")
|
||||
return None
|
||||
|
||||
|
||||
def make_resource_checker(
|
||||
catalog: ResourceCatalog,
|
||||
*,
|
||||
config: WorkflowConfig,
|
||||
current_workflow_id: str = "",
|
||||
):
|
||||
"""Build the sync ``resource_checker`` consumed by ``validate_workflow_graph``."""
|
||||
|
||||
def checker(node_type: str, node_config: dict[str, Any]):
|
||||
try:
|
||||
if node_type == "agent":
|
||||
return _check_agent(node_config, catalog)
|
||||
if node_type == "skill":
|
||||
return _check_skill(node_config, catalog)
|
||||
if node_type == "sql_read":
|
||||
return _check_sql(node_config, catalog, config)
|
||||
if node_type == "http":
|
||||
return _check_http(node_config, catalog)
|
||||
if node_type == "code":
|
||||
return _check_code(node_config, config)
|
||||
if node_type == "output":
|
||||
return _check_output(node_config)
|
||||
if node_type == "subworkflow":
|
||||
return _check_subworkflow(node_config, catalog, current_workflow_id=current_workflow_id)
|
||||
except Exception: # noqa: BLE001 - a broken check must fail closed
|
||||
return _issue("资源配置校验失败,请检查节点配置", "")
|
||||
return None
|
||||
|
||||
return checker
|
||||
|
||||
|
||||
__all__ = ["ResourceCatalog", "build_resource_catalog", "make_resource_checker"]
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Periodic retention sweep for workflow runs, events, and artifacts.
|
||||
|
||||
Consumes ``workflows.retention.event_retention_days`` /
|
||||
``run_retention_days``. Events older than the event window are dropped first;
|
||||
terminal runs older than the run window are deleted with their node-run and
|
||||
artifact rows, and any leftover events for those run ids are cleaned up so the
|
||||
event log cannot outlive the run it describes.
|
||||
|
||||
Artifact *files* stay on the existing sandbox/artifact lifecycle — this job
|
||||
only removes the workflow catalog rows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_INTERVAL_SECONDS = 60 * 60
|
||||
_MAX_BATCHES_PER_SWEEP = 8
|
||||
_SQL_PURGE_BATCH = 500
|
||||
|
||||
|
||||
class WorkflowRetentionCleaner:
|
||||
def __init__(
|
||||
self,
|
||||
runs: WorkflowRunStore,
|
||||
events: WorkflowEventStore,
|
||||
*,
|
||||
config: WorkflowConfig | None = None,
|
||||
) -> None:
|
||||
self._runs = runs
|
||||
self._events = events
|
||||
self._config = config or WorkflowConfig()
|
||||
self._task: asyncio.Task | None = None
|
||||
self._stopping = False
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is not None and not self._task.done():
|
||||
return
|
||||
self._stopping = False
|
||||
self._task = asyncio.create_task(self._loop(), name="workflow-retention")
|
||||
logger.info(
|
||||
"workflow retention cleaner started (events=%sd runs=%sd)",
|
||||
self._config.retention.event_retention_days,
|
||||
self._config.retention.run_retention_days,
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
self._stopping = True
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except (asyncio.CancelledError, Exception): # noqa: BLE001
|
||||
pass
|
||||
self._task = None
|
||||
|
||||
async def sweep(self) -> dict[str, int]:
|
||||
"""One pass. Safe to call from tests without starting the loop."""
|
||||
event_days = max(1, self._config.retention.event_retention_days)
|
||||
run_days = max(1, self._config.retention.run_retention_days)
|
||||
events_deleted = await self._events.purge_before(older_than_days=event_days)
|
||||
runs_deleted = 0
|
||||
for _ in range(_MAX_BATCHES_PER_SWEEP):
|
||||
ids = await self._runs.purge_terminal_before(older_than_days=run_days)
|
||||
if not ids:
|
||||
break
|
||||
runs_deleted += len(ids)
|
||||
events_deleted += await self._events.purge_for_runs(ids)
|
||||
if len(ids) < _SQL_PURGE_BATCH:
|
||||
break
|
||||
if events_deleted or runs_deleted:
|
||||
logger.info("workflow retention sweep: events=%s runs=%s", events_deleted, runs_deleted)
|
||||
return {"events": events_deleted, "runs": runs_deleted}
|
||||
|
||||
async def _loop(self) -> None:
|
||||
try:
|
||||
await self.sweep()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001 - the loop must survive a bad pass
|
||||
logger.exception("workflow retention sweep failed")
|
||||
while not self._stopping:
|
||||
try:
|
||||
await asyncio.sleep(_INTERVAL_SECONDS)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
if self._stopping:
|
||||
return
|
||||
try:
|
||||
await self.sweep()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("workflow retention sweep failed")
|
||||
|
||||
|
||||
__all__ = ["WorkflowRetentionCleaner"]
|
||||
@@ -647,3 +647,45 @@ task_deeplink:
|
||||
taskDirection: "正向引导 + 风险预警"
|
||||
taskName: "任务01:暑期文旅舆情综合研判"
|
||||
taskContent: "围绕暑期文旅消费高峰,对重点城市、重点景区的公众情绪与舆情态势做综合研判,识别情绪拐点与潜在风险,提出正向引导与处置建议。"
|
||||
|
||||
# Workflow Studio (phase 1: definitions/publish; runtime later)
|
||||
workflows:
|
||||
enabled: true
|
||||
max_steps: 500
|
||||
max_loop_iterations: 20
|
||||
max_parallelism: 16
|
||||
run_timeout_seconds: 3600
|
||||
node_timeout_seconds: 600
|
||||
max_concurrent_runs_per_user: 5
|
||||
lease_ttl_seconds: 60
|
||||
embed:
|
||||
allowed_origins: []
|
||||
ticket_ttl_seconds: 60
|
||||
# http 节点出网策略:默认拒绝内网/环回/云元数据,只放行公网。
|
||||
# allowed_hosts 非空时变为白名单模式(精确域名或 .suffix)。
|
||||
http:
|
||||
enabled: true
|
||||
allowed_hosts: []
|
||||
blocked_hosts: []
|
||||
allow_private_networks: false
|
||||
allowed_ports: [80, 443, 8080, 8443]
|
||||
max_response_bytes: 2097152
|
||||
max_redirects: 0
|
||||
timeout_seconds: 60
|
||||
# sql_read 节点:仅单条 SELECT/WITH,参数绑定,强制 LIMIT。
|
||||
sql:
|
||||
enabled: true
|
||||
max_rows: 1000
|
||||
statement_timeout_seconds: 30
|
||||
max_cell_chars: 4000
|
||||
# code 节点默认关闭:本地执行器是加固子进程而非容器,需管理员显式开启。
|
||||
code:
|
||||
enabled: false
|
||||
timeout_seconds: 30
|
||||
max_source_chars: 20000
|
||||
max_output_chars: 100000
|
||||
memory_limit_mb: 512
|
||||
allow_network: false
|
||||
retention:
|
||||
event_retention_days: 30
|
||||
run_retention_days: 90
|
||||
@@ -22,6 +22,16 @@ This directory contains detailed documentation for the DeerFlow backend.
|
||||
| [plan_mode_usage.md](plan_mode_usage.md) | Plan mode with TodoList |
|
||||
| [AUTO_TITLE_GENERATION.md](AUTO_TITLE_GENERATION.md) | Automatic title generation |
|
||||
|
||||
## Workflow Studio
|
||||
|
||||
| Document | Description |
|
||||
|----------|-------------|
|
||||
| [WORKFLOW_STUDIO_BACKEND_DEV_ZH.md](WORKFLOW_STUDIO_BACKEND_DEV_ZH.md) | 后端开发基线方案 + 各阶段完成状态 |
|
||||
| [adr/0001-workflow-studio-runtime.md](adr/0001-workflow-studio-runtime.md) | ADR:Coze 只做画布 / ChatDev 只作参考 / 自建 DAG 调度器(LangGraph 仅作智能体内核) |
|
||||
| [WORKFLOW_SCHEMA_V1_ZH.md](WORKFLOW_SCHEMA_V1_ZH.md) | 内部图 Schema v1.0 |
|
||||
| [WORKFLOW_SSE_V1_ZH.md](WORKFLOW_SSE_V1_ZH.md) | SSE 事件协议 v1.0 |
|
||||
| [WORKFLOW_OPENAPI_DRAFT_V1.yaml](WORKFLOW_OPENAPI_DRAFT_V1.yaml) | 接口 OpenAPI 草案 |
|
||||
|
||||
## Development
|
||||
|
||||
| Document | Description |
|
||||
|
||||
@@ -0,0 +1,740 @@
|
||||
openapi: 3.0.3
|
||||
info:
|
||||
title: DeerFlow Workflow Studio API (Draft v1)
|
||||
version: "1.0.0-draft"
|
||||
description: |
|
||||
Wire models: deerflow.workflows.schemas / events / errors.
|
||||
|
||||
Paths marked `x-implemented: false` are still sketches; everything else is
|
||||
mounted today (app/gateway/routers/workflow*.py). Where the phase-0 sketch
|
||||
and the implementation disagreed, this file follows the implementation.
|
||||
servers:
|
||||
- url: http://localhost:8001
|
||||
description: Local Gateway
|
||||
tags:
|
||||
- name: workflows
|
||||
- name: runs
|
||||
- name: resources
|
||||
- name: data-sources
|
||||
- name: embed
|
||||
- name: coze-compat
|
||||
paths:
|
||||
/api/workflows:
|
||||
get:
|
||||
tags: [workflows]
|
||||
summary: List workflow definitions
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
post:
|
||||
tags: [workflows]
|
||||
summary: Create workflow definition
|
||||
responses:
|
||||
"201":
|
||||
description: Created
|
||||
/api/workflows/{workflow_id}:
|
||||
get:
|
||||
tags: [workflows]
|
||||
summary: Get workflow definition
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
patch:
|
||||
tags: [workflows]
|
||||
summary: Update workflow metadata
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
delete:
|
||||
tags: [workflows]
|
||||
summary: Soft-delete / archive
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
responses:
|
||||
"204":
|
||||
description: Archived
|
||||
/api/workflows/{workflow_id}/copy:
|
||||
post:
|
||||
tags: [workflows]
|
||||
summary: Duplicate a workflow as a new draft owned by the caller
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
responses:
|
||||
"200":
|
||||
description: Created copy
|
||||
/api/workflows/{workflow_id}/draft:
|
||||
put:
|
||||
tags: [workflows]
|
||||
summary: Save draft (optimistic lock)
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [expectedRevision, graph]
|
||||
properties:
|
||||
expectedRevision:
|
||||
type: integer
|
||||
graph:
|
||||
$ref: "#/components/schemas/WorkflowGraph"
|
||||
responses:
|
||||
"200":
|
||||
description: Saved
|
||||
"409":
|
||||
description: Draft revision conflict
|
||||
/api/workflows/{workflow_id}/validate:
|
||||
post:
|
||||
tags: [workflows]
|
||||
summary: Validate draft without publishing
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
responses:
|
||||
"200":
|
||||
description: Validation result
|
||||
/api/workflows/{workflow_id}/publish:
|
||||
post:
|
||||
tags: [workflows]
|
||||
summary: Publish immutable version
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
responses:
|
||||
"201":
|
||||
description: Version created
|
||||
/api/workflows/{workflow_id}/versions:
|
||||
get:
|
||||
tags: [workflows]
|
||||
summary: List published versions
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
/api/workflows/{workflow_id}/versions/{version_id}:
|
||||
get:
|
||||
tags: [workflows]
|
||||
summary: Get one published version snapshot
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
- $ref: "#/components/parameters/VersionId"
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
/api/workflows/node-types:
|
||||
get:
|
||||
tags: [resources]
|
||||
summary: List supported node types
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
/api/workflows/resources/agents:
|
||||
get:
|
||||
tags: [resources]
|
||||
summary: Agents available to workflow nodes
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
/api/workflows/resources/skills:
|
||||
get:
|
||||
tags: [resources]
|
||||
summary: Skills available to workflow nodes
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
/api/workflows/resources/data-sources:
|
||||
get:
|
||||
tags: [resources]
|
||||
summary: Registered read-only data sources
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
/api/workflows/resources/subworkflows:
|
||||
get:
|
||||
tags: [resources]
|
||||
summary: Published workflows usable as subworkflows
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
/api/workflows/data-sources:
|
||||
get:
|
||||
tags: [data-sources]
|
||||
summary: List data sources visible to the caller (own + shared)
|
||||
description: |
|
||||
Never returns the DSN or credential headers — only `masked_target`.
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
post:
|
||||
tags: [data-sources]
|
||||
summary: Register a data source (`shared: true` is admin-only)
|
||||
description: |
|
||||
`kind: sql` requires `dsn`; `kind: http` requires `headers`. The secret
|
||||
is encrypted on write and only ever decrypted inside the run executor.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/DataSourceCreate"
|
||||
responses:
|
||||
"200":
|
||||
description: Created (masked view)
|
||||
"503":
|
||||
description: WORKFLOW_SECRET_KEY not configured
|
||||
/api/workflows/data-sources/{id}:
|
||||
get:
|
||||
tags: [data-sources]
|
||||
summary: Get one data source (masked view)
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/DataSourceId"
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
put:
|
||||
tags: [data-sources]
|
||||
summary: Update data source (owner or admin)
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/DataSourceId"
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
delete:
|
||||
tags: [data-sources]
|
||||
summary: Delete data source (owner or admin)
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/DataSourceId"
|
||||
responses:
|
||||
"200":
|
||||
description: '{ "deleted": true }'
|
||||
/api/workflows/data-sources/{id}/introspect:
|
||||
post:
|
||||
tags: [data-sources]
|
||||
summary: Introspect whitelisted schema/table/column metadata
|
||||
description: Returns the configured table allowlist only. Never returns sample rows or secrets.
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/DataSourceId"
|
||||
responses:
|
||||
"200":
|
||||
description: '{ dataSourceId, kind, tables, schemas }'
|
||||
/api/workflows/data-sources/{id}/validate-query:
|
||||
post:
|
||||
tags: [data-sources]
|
||||
summary: Validate SQL against read-only policy (no execution)
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/DataSourceId"
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [statement]
|
||||
properties:
|
||||
statement:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: '{ ok: true, statement }'
|
||||
"400":
|
||||
description: WORKFLOW_SQL_POLICY_DENIED
|
||||
/api/workflows/{workflow_id}/runs:
|
||||
post:
|
||||
tags: [runs]
|
||||
summary: Start a run against a published version
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/StartRunRequest"
|
||||
responses:
|
||||
"200":
|
||||
description: |
|
||||
Run view plus `created`; `created: false` means the idempotencyKey
|
||||
matched an existing run (scoped per workflow).
|
||||
"400":
|
||||
description: WORKFLOW_INPUT_INVALID / WORKFLOW_VERSION_NOT_FOUND
|
||||
"429":
|
||||
description: WORKFLOW_LIMIT_EXCEEDED (per-user concurrent runs)
|
||||
get:
|
||||
tags: [runs]
|
||||
summary: List runs for a workflow
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkflowId"
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
/api/workflows/runs/{run_id}:
|
||||
get:
|
||||
tags: [runs]
|
||||
summary: Get run status projection plus current node runs
|
||||
description: |
|
||||
Never includes `resumeToken` — that is delivered on the event stream only.
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/RunId"
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
"403":
|
||||
description: WORKFLOW_FORBIDDEN (not owner, not admin)
|
||||
"404":
|
||||
description: WORKFLOW_RUN_NOT_FOUND
|
||||
/api/workflows/runs/{run_id}/events:
|
||||
get:
|
||||
tags: [runs]
|
||||
summary: Page durable events
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/RunId"
|
||||
- name: after
|
||||
in: query
|
||||
description: Exclusive `seq` cursor. `after_seq` is accepted as an alias.
|
||||
schema:
|
||||
type: integer
|
||||
default: 0
|
||||
- name: after_seq
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
- name: limit
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
default: 200
|
||||
maximum: 1000
|
||||
responses:
|
||||
"200":
|
||||
description: '{ "events": [...], "lastSeq": 42 }'
|
||||
/api/workflows/runs/{run_id}/stream:
|
||||
get:
|
||||
tags: [runs]
|
||||
summary: SSE replay + live tail
|
||||
description: |
|
||||
Replays from `Last-Event-ID` (preferred) or `after` / `after_seq`, then
|
||||
tails live. Closes on the first terminal event; if the run is already
|
||||
terminal without one, a synthesised terminal frame is emitted so the
|
||||
client never hangs.
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/RunId"
|
||||
- name: after
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
default: 0
|
||||
- name: after_seq
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
- name: Last-Event-ID
|
||||
in: header
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: text/event-stream
|
||||
headers:
|
||||
X-Workflow-Run-Status:
|
||||
description: Run status at connection time.
|
||||
schema:
|
||||
type: string
|
||||
content:
|
||||
text/event-stream:
|
||||
schema:
|
||||
type: string
|
||||
/api/workflows/runs/{run_id}/cancel:
|
||||
post:
|
||||
tags: [runs]
|
||||
summary: Request cancellation (idempotent)
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/RunId"
|
||||
responses:
|
||||
"200":
|
||||
description: |
|
||||
Run view plus `cancelled`; `false` means it was already terminal.
|
||||
/api/workflows/runs/{run_id}/resume:
|
||||
post:
|
||||
tags: [runs]
|
||||
summary: Resume after human_input
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/RunId"
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: "#/components/schemas/ResumeRunRequest"
|
||||
responses:
|
||||
"200":
|
||||
description: Requeued
|
||||
"409":
|
||||
description: WORKFLOW_RUN_NOT_RESUMABLE / WORKFLOW_RESUME_TOKEN_INVALID
|
||||
/api/workflows/runs/{run_id}/retry:
|
||||
post:
|
||||
tags: [runs]
|
||||
summary: Retry a failed run from the first unfinished node
|
||||
description: |
|
||||
Creates a new queued run with `retryOfRunId` pointing at the failed
|
||||
original. Completed `workflow_node_runs` are copied so the engine
|
||||
resumes from the failed node; if nothing is replayable it is a full
|
||||
re-run of the same version and inputs. Only `failed` runs are retryable.
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/RunId"
|
||||
responses:
|
||||
"201":
|
||||
description: '{ ..., retryOfRunId, created: true }'
|
||||
"409":
|
||||
description: WORKFLOW_RUN_NOT_RETRYABLE
|
||||
"429":
|
||||
description: WORKFLOW_LIMIT_EXCEEDED
|
||||
/api/workflows/runs/{run_id}/artifacts:
|
||||
get:
|
||||
tags: [runs]
|
||||
summary: List artifacts for a run
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/RunId"
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
/api/workflows/embed/tickets:
|
||||
post:
|
||||
tags: [embed]
|
||||
summary: Issue a short-lived iframe ticket (authenticated parent)
|
||||
description: |
|
||||
`origin` must exactly match `workflows.embed.allowed_origins`. The
|
||||
response also carries `frameAncestors` for the host's CSP.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [workflowId, origin]
|
||||
properties:
|
||||
workflowId:
|
||||
type: string
|
||||
origin:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: '{ ticket, expiresIn, origin, frameAncestors }'
|
||||
"403":
|
||||
description: WORKFLOW_EMBED_ORIGIN_DENIED / WORKFLOW_FORBIDDEN
|
||||
"503":
|
||||
description: No signing key configured
|
||||
/api/workflows/embed/verify:
|
||||
post:
|
||||
tags: [embed]
|
||||
summary: Verify a ticket (single-use, best effort per process)
|
||||
description: |
|
||||
A ticket proves "this origin may embed this workflow for this user right
|
||||
now". It is NOT an API credential — embedded pages still authenticate
|
||||
their API calls with the normal session.
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required: [ticket]
|
||||
properties:
|
||||
ticket:
|
||||
type: string
|
||||
origin:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: '{ ok, workflowId, userId, origin, expiresAt }'
|
||||
"401":
|
||||
description: WORKFLOW_EMBED_TICKET_INVALID (expired or replayed)
|
||||
"403":
|
||||
description: WORKFLOW_EMBED_ORIGIN_DENIED (origin mismatch)
|
||||
/api/workflow_api/canvas:
|
||||
post:
|
||||
tags: [coze-compat]
|
||||
summary: Coze canvas load adapter
|
||||
responses:
|
||||
"200":
|
||||
description: Coze DTO
|
||||
/api/workflow_api/save:
|
||||
post:
|
||||
tags: [coze-compat]
|
||||
summary: Coze canvas save adapter
|
||||
responses:
|
||||
"200":
|
||||
description: Saved
|
||||
/api/workflow_api/create:
|
||||
post:
|
||||
tags: [coze-compat]
|
||||
summary: Coze create workflow adapter
|
||||
responses:
|
||||
"200":
|
||||
description: Created
|
||||
/api/workflow_api/node_template_list:
|
||||
post:
|
||||
tags: [coze-compat]
|
||||
summary: Coze node template list adapter
|
||||
responses:
|
||||
"200":
|
||||
description: Templates
|
||||
components:
|
||||
parameters:
|
||||
WorkflowId:
|
||||
name: workflow_id
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
VersionId:
|
||||
name: version_id
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
RunId:
|
||||
name: run_id
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
DataSourceId:
|
||||
name: id
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
schemas:
|
||||
WorkflowGraph:
|
||||
type: object
|
||||
description: Internal standard graph (see WORKFLOW_SCHEMA_V1_ZH.md)
|
||||
required: [schemaVersion, id, nodes, edges]
|
||||
properties:
|
||||
schemaVersion:
|
||||
type: string
|
||||
enum: ["1.0"]
|
||||
id:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
description:
|
||||
type: string
|
||||
inputSchema:
|
||||
type: object
|
||||
outputSchema:
|
||||
type: object
|
||||
nodes:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/WorkflowNode"
|
||||
edges:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/WorkflowEdge"
|
||||
settings:
|
||||
$ref: "#/components/schemas/WorkflowGraphSettings"
|
||||
WorkflowNode:
|
||||
type: object
|
||||
required: [id, type]
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
type:
|
||||
type: string
|
||||
enum:
|
||||
- start
|
||||
- output
|
||||
- agent
|
||||
- skill
|
||||
- http
|
||||
- sql_read
|
||||
- code
|
||||
- transform
|
||||
- condition
|
||||
- merge
|
||||
- human_input
|
||||
- subworkflow
|
||||
- loop
|
||||
name:
|
||||
type: string
|
||||
config:
|
||||
type: object
|
||||
WorkflowEdge:
|
||||
type: object
|
||||
required: [id, source, target]
|
||||
properties:
|
||||
id:
|
||||
type: string
|
||||
source:
|
||||
type: string
|
||||
target:
|
||||
type: string
|
||||
sourcePort:
|
||||
type: string
|
||||
nullable: true
|
||||
WorkflowGraphSettings:
|
||||
type: object
|
||||
properties:
|
||||
runTimeoutSeconds:
|
||||
type: integer
|
||||
default: 1800
|
||||
nodeTimeoutSeconds:
|
||||
type: integer
|
||||
default: 300
|
||||
maxSteps:
|
||||
type: integer
|
||||
default: 200
|
||||
maxLoopIterations:
|
||||
type: integer
|
||||
default: 5
|
||||
maxParallelism:
|
||||
type: integer
|
||||
default: 4
|
||||
NodeResult:
|
||||
type: object
|
||||
properties:
|
||||
data:
|
||||
type: object
|
||||
messages:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
artifacts:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/ArtifactRef"
|
||||
metadata:
|
||||
type: object
|
||||
warnings:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
ArtifactRef:
|
||||
type: object
|
||||
required: [artifactId, name]
|
||||
properties:
|
||||
artifactId:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
mimeType:
|
||||
type: string
|
||||
path:
|
||||
type: string
|
||||
sizeBytes:
|
||||
type: integer
|
||||
nullable: true
|
||||
preview:
|
||||
type: string
|
||||
nullable: true
|
||||
StartRunRequest:
|
||||
type: object
|
||||
properties:
|
||||
versionId:
|
||||
type: string
|
||||
description: Omit to use the latest published version.
|
||||
inputs:
|
||||
type: object
|
||||
idempotencyKey:
|
||||
type: string
|
||||
maxLength: 128
|
||||
executionMode:
|
||||
type: string
|
||||
enum: [normal]
|
||||
default: normal
|
||||
ResumeRunRequest:
|
||||
type: object
|
||||
required: [resumeToken]
|
||||
properties:
|
||||
resumeToken:
|
||||
type: string
|
||||
nodeId:
|
||||
type: string
|
||||
description: Defaults to the paused node from `pendingInput`.
|
||||
action:
|
||||
type: string
|
||||
default: submit
|
||||
values:
|
||||
type: object
|
||||
DataSourceCreate:
|
||||
type: object
|
||||
required: [name]
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
description:
|
||||
type: string
|
||||
kind:
|
||||
type: string
|
||||
enum: [sql, http]
|
||||
default: sql
|
||||
dsn:
|
||||
type: string
|
||||
description: Write-only; required for `kind: sql`.
|
||||
headers:
|
||||
type: object
|
||||
additionalProperties:
|
||||
type: string
|
||||
description: Write-only; required for `kind: http`.
|
||||
maxRows:
|
||||
type: integer
|
||||
default: 1000
|
||||
allowedTables:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
enabled:
|
||||
type: boolean
|
||||
default: true
|
||||
shared:
|
||||
type: boolean
|
||||
default: false
|
||||
description: Admin-only. Shared sources have a null owner.
|
||||
WorkflowEventEnvelope:
|
||||
type: object
|
||||
required: [schemaVersion, runId, workflowId, versionId, seq, event]
|
||||
properties:
|
||||
schemaVersion:
|
||||
type: string
|
||||
enum: ["1.0"]
|
||||
runId:
|
||||
type: string
|
||||
workflowId:
|
||||
type: string
|
||||
versionId:
|
||||
type: string
|
||||
seq:
|
||||
type: integer
|
||||
event:
|
||||
type: string
|
||||
nodeId:
|
||||
type: string
|
||||
nullable: true
|
||||
nodeRunId:
|
||||
type: string
|
||||
nullable: true
|
||||
timestamp:
|
||||
type: string
|
||||
format: date-time
|
||||
data:
|
||||
type: object
|
||||
WorkflowErrorBody:
|
||||
type: object
|
||||
required: [code, message]
|
||||
properties:
|
||||
code:
|
||||
type: string
|
||||
message:
|
||||
type: string
|
||||
retryable:
|
||||
type: boolean
|
||||
nodeId:
|
||||
type: string
|
||||
nullable: true
|
||||
details:
|
||||
type: object
|
||||
@@ -0,0 +1,93 @@
|
||||
# 工作流图 Schema v1.0(冻结)
|
||||
|
||||
> 阶段 0 交付。实现源:`packages/harness/deerflow/workflows/schemas.py`
|
||||
> 样例:`packages/harness/deerflow/workflows/samples/`
|
||||
> ADR:[`adr/0001-workflow-studio-runtime.md`](adr/0001-workflow-studio-runtime.md)
|
||||
|
||||
## 原则
|
||||
|
||||
- Coze 画布 DTO **进入 Gateway 后**转换成此内部结构;运行时不依赖 Coze 私有字段。
|
||||
- Wire JSON 使用 **camelCase**;Python 模型使用 snake_case(`populate_by_name=True`)。
|
||||
- `schemaVersion` 固定为 `"1.0"`;破坏性变更必须升版本并双写迁移说明。
|
||||
|
||||
## 顶层 `WorkflowGraph`
|
||||
|
||||
| 字段 | 类型 | 说明 |
|
||||
|------|------|------|
|
||||
| `schemaVersion` | `"1.0"` | 协议版本 |
|
||||
| `id` | string | 工作流逻辑 id(定义侧稳定标识) |
|
||||
| `name` / `description` | string | 展示用 |
|
||||
| `inputSchema` / `outputSchema` | JSON Schema object | 开始/输出校验 |
|
||||
| `nodes` | `WorkflowNode[]` | 节点列表,`id` 唯一 |
|
||||
| `edges` | `WorkflowEdge[]` | 边列表,`id` 唯一;`source`/`target` 必须引用已有节点 |
|
||||
| `settings` | object | 见下 |
|
||||
|
||||
### `settings`
|
||||
|
||||
| 字段 | 默认 | 说明 |
|
||||
|------|------|------|
|
||||
| `runTimeoutSeconds` | 1800 | 整 run 超时 |
|
||||
| `nodeTimeoutSeconds` | 300 | 单节点默认超时 |
|
||||
| `maxSteps` | 200 | 最大节点步数 |
|
||||
| `maxLoopIterations` | 5 | 全局循环上限 |
|
||||
| `maxParallelism` | 4 | 同层并行度 |
|
||||
|
||||
发布时仍受系统级配额覆盖(阶段 6)。
|
||||
|
||||
## 节点类型(封闭集合)
|
||||
|
||||
`start` · `output` · `agent` · `skill` · `http` · `sql_read` · `code` · `transform` · `condition` · `merge` · `human_input` · `subworkflow` · `loop`
|
||||
|
||||
每类最小样例见 `workflows/samples/nodes/<type>.json`。
|
||||
|
||||
## `NodeResult`(统一节点结果)
|
||||
|
||||
```json
|
||||
{
|
||||
"data": {},
|
||||
"messages": [],
|
||||
"artifacts": [
|
||||
{
|
||||
"artifactId": "artifact_01",
|
||||
"name": "专题报告.md",
|
||||
"mimeType": "text/markdown",
|
||||
"path": "/mnt/data/专题报告.md"
|
||||
}
|
||||
],
|
||||
"metadata": {},
|
||||
"warnings": []
|
||||
}
|
||||
```
|
||||
|
||||
大文本 / 二进制 / 超大查询结果写入 artifact,事件与 `output_json` 只传引用与有界预览。
|
||||
|
||||
## 运行状态
|
||||
|
||||
`queued` → `running` → `awaiting_input` | `completed` | `failed` | `cancel_requested` → `cancelled`
|
||||
|
||||
终态:`completed` / `failed` / `cancelled`。
|
||||
|
||||
## 安全表达式(约定,求值器阶段 2+)
|
||||
|
||||
允许只读引用:
|
||||
|
||||
- `{{ inputs.<field> }}`
|
||||
- `{{ nodes.<nodeId>.data... }}`
|
||||
- `{{ run.id }}`
|
||||
|
||||
禁止:`eval`、任意属性反射、字符串拼接构造 SQL/HTTP。条件节点运算符白名单:相等、包含、数值比较、空值、布尔组合。
|
||||
|
||||
## `graph_hash`
|
||||
|
||||
对规范化 camelCase JSON(`model_dump_json(by_alias=True, exclude_none=True)`)做 SHA-256。发布版本写入 `workflow_versions.graph_hash`;编译缓存可按 hash 做有界缓存,缓存对象不得含用户密钥。
|
||||
|
||||
## 样例图
|
||||
|
||||
| 文件 | 用途 |
|
||||
|------|------|
|
||||
| `samples/report_generation.json` | 专题报告端到端验收基线 |
|
||||
| `samples/start_output.json` | 最小 start→output(运行骨架) |
|
||||
| `samples/nodes/*.json` | 各节点最小 config |
|
||||
| `samples/event_envelope_example.json` | SSE envelope 契约样例 |
|
||||
|
||||
加载:`from deerflow.workflows.samples import load_sample_graph`。
|
||||
@@ -0,0 +1,107 @@
|
||||
# 工作流 SSE / 事件协议 v1.0(冻结)
|
||||
|
||||
> 实现源:`packages/harness/deerflow/workflows/events.py`
|
||||
> 错误体:`packages/harness/deerflow/workflows/errors.py`
|
||||
> 事件清单与开发方案对齐:`WORKFLOW_STUDIO_BACKEND_DEV_ZH.md` §9
|
||||
|
||||
## 不变式
|
||||
|
||||
1. **先持久化,后发布** — 用户可见事件先写入 `workflow_run_events`,提交成功后再 fan-out。
|
||||
2. **`seq` 严格单调** — 唯一键 `(run_id, seq)`。
|
||||
3. **可重放** — `Last-Event-ID` 优先于 `after_seq`,否则从 `0`。
|
||||
4. **增量合并** — 模型 token 建议 30–100ms 或约 1KB 合并落库。
|
||||
5. **终态停连** — `run.completed` / `run.failed` / `run.cancelled` 且库已补齐后关闭 SSE。
|
||||
6. **不泄密** — 无凭据、无完整认证头、无 DB 连接串、无隐藏思维链。
|
||||
|
||||
## 端点(阶段 2 已实现)
|
||||
|
||||
```http
|
||||
GET /api/workflows/runs/{run_id}/stream?after=128
|
||||
Accept: text/event-stream
|
||||
Last-Event-ID: 128
|
||||
```
|
||||
|
||||
实现:`app/gateway/routers/workflow_runs.py`。
|
||||
|
||||
- **库是排序的权威来源**:连接先按 `Last-Event-ID`(优先)或 `after` 从 `workflow_run_events` 回放,再挂 `WorkflowLiveHub` 订阅实时帧;`seq` 不大于已回放游标的实时帧被丢弃,下一轮 DB 轮询(0.8s)会带上它。断线重连因此不会看到空洞或乱序。
|
||||
- **终态必闭**:读到 `run.completed` / `run.failed` / `run.cancelled` 即关闭。若运行已是终态却没有终态事件(例如崩在写事件之前),补发一条 `synthesised: true` 的终态帧再关闭,客户端不会永久挂住。
|
||||
- **响应头** `X-Workflow-Run-Status` 给出连接建立瞬间的状态。
|
||||
- Heartbeat:SSE comment `: heartbeat`(**不写库**),15s 静默后发送。
|
||||
- 同一份事件也可用 `GET /api/workflows/runs/{run_id}/events?after=&limit=` 分页拉取(无长连接场景)。
|
||||
|
||||
## Envelope
|
||||
|
||||
```json
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"runId": "run_01",
|
||||
"workflowId": "wf_01",
|
||||
"versionId": "wv_03",
|
||||
"seq": 42,
|
||||
"event": "node.output.delta",
|
||||
"nodeId": "agent_writer",
|
||||
"nodeRunId": "nr_09",
|
||||
"timestamp": "2026-08-26T10:00:00+08:00",
|
||||
"data": {}
|
||||
}
|
||||
```
|
||||
|
||||
SSE 帧:
|
||||
|
||||
```text
|
||||
id: 42
|
||||
event: node.output.delta
|
||||
data: { ... envelope ... }
|
||||
```
|
||||
|
||||
Python:`WorkflowEventEnvelope.to_sse_frame()`。
|
||||
|
||||
## 事件类型(封闭集合)
|
||||
|
||||
| 事件 | `data` 要点 |
|
||||
|------|-------------|
|
||||
| `run.created` | input 摘要、version |
|
||||
| `run.queued` | queue 信息 |
|
||||
| `run.started` | startedAt |
|
||||
| `node.queued` | node 信息 |
|
||||
| `node.started` | attempt、输入摘要 |
|
||||
| `node.progress` | phase、message、percent |
|
||||
| `node.output.delta` | channel、delta |
|
||||
| `node.tool.started` | toolCallId、name、input 摘要 |
|
||||
| `node.tool.finished` | toolCallId、status、output 摘要 |
|
||||
| `artifact.created` | artifact 元数据 |
|
||||
| `node.completed` | output 摘要、duration |
|
||||
| `node.failed` | error code/message/retryable |
|
||||
| `run.awaiting_input` | formSchema、actions、resumeToken |
|
||||
| `run.resumed` | action |
|
||||
| `run.cancel_requested` | requestedBy |
|
||||
| `run.cancelled` | finishedAt |
|
||||
| `run.completed` | output、artifacts |
|
||||
| `run.failed` | error、failedNodeId |
|
||||
|
||||
## 统一错误体
|
||||
|
||||
```json
|
||||
{
|
||||
"code": "WORKFLOW_SQL_POLICY_DENIED",
|
||||
"message": "SQL 节点只允许单条只读查询",
|
||||
"retryable": false,
|
||||
"nodeId": "sql_1",
|
||||
"details": { "rule": "single_read_statement" }
|
||||
}
|
||||
```
|
||||
|
||||
完整错误码列表:`ALL_WORKFLOW_ERROR_CODES`(`deerflow.workflows.errors`)。
|
||||
|
||||
## `resumeToken` 只走事件流
|
||||
|
||||
`run.awaiting_input` 的 `data.resumeToken` 是恢复运行的唯一凭证,**只在事件流里出现**——`GET /api/workflows/runs/{run_id}` 的运行视图不包含它。恢复时 `POST /api/workflows/runs/{run_id}/resume` 携带该 token,服务端以「消费即失效」的方式原子翻转状态(同一个 token 不能恢复两次)。
|
||||
|
||||
## 前后端契约测试
|
||||
|
||||
前后端共用 fixture:
|
||||
|
||||
- `workflows/samples/event_envelope_example.json`
|
||||
- 本文件事件名表
|
||||
|
||||
实现测试:`tests/test_workflow_runs.py`(回放游标、终态关闭、暂停/恢复的事件序列)。
|
||||
@@ -3,7 +3,16 @@
|
||||
> 文档状态:开发基线方案
|
||||
> 适用项目:`offline-backend-20260512/backend`
|
||||
> 对接前端:Coze Studio 独立工作流页面 `frontend/apps/coze-studio/src/workflow-standalone/`
|
||||
> 更新时间:2026-08-26
|
||||
> 更新时间:2026-08-29
|
||||
> **阶段 0 状态:已完成** — 协议代码 `packages/harness/deerflow/workflows/`;文档见 `WORKFLOW_SCHEMA_V1_ZH.md` / `WORKFLOW_SSE_V1_ZH.md` / `WORKFLOW_OPENAPI_DRAFT_V1.yaml` / `adr/0001-workflow-studio-runtime.md`;契约测试 `tests/test_workflow_schemas.py`
|
||||
> **阶段 1 状态:已完成** — 定义/版本持久化、草稿乐观锁、validate/publish、节点目录、Coze canvas/save/create/node_template 最小适配;路由 `/api/workflows*` + `/api/workflow_api*`;测试 `tests/test_workflow_definitions.py`(尚不执行运行)
|
||||
> **阶段 2 状态:已完成** — run/node_run/event/artifact 表 + 迁移 `20260828_02`;幂等启动、租约调度、重启恢复、持久事件日志、SSE 回放/追尾。**编排内核改为自建 DAG 调度器,不再编译成 LangGraph**(理由见 `adr/0001` 阶段 2 修订)
|
||||
> **阶段 3 状态:已完成** — start/output/transform/condition/merge + 安全表达式引擎;http 节点 SSRF 默认拒内网;sql_read 只读策略 + 数据源登记(密钥加密、只写不读,迁移 `20260828_03`);code 节点加固子进程执行器(**默认关闭**)
|
||||
> **阶段 4 状态:已完成** — agent / skill 节点走 DeerFlow lead agent,模型增量与工具步骤映射为 `node.output.delta` / `node.tool.*`
|
||||
> **阶段 5 状态:已完成** — `human_input` 暂停 + 一次性 resume token、子工作流(深度上限 3)、显式循环与 `max_loop_iterations` / `max_steps` 上限
|
||||
> **阶段 6 状态:已完成** — iframe 票据(origin 白名单 + 60s 一次性 + `frameAncestors`)、按 owner/admin 的运行与数据源鉴权、每用户并发运行上限、结构化审计日志、事件/运行保留期配置与后台清理作业(`WorkflowRetentionCleaner`,每小时扫一次)、失败运行 `POST /retry`(复制已完成节点从失败点续跑,新 run 标 `retry_of_run_id`)、SSE `Last-Event-ID` 优先于 `after`。制品落盘文件仍跟随现有 sandbox/artifact 生命周期,不做工作流专属文件回收。多实例租约竞争 + SSE 重连有单机并发测试(`test_two_dispatchers_only_one_claims_a_run` / `test_sse_reconnect_storm_is_ordered_and_complete`),不是外部分布式压测。
|
||||
> **审计整改(2026-08-29)状态:功能健壮性项已完成** — 按 `WORKFLOW_STUDIO_BACKEND_REMEDIATION_ZH.md`「0. 整改决策记录」:①人工恢复原子化(`resume_with_payload` 单条件更新 + `set_awaiting_input` 一次写入 checkpoint);②草稿保存 CAS 条件更新(409 带 `currentRevision`)+ 发布版本号计数器 `workflow_definitions.next_version_number`;③事件 seq 计数器 `workflow_runs.next_event_seq`(同事务取号,冲突自愈重试),事件落库失败**丢弃不发布**(删除负序号幽灵事件);④输入/输出完整 JSON Schema 校验(`workflows/schema_validation.py`,路由 400 + 节点错误均带字段路径);⑤测试纳入 Git(`.gitignore` 白名单)+ 新增 SQL 并发回归 `tests/test_workflow_concurrency.py`。迁移 `20260828_04_workflow_counter_columns.py` 增加两个计数器列(存量库由启动列同步自动补齐)。P0-01/02(内网可信环境,不实施)、P1-04/05(安全隔离,暂缓)见整改文档决策记录。
|
||||
> 运行时测试:`tests/test_workflow_engine.py`(引擎、表达式、节点、安全策略)+ `tests/test_workflow_runs.py`(HTTP 生命周期、SSE、暂停恢复、租约回收、数据源、票据)+ `tests/test_workflow_concurrency.py`(整改并发回归:内存 + 真实 SQLite)
|
||||
|
||||
## 1. 目标与结论
|
||||
|
||||
@@ -189,6 +198,8 @@ ChatDev 适合参考的源码路径见第 15 节。
|
||||
|
||||
画布只保存 `data_source_id`。前端永远拿不到数据库密码,运行事件也不能回传连接串或认证头。
|
||||
|
||||
HTTP 接口资源使用同一张表:kind=http 时,base_url 和 allowed_methods 是可展示的非密钥元数据;请求头仍加密放在 credential blob,列表、详情、资源目录和运行事件均不得回显。创建时至少提供一个允许方法,地址只接受不含账户密码、查询参数或片段的 http(s) URL。迁移:20260829_01_workflow_http_source_metadata.py。
|
||||
|
||||
### 5.4 iframe 短时票据
|
||||
|
||||
如 Coze 页面与 DeerFlow 不同域,新增短时、一次性的嵌入票据:
|
||||
@@ -581,6 +592,7 @@ DELETE /api/workflows/{workflow_id} # 软删除/归档
|
||||
PUT /api/workflows/{workflow_id}/draft # draft_revision 乐观锁
|
||||
POST /api/workflows/{workflow_id}/validate
|
||||
POST /api/workflows/{workflow_id}/publish
|
||||
POST /api/workflows/{workflow_id}/copy
|
||||
GET /api/workflows/{workflow_id}/versions
|
||||
GET /api/workflows/{workflow_id}/versions/{version_id}
|
||||
```
|
||||
@@ -602,18 +614,19 @@ GET /api/workflows/resources/subworkflows
|
||||
### 10.3 数据源
|
||||
|
||||
```text
|
||||
GET /api/workflow-data-sources
|
||||
POST /api/workflow-data-sources # 管理员或授权角色
|
||||
GET /api/workflow-data-sources/{id}
|
||||
PATCH /api/workflow-data-sources/{id}
|
||||
DELETE /api/workflow-data-sources/{id}
|
||||
POST /api/workflow-data-sources/{id}/test
|
||||
POST /api/workflow-data-sources/{id}/introspect
|
||||
POST /api/workflow-data-sources/{id}/validate-query
|
||||
GET /api/workflows/data-sources
|
||||
POST /api/workflows/data-sources
|
||||
GET /api/workflows/data-sources/{id}
|
||||
PUT /api/workflows/data-sources/{id}
|
||||
DELETE /api/workflows/data-sources/{id}
|
||||
POST /api/workflows/data-sources/{id}/introspect
|
||||
POST /api/workflows/data-sources/{id}/validate-query
|
||||
```
|
||||
|
||||
`introspect` 只返回白名单内 schema/table/column 元数据,不返回样例敏感数据。
|
||||
|
||||
HTTP 接口资源的创建体使用 kind=http、baseUrl、allowedMethods 和可选 headers;headers 是只写字段,服务端加密后不会出现在响应。资源目录 GET /api/workflows/resources/data-sources 返回前端所需的 dataSourceId、baseUrl、allowedMethods、description 等精简字段。
|
||||
|
||||
### 10.4 运行
|
||||
|
||||
```text
|
||||
@@ -743,7 +756,7 @@ POST /api/workflow-embed/tickets/exchange
|
||||
- SQL 只读:连接临时失败可有限重试;语法/权限错误不重试。
|
||||
- HTTP:GET/HEAD 可按策略重试;POST 默认不重试,除非配置幂等键。
|
||||
- 代码:资源不足或代码错误默认不重试。
|
||||
- 工作流级 retry 建议从失败节点恢复;若 checkpoint 不可靠则创建新 run 并标记 `retry_of_run_id`。
|
||||
- 工作流级 retry:`POST /api/workflows/runs/{run_id}/retry` 仅接受 `failed` 运行。新 run 复制已完成节点输出并从失败节点续跑;没有任何可回放 checkpoint 时则整图重跑,并标记 `retry_of_run_id`。
|
||||
|
||||
### 12.3 取消
|
||||
|
||||
@@ -1083,6 +1096,8 @@ tests/test_workflow_migrations.py
|
||||
- 多 Gateway 实例、租约竞争、数据库短暂失败、SSE 大量重连压测。
|
||||
- 事件保留、清理和 artifact 生命周期。
|
||||
|
||||
阶段 6 已落地:`WorkflowRetentionCleaner` 消费 `workflows.retention.*`;`POST /api/workflows/runs/{run_id}/retry`;SSE `Last-Event-ID`;单机租约竞争与 SSE 重连风暴测试。制品文件仍走 sandbox/artifact,不另做工作流专属文件回收。
|
||||
|
||||
### 阶段 7:试点与扩展(持续)
|
||||
|
||||
- 选择 1–2 个报告业务真实试点。
|
||||
|
||||
@@ -0,0 +1,674 @@
|
||||
# Workflow Studio 后端审计整改方案
|
||||
|
||||
> 文档状态:已按 2026-08-29 决策部分实施(见「0. 整改决策记录」)
|
||||
> 适用范围:`offline-backend-20260512/backend`
|
||||
> 审核日期:2026-08-28(决策与实施:2026-08-29)
|
||||
> 关联文档:[工作流编排与流式执行后端开发方案](./WORKFLOW_STUDIO_BACKEND_DEV_ZH.md)
|
||||
> 适用对象:后端开发、测试、安全评审、运维发布
|
||||
|
||||
## 0. 整改决策记录(2026-08-29)
|
||||
|
||||
本轮整改以**功能健壮性**为唯一目标(用户决策:"咱们这次的整改主要是保证功能的健壮"),安全隔离类项目暂缓。逐项状态:
|
||||
|
||||
| 项目 | 状态 | 说明 |
|
||||
|---|---|---|
|
||||
| P1-01 人工介入恢复原子性 | ✅ 已实施 | `resume_with_payload` 单条件更新,并发恢复恰一个成功 |
|
||||
| P1-02 草稿乐观锁 + 发布版本号 | ✅ 已实施 | 草稿 CAS 条件更新;`next_version_number` 计数器分配版本号 |
|
||||
| P1-03 事件序号 + SSE 一致性 | ✅ 已实施 | `next_event_seq` 计数器;落库失败**丢弃不发布**(无幽灵事件) |
|
||||
| P1-06 测试纳入版本控制 | ✅ 已实施 | `.gitignore` 白名单 + 新增 `tests/test_workflow_concurrency.py`(SQL 并发) |
|
||||
| P2 §11.1 完整 JSON Schema 校验 | ✅ 已实施 | `schema_validation.py`(Draft202012,中文字段路径错误) |
|
||||
| P0-01 数据源越权 / HTTP 凭证外泄 | ❌ 不实施 | **内网可信环境**,按 ID 共享数据源、HTTP 节点访问任意内网地址是既有使用方式,加 owner 校验 / host 白名单会直接破坏现网用法,风险由部署方接受 |
|
||||
| P0-02 子工作流跨用户执行 | ❌ 不实施 | 同上:内网用户间按 ID 引用子工作流是预期用法 |
|
||||
| P1-04 智能体/技能最小权限 | ⏸ 暂缓 | 安全隔离类,本轮不做(用户决策:"安全性的问题咱们暂时不用做") |
|
||||
| P1-05 代码节点沙箱化 | ⏸ 暂缓 | 同上;`config.yaml` 中 `workflows.code.enabled` 保持默认 `false`,代码节点不开放 |
|
||||
| P2 §11.2 iframe Redis nonce | ⏸ 暂缓 | 单实例部署,进程内 replay cache + 60s TTL 足够;多实例化时再引入 Redis |
|
||||
|
||||
已实施项的回归测试集中在 `tests/test_workflow_concurrency.py`(内存 + 真实 SQLite 双轨),外加 `tests/test_workflow_runs.py` 的路由级 schema 校验回归;全部 88 个 workflow 测试通过。
|
||||
|
||||
## 1. 结论与整改目标
|
||||
|
||||
当前 Workflow Studio 的定义、发布、运行、SSE、SQL/HTTP、智能体、技能、人工介入等功能骨架已经具备,且基础测试可运行。但在共享资源授权、凭证使用边界、并发一致性、事件可靠性和代码执行隔离方面存在上线阻断问题。
|
||||
|
||||
本次整改的目标不是继续扩展节点功能,而是让现有节点满足以下底线:
|
||||
|
||||
1. 一个用户不能通过资源 ID、流程 JSON 或子工作流引用使用别人的私有资源。
|
||||
2. HTTP 凭证只能发送至管理员明确批准的目标域名,不能被流程编辑者导出到任意 URL。
|
||||
3. 人工介入恢复、草稿保存、发布版本和事件序号在多请求、多实例条件下保持一致。
|
||||
4. SSE 中显示的业务事件必须能够从数据库重放;不能出现只在内存短暂存在的“幽灵事件”。
|
||||
5. 智能体和技能节点只能使用流程配置所授予的能力。
|
||||
6. 代码节点只有在隔离沙箱中才能开启,不能在 Gateway 宿主机执行用户代码。
|
||||
|
||||
在 P0 项全部完成前,生产环境不得向普通用户开放:共享 SQL 数据源、共享 HTTP 凭证、子工作流跨用户引用和工作流编辑/执行权限。
|
||||
|
||||
## 2. 总体整改策略
|
||||
|
||||
### 2.1 建立运行时资源授权边界
|
||||
|
||||
当前 API 层可以识别当前用户,但执行器通常只持有 `source_id`、`workflow_id` 或 `agent_id`。整改后必须将“谁在执行”作为运行时一等参数,从创建运行记录开始贯穿到每一个节点。
|
||||
|
||||
新增领域对象,建议放在 `packages/harness/deerflow/workflows/`:
|
||||
|
||||
```python
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WorkflowExecutionPrincipal:
|
||||
"""工作流运行时的可信执行身份,由 Gateway 创建,不接受前端传入。"""
|
||||
|
||||
user_id: str
|
||||
is_admin: bool = False
|
||||
tenant_id: str | None = None
|
||||
```
|
||||
|
||||
运行创建时将发起人固化在 `workflow_runs.owner_id`,恢复、重试和子工作流也沿用原始运行的 `owner_id`,不得使用前端请求中的任意 `user_id`。
|
||||
|
||||
```python
|
||||
principal = WorkflowExecutionPrincipal(
|
||||
user_id=run.owner_id,
|
||||
is_admin=await permission_service.is_admin(run.owner_id),
|
||||
tenant_id=run.tenant_id,
|
||||
)
|
||||
```
|
||||
|
||||
所有需要使用资源的运行时入口必须接收 `principal`:
|
||||
|
||||
```python
|
||||
data_source_store.resolve_for_execution(source_id, principal)
|
||||
workflow_store.get_published_for_execution(workflow_id, version_id, principal)
|
||||
agent_store.get_visible(agent_id, principal.user_id)
|
||||
skill_registry.resolve_allowed(skill_id, principal, agent)
|
||||
```
|
||||
|
||||
资源不存在、无权限、已禁用三种情况对普通用户应统一返回 `RESOURCE_NOT_AVAILABLE`,避免通过 UUID 枚举资源。
|
||||
|
||||
### 2.2 采用“发布校验 + 运行时校验”的双层防线
|
||||
|
||||
发布时校验是为了尽早提示编辑者;运行时校验是为了防止发布后授权被收回、资源被禁用、恶意请求绕过 UI 或历史版本继续引用资源。
|
||||
|
||||
| 阶段 | 应做的事 |
|
||||
|---|---|
|
||||
| 保存草稿 | 校验节点结构、字段格式,不因暂时无权限而破坏用户草稿 |
|
||||
| 发布版本 | 校验每个外部资源对发布者是否可用,并将资源依赖写入版本元数据 |
|
||||
| 创建运行 | 再次校验工作流版本和根工作流权限 |
|
||||
| 节点执行 | 每次解析数据源、智能体、技能、子工作流时按 `principal` 校验 |
|
||||
| 重试/恢复 | 使用原 run 的身份和已发布版本,不重新信任浏览器传参 |
|
||||
|
||||
## 3. P0-01:数据源越权与 HTTP 凭证外泄
|
||||
|
||||
> **决策(2026-08-29):不实施。** 本部署运行于内网可信环境,用户之间按数据源 ID 共享、HTTP 节点访问任意内网 URL 是既有且必需的使用方式;owner 校验、baseUrl+path 拆分、host 白名单会使现网流程全部失效。风险由部署方明确接受("这个改了我们就没法用了")。
|
||||
|
||||
### 3.1 问题说明
|
||||
|
||||
当前运行时在 `app/gateway/workflow_executor.py` 中按数据源 ID 直接解析 DSN/凭证;持久化层的 `resolve_dsn` 也没有传入 owner 或授权上下文。HTTP 节点可由流程 JSON 提供任意 URL,再把已解析的认证头附加到该 URL。
|
||||
|
||||
因此存在两类风险:
|
||||
|
||||
1. 知道或获得他人私有数据源 ID 的用户,可能在自己的流程中引用该数据源。
|
||||
2. 可使用共享 HTTP 凭证的用户,可能把请求 URL 写为攻击者控制的公网域名,从而收到服务端附带的 API Key、Bearer Token 或 Basic Auth。
|
||||
|
||||
### 3.2 数据模型调整
|
||||
|
||||
在 `workflow_data_sources` 中增加显式共享与用途约束。字段命名可按现有 ORM 风格调整:
|
||||
|
||||
```sql
|
||||
ALTER TABLE workflow_data_sources
|
||||
ADD COLUMN visibility VARCHAR(16) NOT NULL DEFAULT 'private',
|
||||
ADD COLUMN allowed_user_ids JSON NULL,
|
||||
ADD COLUMN allowed_tenant_ids JSON NULL,
|
||||
ADD COLUMN allowed_hosts JSON NULL,
|
||||
ADD COLUMN allowed_methods JSON NULL;
|
||||
```
|
||||
|
||||
字段语义:
|
||||
|
||||
| 字段 | 含义 |
|
||||
|---|---|
|
||||
| `visibility` | `private`、`shared`、`system` |
|
||||
| `allowed_user_ids` | 共享给指定用户;第一期可先支持此项 |
|
||||
| `allowed_tenant_ids` | 多租户部署时按租户共享;单租户可暂不启用 |
|
||||
| `allowed_hosts` | HTTP 数据源允许请求的 hostname 白名单,必须填写 |
|
||||
| `allowed_methods` | HTTP 数据源可执行的方法白名单,例如 `GET`、`POST` |
|
||||
|
||||
SQL 数据源的数据库账号仍必须由 DBA 创建为只读账号;应用层 SQL 解析和只读 SQL policy 是第二道防线,不能替代数据库权限。
|
||||
|
||||
### 3.3 Store 接口调整
|
||||
|
||||
将现有无身份的解析接口废弃:
|
||||
|
||||
```python
|
||||
# 禁止继续在节点执行路径使用
|
||||
resolve_dsn(source_id: str)
|
||||
```
|
||||
|
||||
替换为:
|
||||
|
||||
```python
|
||||
async def resolve_for_execution(
|
||||
self,
|
||||
source_id: str,
|
||||
principal: WorkflowExecutionPrincipal,
|
||||
expected_kind: Literal["sql", "http"],
|
||||
) -> WorkflowDataSource:
|
||||
...
|
||||
```
|
||||
|
||||
实现规则:
|
||||
|
||||
1. 资源必须存在且 `enabled=True`。
|
||||
2. `kind` 必须与节点类型匹配。
|
||||
3. 允许 owner、被显式共享的用户、同租户授权用户或管理员访问。
|
||||
4. 禁止仅凭“已发布工作流”绕过资源权限;每一次运行均需检查。
|
||||
5. 无权限统一抛 `ResourceNotAvailableError`,日志中可记录真实原因和 source ID。
|
||||
|
||||
示例伪代码:
|
||||
|
||||
```python
|
||||
def can_execute(source: WorkflowDataSource, principal: WorkflowExecutionPrincipal) -> bool:
|
||||
if principal.is_admin:
|
||||
return True
|
||||
if source.owner_id == principal.user_id:
|
||||
return True
|
||||
if source.visibility == "shared" and principal.user_id in source.allowed_user_ids:
|
||||
return True
|
||||
if source.visibility == "system" and principal.tenant_id in source.allowed_tenant_ids:
|
||||
return True
|
||||
return False
|
||||
```
|
||||
|
||||
需要同步修改:
|
||||
|
||||
- `packages/harness/deerflow/persistence/workflow_data_sources/store.py`
|
||||
- `packages/harness/deerflow/persistence/workflow_data_sources/sql.py`
|
||||
- 内存 store 实现与测试 fake
|
||||
- `app/gateway/workflow_executor.py`
|
||||
- 使用数据源的 SQL / HTTP 节点执行器
|
||||
|
||||
### 3.4 HTTP 节点配置改造
|
||||
|
||||
不要继续允许流程节点同时填写任意完整 URL 与 `credentialRef`。推荐将 HTTP 数据源定义为受控目标,节点只填写相对路径:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "http",
|
||||
"dataSourceId": "crm-production-api",
|
||||
"method": "GET",
|
||||
"path": "/v1/customers",
|
||||
"query": {
|
||||
"page": "{{ inputs.page }}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
数据源的机密配置中保存:
|
||||
|
||||
```json
|
||||
{
|
||||
"baseUrl": "https://api.company.com",
|
||||
"allowedHosts": ["api.company.com"],
|
||||
"allowedMethods": ["GET", "POST"],
|
||||
"auth": {
|
||||
"type": "bearer",
|
||||
"secret": "加密存储"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
执行顺序必须固定:
|
||||
|
||||
1. 读取经授权的数据源。
|
||||
2. 变量渲染 `path`、query、body。
|
||||
3. 由 `baseUrl + path` 构造最终 URL,拒绝绝对 URL 和 `//host/path` 形式。
|
||||
4. 校验最终 host、端口、DNS 解析结果、方法、请求体上限。
|
||||
5. 最后才注入认证头。
|
||||
6. 每次重定向重新执行 host/SSRF 校验。
|
||||
7. 一旦发生跨 origin 重定向,移除 `Authorization`、Cookie、`X-API-Key` 等敏感头;建议默认拒绝跨 origin 重定向。
|
||||
|
||||
若业务确实需要任意 URL 调用,应提供另一种**无凭证 HTTP 节点**,并且其网络范围仍受全局 SSRF policy 约束。
|
||||
|
||||
### 3.5 验收测试
|
||||
|
||||
- 用户 A 创建私有 SQL 数据源;用户 B 使用其 ID 运行,返回 `RESOURCE_NOT_AVAILABLE`。
|
||||
- 用户 A 发布引用私有数据源的流程后,用户 B 即使复制 JSON 也不能运行。
|
||||
- 共享 HTTP 数据源只能请求 `allowed_hosts` 中的地址。
|
||||
- URL 模板渲染为不在白名单中的主机时被拒绝,且认证头没有发送。
|
||||
- 302/307 跨域跳转时凭证不会跟随;推荐断言请求整体被拒绝。
|
||||
- SQL 节点使用只读账号执行 `INSERT`、`CALL`、多语句均被数据库和 SQL policy 双重拒绝。
|
||||
|
||||
## 4. P0-02:子工作流跨用户执行
|
||||
|
||||
> **决策(2026-08-29):不实施。** 内网可信环境下按 workflow/version ID 跨用户引用子工作流是预期用法,运行时沿用原 run 的 `owner_id` 已满足功能正确性;完整 ACL 见 P0-01 同批决策,一并接受风险。
|
||||
|
||||
### 4.1 工作流 ACL
|
||||
|
||||
在 workflow 主表增加与数据源一致的共享模型:
|
||||
|
||||
```sql
|
||||
ALTER TABLE workflows
|
||||
ADD COLUMN visibility VARCHAR(16) NOT NULL DEFAULT 'private',
|
||||
ADD COLUMN allowed_user_ids JSON NULL,
|
||||
ADD COLUMN allowed_tenant_ids JSON NULL;
|
||||
```
|
||||
|
||||
建议第一期仅实现:`private`、`shared`、`system`。不要把“已发布”当作“所有用户可执行”;发布只代表版本不可变并可被授权对象使用。
|
||||
|
||||
### 4.2 接口改动
|
||||
|
||||
以下接口均应由 `principal` 过滤:
|
||||
|
||||
| 接口能力 | 整改要求 |
|
||||
|---|---|
|
||||
| 已发布工作流目录 | 仅返回当前用户可执行的工作流 |
|
||||
| 获取版本详情 | owner / 被分享对象 / 管理员才可读 |
|
||||
| 创建根运行 | 检查 workflow 与 version 的 ACL |
|
||||
| 子工作流节点 | 按父 run 的 principal 再次检查子流程 ACL |
|
||||
| 复制流程 | 允许复制公开定义不等于允许继承数据源和凭证引用 |
|
||||
|
||||
子工作流执行应改为:
|
||||
|
||||
```python
|
||||
child_version = await workflow_store.get_published_for_execution(
|
||||
workflow_id=node.workflow_id,
|
||||
version_id=node.version_id,
|
||||
principal=context.principal,
|
||||
)
|
||||
if child_version is None:
|
||||
raise ResourceNotAvailableError(node.workflow_id)
|
||||
```
|
||||
|
||||
并在 child run 写入:
|
||||
|
||||
```python
|
||||
parent_run_id=parent_run.id
|
||||
owner_id=parent_run.owner_id
|
||||
tenant_id=parent_run.tenant_id
|
||||
```
|
||||
|
||||
### 4.3 验收测试
|
||||
|
||||
- 用户 B 的资源目录不包含用户 A 的私有已发布流程。
|
||||
- 用户 B 手工构造 A 的 workflow/version ID 创建运行,被拒绝。
|
||||
- 用户 B 的父流程引用 A 的子流程,被拒绝。
|
||||
- A 共享子流程给 B 后,B 可运行;收回授权后,历史草稿仍可保存但新运行失败。
|
||||
|
||||
## 5. P1-01:人工介入恢复的原子性
|
||||
|
||||
> **状态(2026-08-29):已实施。** Store 新增 `resume_with_payload(run_id, resume_token, payload, *, context=None)`(SQL/memory 双实现),暂停侧 `set_awaiting_input` 扩展 `context` 参数一次性写入 checkpoint。路由 `resume_run` 与执行器 `_pause` 均已切换到单条条件更新;`consume_resume_token` 仅为兼容保留。回归测试 `tests/test_workflow_concurrency.py`(内存 + SQLite 并发恢复、错 token、单 winner)。
|
||||
|
||||
### 5.1 问题与目标
|
||||
|
||||
当前恢复操作先写 `resume_payload`,再消费 token。两个并发请求可能先后覆盖 payload,最终执行的数据与收到成功响应的请求不一致。
|
||||
|
||||
目标是把“校验 token、写入输入、清除 token、变更状态”为单条条件更新。
|
||||
|
||||
### 5.2 Store 方法
|
||||
|
||||
新增方法,替代 `update_run(...) + consume_resume_token(...)` 组合调用:
|
||||
|
||||
```python
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
) -> WorkflowRun | None:
|
||||
"""成功时返回已恢复的 run,token 无效或已消费时返回 None。"""
|
||||
```
|
||||
|
||||
SQL 示例,JSON 写法按当前数据库方言封装:
|
||||
|
||||
```sql
|
||||
UPDATE workflow_runs
|
||||
SET
|
||||
context_json = :new_context_json,
|
||||
resume_token = NULL,
|
||||
pending_human_node_id = NULL,
|
||||
status = 'queued',
|
||||
updated_at = NOW()
|
||||
WHERE id = :run_id
|
||||
AND status = 'awaiting_input'
|
||||
AND resume_token = :resume_token;
|
||||
```
|
||||
|
||||
应用层规则:
|
||||
|
||||
1. `rowcount != 1` 时返回 409,且不得投递调度器。
|
||||
2. `rowcount == 1` 后再投递调度器。
|
||||
3. 投递失败时保留 `queued` 状态,由扫描器补投;不要把 token 恢复。
|
||||
4. 对人工输入 payload 先完成 schema 校验、大小限制和脱敏审计,再进入事务。
|
||||
|
||||
暂停节点也应使用单次 store 调用,同时写入:状态、token、等待节点 ID 和上下文,避免执行器后续的独立 `update_run` 覆盖已恢复的数据。
|
||||
|
||||
### 5.3 验收测试
|
||||
|
||||
- 两个协程并发用同一 token 提交不同 payload,只有一个 200,另一个 409。
|
||||
- run context 中只存在 200 对应 payload。
|
||||
- worker 最终读取的 payload 与成功响应一致。
|
||||
- token 被消费后,多次重试均为 409。
|
||||
|
||||
## 6. P1-02:草稿与发布版本并发控制
|
||||
|
||||
> **状态(2026-08-29):已实施。** 草稿保存改为 `UPDATE ... WHERE id AND draft_revision = :expected`,rowcount=0 时回读当前 revision 抛 `WorkflowDraftConflictError`(409 带 `currentRevision`)。发布版本号改由 `workflow_definitions.next_version_number` 计数器 CAS 分配(`max(counter, MAX(version)+1)` 兼容存量行),唯一约束兜底、冲突重试而非 500。计数器列由迁移 `20260828_04_workflow_counter_columns.py` 增加(server_default,存量库由启动列同步自动补列)。回归测试:并发草稿双写单 winner、5 并发发布得相邻版本号、存量版本行计数器自愈。
|
||||
|
||||
### 6.1 草稿乐观锁
|
||||
|
||||
禁止“先读取 revision,再修改 ORM 对象,再 commit”的方式。改为带 revision 的条件更新:
|
||||
|
||||
```sql
|
||||
UPDATE workflows
|
||||
SET graph_json = :graph_json,
|
||||
draft_revision = draft_revision + 1,
|
||||
updated_at = NOW()
|
||||
WHERE id = :workflow_id
|
||||
AND draft_revision = :expected_revision;
|
||||
```
|
||||
|
||||
若受影响行数为零,则读取当前 revision,并返回:
|
||||
|
||||
```json
|
||||
{
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"currentRevision": 18
|
||||
}
|
||||
```
|
||||
|
||||
前端收到 409 后应提示“流程已被其他编辑者保存”,支持刷新、比较或另存副本;不能静默覆盖。
|
||||
|
||||
### 6.2 发布版本号
|
||||
|
||||
禁止 `MAX(version_number) + 1`。建议在 workflow 行维护 `next_version_number`,在一个事务内锁定并递增:
|
||||
|
||||
```sql
|
||||
SELECT next_version_number FROM workflows WHERE id = :id FOR UPDATE;
|
||||
INSERT INTO workflow_versions (..., version_number) VALUES (..., :next_number);
|
||||
UPDATE workflows
|
||||
SET next_version_number = next_version_number + 1
|
||||
WHERE id = :id;
|
||||
```
|
||||
|
||||
若现有数据库不方便 `FOR UPDATE`,可采用具备唯一约束的 sequence/counter 表,捕获唯一冲突后重试;不要将冲突直接抛为 500。
|
||||
|
||||
### 6.3 验收测试
|
||||
|
||||
- 两个独立 SQLAlchemy session 同时保存同一 revision,严格只有一个成功。
|
||||
- 两个发布请求同时执行,生成相邻但不同的版本号。
|
||||
- 冲突被转换为稳定的 409 业务错误,而非数据库异常。
|
||||
|
||||
## 7. P1-03:事件持久化、序号和 SSE 一致性
|
||||
|
||||
> **状态(2026-08-29):已实施(采用 7.2 计数器方案 + 7.3 的"丢弃不发布"策略)。** `workflow_runs.next_event_seq` 计数器在同一事务内 `UPDATE ... SET next_event_seq = next_event_seq + 1` 后读自增并插入事件,行写锁串行化并发追加(seq 严格递增、唯一、无空洞);唯一约束冲突时 `_resync_counter` 以独立事务回写 `MAX(seq)` 自愈存量行后重试。落库失败的事件**直接丢弃、绝不向 live hub 发布**(原负序号幽灵事件实现已删除),信封带 `eventPersistenceDegraded` 标记、`sink.degraded_count` 计数并记结构化告警日志;SSE 端点对无终态事件的终态 run 合成关闭帧。回归测试:10 并发追加 seq 连续唯一、存量事件计数器自愈、落库失败零发布。
|
||||
|
||||
### 7.1 改造原则
|
||||
|
||||
业务事件必须遵循:**获得持久序号 → 持久化成功 → 实时发布**。不能生成负序号临时事件,也不能让前端看到无法通过重连补回的事件。
|
||||
|
||||
### 7.2 序号分配
|
||||
|
||||
不要使用 `MAX(seq) + 1`。在 `workflow_runs` 增加计数器:
|
||||
|
||||
```sql
|
||||
ALTER TABLE workflow_runs
|
||||
ADD COLUMN next_event_seq BIGINT NOT NULL DEFAULT 1;
|
||||
```
|
||||
|
||||
追加事件时在同一个数据库事务中:
|
||||
|
||||
1. 对 run 原子递增 `next_event_seq` 并得到当前 seq。
|
||||
2. 以该 seq 插入 `workflow_events`。
|
||||
3. 提交事务。
|
||||
4. 仅提交成功后调用 `live_hub.publish(event)`。
|
||||
|
||||
不同数据库可选择 `UPDATE ... RETURNING` 或者用 run 行锁实现,但不可回退为 `MAX(seq)` 查询。
|
||||
|
||||
### 7.3 持久化失败处理
|
||||
|
||||
事件库短暂失败时:
|
||||
|
||||
1. 在有限次数内重试;
|
||||
2. 仍失败则将运行标记为 `retryable_failure` 或 `paused_system`;
|
||||
3. 记录包含 `run_id`、node ID、异常分类的结构化日志和告警;
|
||||
4. SSE 至少发送一个已成功持久化的终态或系统状态事件;
|
||||
5. 用户重连后只能从数据库事件日志恢复,不依赖当前进程内存。
|
||||
|
||||
不要使用“事件落库失败后仍向 live hub 发负数 seq”的实现,因为按 cursor 追尾的 SSE 客户端通常会忽略它。
|
||||
|
||||
### 7.4 验收测试
|
||||
|
||||
- 并行节点大量写事件,序号严格递增且无重复。
|
||||
- 模拟一次插入事件失败,前端不会收到不可回放事件。
|
||||
- 客户端从 `Last-Event-ID=N` 重连,得到所有 `seq>N` 的事件且顺序一致。
|
||||
- 多实例下任一实例订阅 SSE,均可通过数据库补放完整事件。
|
||||
|
||||
## 8. P1-04:智能体和技能最小权限
|
||||
|
||||
> **决策(2026-08-29):暂缓。** 属安全隔离类项目,本轮整改只做功能性优化(用户决策:"P1-04/P1-05 偏安全隔离不用做,安全性的问题咱们暂时不用做,咱们只改功能性的,优化的")。后续若对外开放多租户再评估。
|
||||
|
||||
### 8.1 智能体可见性
|
||||
|
||||
`agent` 节点收到 `agent_id` 后,必须在 `WorkflowAgentRunner` 内部再次调用可见性查询:
|
||||
|
||||
```python
|
||||
agent = await agent_store.get_visible(
|
||||
agent_id=node.agent_id,
|
||||
user_id=context.principal.user_id,
|
||||
)
|
||||
if agent is None or not agent.enabled:
|
||||
raise ResourceNotAvailableError(node.agent_id)
|
||||
```
|
||||
|
||||
不能仅依赖资源目录接口已过滤,也不能仅依赖发布阶段校验。
|
||||
|
||||
### 8.2 技能白名单
|
||||
|
||||
现有将 `skill_names` 传给 Lead Agent 的方式必须核实是否被真正消费。建议新增语义明确的运行时参数:
|
||||
|
||||
```python
|
||||
configurable["workflow_allowed_skill_ids"] = selected_skill_ids
|
||||
```
|
||||
|
||||
并在 Lead Agent 加载技能、MCP 工具和工具调用路由之前强制过滤:
|
||||
|
||||
```python
|
||||
requested = set(configurable.get("workflow_allowed_skill_ids", []))
|
||||
available = [skill for skill in all_enabled_skills if skill.id in requested]
|
||||
```
|
||||
|
||||
还应校验:
|
||||
|
||||
1. 选中的 skill 对当前用户可见。
|
||||
2. skill 是否允许在 workflow 场景执行。
|
||||
3. skill 的网络、数据库、文件权限是否与工作流 principal 相兼容。
|
||||
4. 系统级高风险 skill 是否只能由管理员在数据源/策略中启用。
|
||||
|
||||
### 8.3 验收测试
|
||||
|
||||
- 用户手工填写不可见 agent ID,运行被拒绝。
|
||||
- 未选中的技能不出现在模型可调用工具列表中。
|
||||
- 提示词诱导模型调用未选技能,调用被拒绝且记录审计事件。
|
||||
- 选中的技能正常可执行,节点事件中仅暴露必要的脱敏参数。
|
||||
|
||||
## 9. P1-05:代码节点沙箱化
|
||||
|
||||
> **决策(2026-08-29):暂缓,保持现状即安全。** 本轮不迁移到容器沙箱;`config.yaml` 中 `workflows.code.enabled` 保持默认 `false`(代码节点全局关闭,`tests/test_workflow_engine.py::test_code_node_disabled_by_default` 已锁定该默认值),如需启用由运维在受限宿主上自行开启。现有 `code_runner.py` 子进程加固(净化环境、临时目录、rlimit、断网、超时杀进程组)在可信内网下作为过渡方案。
|
||||
|
||||
### 9.1 上线策略
|
||||
|
||||
在 DeerFlow 沙箱实现并通过安全验证前,保持:
|
||||
|
||||
```yaml
|
||||
workflows:
|
||||
code:
|
||||
enabled: false
|
||||
```
|
||||
|
||||
不得以本地 `exec`、`subprocess` 或 monkey-patch `socket` 作为生产隔离方案。用户代码可借助原生扩展、子进程、文件系统或环境变量绕过此类限制。
|
||||
|
||||
### 9.2 目标接口
|
||||
|
||||
代码节点改为只调用现有 DeerFlow sandbox 抽象,而不自行启动宿主解释器:
|
||||
|
||||
```python
|
||||
result = await sandbox.run(
|
||||
code=rendered_code,
|
||||
language="python",
|
||||
timeout_seconds=30,
|
||||
memory_limit_mb=256,
|
||||
cpu_limit=1,
|
||||
max_processes=8,
|
||||
network_enabled=False,
|
||||
writable_paths=["/mnt/user-data/workflows/{run_id}"],
|
||||
)
|
||||
```
|
||||
|
||||
沙箱最低要求:
|
||||
|
||||
- 独立容器或受限命名空间,不能读取 Gateway 项目、环境变量与宿主文件。
|
||||
- 默认无网络;若未来允许网络,也需复用 HTTP 节点的数据源、SSRF 和凭证策略。
|
||||
- CPU、内存、执行时长、磁盘、进程数和文件数限制。
|
||||
- stdout/stderr 使用流式读取并在读取中截断,例如各 1MB;禁止 `communicate()` 后再截断。
|
||||
- 制品只能输出到工作流专属目录,并通过既有 artifact 机制登记。
|
||||
- 记录代码 hash、镜像版本、资源消耗、退出码,日志不得泄露源代码中的密钥。
|
||||
|
||||
### 9.3 验收测试
|
||||
|
||||
- 代码读取 `/etc`、项目目录、环境变量失败。
|
||||
- 导入 socket、创建子进程、连接外网失败。
|
||||
- 死循环超时结束,运行正确失败或可重试。
|
||||
- 大量 stdout/stderr 不造成 Gateway 内存膨胀。
|
||||
- 制品只能写入限定目录。
|
||||
|
||||
## 10. P1-06:测试纳入版本控制与 CI
|
||||
|
||||
> **状态(2026-08-29):已实施。** 采用方案 1(白名单):仓库根 `.gitignore` 对 `tests/test_workflow_{schemas,definitions,engine,runs}.py` 逐个 `!` 白名单,并新增 `tests/test_workflow_concurrency.py`(本轮整改的内存 + 真实 SQLite 并发回归,含 P1-01/02/03 与 schema 校验)。CI 需在该清单基础上加 `tests/test_workflow_concurrency.py`。
|
||||
|
||||
当前工作流测试文件位于后端 `tests/`,但仓库根目录的 `.gitignore` 规则会忽略这一目录内容。必须在合并前处理,否则本地通过的测试不会进入提交和 CI。
|
||||
|
||||
推荐方案:
|
||||
|
||||
1. 调整 `.gitignore`,仅忽略真正的生成物,不要忽略后端测试源码;或
|
||||
2. 若忽略规则短期不能改,使用 `git add -f` 纳入现有测试文件,但仍应补充注释说明原因;
|
||||
3. CI 显式执行:
|
||||
|
||||
```bash
|
||||
cd offline-backend-20260512/backend
|
||||
PYTHONPATH=. uv run pytest \
|
||||
tests/test_workflow_schemas.py \
|
||||
tests/test_workflow_definitions.py \
|
||||
tests/test_workflow_engine.py \
|
||||
tests/test_workflow_runs.py -v
|
||||
```
|
||||
|
||||
4. 新增 SQL 集成测试 job,至少覆盖真实事务并发;不能只依赖 memory store。
|
||||
|
||||
## 11. P2:契约校验与 iframe 票据
|
||||
|
||||
> **状态(2026-08-29):§11.1 已实施;§11.2 暂缓**(单实例部署,进程内 replay cache + 60s TTL 足够,多实例化时再引入 Redis)。
|
||||
|
||||
### 11.1 完整 JSON Schema 校验
|
||||
|
||||
> **实施说明(2026-08-29)**:新增纯函数模块 `packages/harness/deerflow/workflows/schema_validation.py` 的 `schema_issues(value, schema)`(Draft202012,中文短消息、字段路径、去重、上限 20 条、schema 本身非法时返回友好错误而非 500)。接入点:路由 `start_run`(启动前 400 `WORKFLOW_INPUT_INVALID` + `details.errors`)、引擎 Start/Output 节点(`WORKFLOW_INPUT_INVALID` / `WORKFLOW_OUTPUT_SCHEMA_MISMATCH`)。空 schema 不强制(未声明即不校验)。回归测试见 `tests/test_workflow_concurrency.py` 与 `tests/test_workflow_runs.py::test_typed_input_schema_violations_return_structured_400`。
|
||||
|
||||
输入节点和输出节点不应只校验 `required` 字段。应使用完整 JSON Schema validator,至少支持:`type`、`enum`、`items`、`properties`、`required`、`additionalProperties`、嵌套对象和数组。
|
||||
|
||||
```python
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
validator = Draft202012Validator(schema)
|
||||
errors = sorted(validator.iter_errors(value), key=lambda item: list(item.path))
|
||||
if errors:
|
||||
raise WorkflowSchemaValidationError(
|
||||
errors=[{"path": list(err.path), "message": err.message} for err in errors]
|
||||
)
|
||||
```
|
||||
|
||||
错误响应需要包含节点 ID、字段路径和可展示的消息;不应把内部 Python trace 返回给前端。
|
||||
|
||||
### 11.2 iframe 一次性票据
|
||||
|
||||
“一次性”语义不能使用单进程内存字典。若部署多实例,应使用 Redis 或数据库的原子写入:
|
||||
|
||||
```text
|
||||
SET workflow_embed_nonce:{jti} consumed NX EX 60
|
||||
```
|
||||
|
||||
只有返回成功的实例可以继续验证。服务端还必须读取真实请求 `Origin`,并要求以下三者一致:
|
||||
|
||||
1. ticket 签发时绑定的 origin;
|
||||
2. 浏览器请求头中的 `Origin`;
|
||||
3. 后台 `workflows.embed.allowed_origins` 白名单。
|
||||
|
||||
不能只比较前端 body/query 参数中的 origin。
|
||||
|
||||
## 12. 建议实施顺序与分批发布
|
||||
|
||||
> 2026-08-29 实况:第二批全部落地;第四批仅 §11.1 落地(§11.2 暂缓);第一批(P0-01/02)经部署方决策不实施;第三批(P1-04/05)暂缓。
|
||||
|
||||
### 第一批:安全阻断项
|
||||
|
||||
范围:资源 principal、数据源授权、HTTP host 绑定、子工作流 ACL。
|
||||
|
||||
发布策略:先迁移数据库字段,默认所有历史资源为 `private`;只允许管理员手工将必要资源改为 shared。此批完成并通过越权测试后,才允许开放 SQL/HTTP/子工作流节点。
|
||||
|
||||
### 第二批:一致性与可靠性
|
||||
|
||||
范围:人工恢复原子操作、草稿 CAS、发布版本锁、事件序号与持久化失败策略。
|
||||
|
||||
发布策略:需要停写窗口或兼容性迁移时,先双写/双读一版;对运行中的旧 run 保持旧 schema 可读,新增 run 使用新序号和新恢复逻辑。
|
||||
|
||||
### 第三批:能力隔离
|
||||
|
||||
范围:agent 可见性、skill 白名单、代码节点沙箱。
|
||||
|
||||
发布策略:代码节点继续全局关闭;先灰度给管理员和测试租户。技能节点在完成真实白名单前,不得宣称“只调用所选技能”。
|
||||
|
||||
### 第四批:契约与嵌入完善
|
||||
|
||||
范围:JSON Schema 完整校验、iframe Redis nonce、接口错误码统一、运维监控。
|
||||
|
||||
## 13. 发布准入清单
|
||||
|
||||
> 2026-08-29 更新:本清单按原始审计口径保留。结合「0. 整改决策记录」,P0 两项经部署方决策**不实施**(内网可信环境,风险已接受),与 P0 相关的准入项在内网部署中视为豁免;✅ 为本轮已达成项。
|
||||
|
||||
以下条件全部满足才可开放给普通用户:
|
||||
|
||||
- [ ] 所有资源解析接口都需要 `WorkflowExecutionPrincipal`。(❌ 不实施——内网可信环境,见 §3/§4 决策)
|
||||
- [ ] 私有数据源、私有工作流、私有智能体均有跨用户拒绝测试。(❌ 不实施——同上)
|
||||
- [ ] HTTP 凭证绑定 host/method,跨域重定向不携带敏感认证头。(❌ 不实施——内网需访问任意地址,见 §3 决策)
|
||||
- [ ] SQL 数据源使用数据库只读账号,且连接/查询审计可追踪到 run ID。(部分:SQL 只读校验已有;账号隔离不在本轮范围)
|
||||
- [ ] 子工作流按运行身份校验 ACL。(❌ 不实施——见 §4 决策)
|
||||
- [x] 人工输入恢复是单事务条件更新,并发测试通过。(✅ `test_workflow_concurrency.py`)
|
||||
- [x] 草稿保存和发布版本具备数据库级并发保护。(✅ 同上)
|
||||
- [x] 事件序号单调、唯一、可重放;没有负序号临时事件。(✅ 同上)
|
||||
- [ ] agent / skill 白名单在实际工具加载链路生效。(⏸ 暂缓——P1-04)
|
||||
- [x] 代码节点运行在受限 sandbox;否则保持关闭。(✅ 保持关闭:`workflows.code.enabled: false` 为默认且被测试锁定)
|
||||
- [x] 所有 workflow 测试文件已纳入 Git 与 CI。(✅ `.gitignore` 白名单 + 新增并发回归)
|
||||
- [x] 安全测试、SQL 并发集成测试、SSE 重连测试全部通过。(✅ 88 个 workflow 测试全绿;安全类测试随 P0/P1-04 决策豁免)
|
||||
|
||||
## 14. 相关代码入口
|
||||
|
||||
| 主题 | 当前主要代码入口 |
|
||||
|---|---|
|
||||
| 路由与运行 API | `app/gateway/routers/workflow_runs.py` |
|
||||
| 工作流执行与依赖装配 | `app/gateway/workflow_executor.py` |
|
||||
| 智能体/技能运行 | `app/gateway/workflow_agent_runner.py` |
|
||||
| 数据源 API | `app/gateway/routers/workflow_data_sources.py` |
|
||||
| 工作流资源目录 | `app/gateway/routers/workflow_resources.py` |
|
||||
| iframe ticket | `app/gateway/routers/workflow_embed.py` |
|
||||
| 数据源持久化 | `packages/harness/deerflow/persistence/workflow_data_sources/` |
|
||||
| 工作流/版本持久化 | `packages/harness/deerflow/persistence/workflows/` |
|
||||
| 运行与租约持久化 | `packages/harness/deerflow/persistence/workflow_runs/` |
|
||||
| 事件持久化 | `packages/harness/deerflow/persistence/workflow_events/` |
|
||||
| SQL/HTTP 节点 | `packages/harness/deerflow/workflows/nodes/integrations.py` |
|
||||
| 事件 sink | `packages/harness/deerflow/workflows/runtime/sink.py` |
|
||||
| 代码节点运行器 | `packages/harness/deerflow/workflows/runtime/code_runner.py` |
|
||||
| 工作流安全策略 | `packages/harness/deerflow/workflows/security/` |
|
||||
| 数据库迁移 | `packages/harness/deerflow/persistence/migrations/` |
|
||||
| 现有工作流测试 | `tests/test_workflow_*.py` |
|
||||
|
||||
## 15. 非目标与注意事项
|
||||
|
||||
- 本整改不改变 Coze Studio 画布的基础交互,也不要求先改前端;前端仅需后续适配资源权限失败、409 冲突和结构化 schema 错误。
|
||||
- 不要把资源密钥、数据库 DSN、人工输入 token 或工具原始参数写入前端 SSE 事件。
|
||||
- 不要依赖“前端资源选择器只显示可见资源”作为授权机制;浏览器请求和画布 JSON 均视为不可信输入。
|
||||
- 新字段上线后,历史资源默认按私有处理,避免一次迁移导致所有历史凭证意外共享。
|
||||
- 每项整改完成后都要更新 `WORKFLOW_STUDIO_BACKEND_DEV_ZH.md` 中对应的“已完成”状态,避免文档声明与实际安全边界不一致。
|
||||
@@ -0,0 +1,486 @@
|
||||
# Workflow Studio:Coze 独立前端对接 DeerFlow 后端适配开发文档
|
||||
|
||||
> 状态:待实施
|
||||
> 日期:2026-08-29
|
||||
> 前端范围:F:/react01/code-stutsiu/frontend/apps/coze-studio/src/workflow-standalone/
|
||||
> 后端范围:offline-backend-20260512/backend/
|
||||
> 关联文档:
|
||||
>
|
||||
> - docs/WORKFLOW_STUDIO_BACKEND_DEV_ZH.md:工作流运行时总体设计与已有能力。
|
||||
> - docs/WORKFLOW_STUDIO_BACKEND_REMEDIATION_ZH.md:安全、并发、租约与事件持久化整改。
|
||||
>
|
||||
> 本文只处理“当前独立 Coze 页面与真实 DeerFlow 后端之间”的适配,不重新设计工作流运行内核。
|
||||
|
||||
## 1. 目标与结论
|
||||
|
||||
当前独立页的交互、三栏布局、资源卡、运行历史、SSE reducer 与 mock 演示已可用,但其保存、校验、发布和部分运行事件仍使用 mock 协议。真实 DeerFlow 后端采用 WorkflowGraph v1.0 和持久化运行时,两者不能直接互换。
|
||||
|
||||
本次适配目标:
|
||||
|
||||
1. Coze 画布能够保存、重新加载、校验、发布,并由 DeerFlow 运行。
|
||||
2. 画布上的 Agent、Skill、SQL、HTTP、Code、Input、Output 节点都有真实、可审计的执行配置。
|
||||
3. 左侧对话运行和右侧执行过程能从同一条 DeerFlow SSE 事件流稳定恢复工具步骤、人工介入与文件制品。
|
||||
4. 不把 Coze 的内部数值节点类型、任意画布 JSON、凭据明文或完整工具参数泄漏到运行时协议中。
|
||||
|
||||
结论:不能仅在现有 POST /api/workflows/{id}/runs 中增加几个兼容字段。必须先建立“画布表示”和“执行图表示”的明确边界;否则保存成功的流程仍无法发布或运行。
|
||||
|
||||
## 2. 当前差异清单
|
||||
|
||||
| 编号 | 当前独立页行为 | DeerFlow 当前行为 | 影响 | 所属 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| A01 | 保存 body 为 schemaJson,读取 workflowId / draftRevision / schemaJson | 草稿接口要求 graph,读取返回 id / draft_graph / draft_revision | 自动保存、冲突恢复、发布均无法真实联调 | 前端主改,后端提供工作室 DTO |
|
||||
| A02 | 校验期待 valid 字段 | 后端返回 ok 字段 | 前端始终判定校验不通过 | 前后端契约统一 |
|
||||
| A03 | Coze JSON 被原样持久化 | 后端运行时只接受 WorkflowGraph v1.0 | Coze 数值节点类型不可执行 | 前端编译,后端持久化与验证 |
|
||||
| A04 | 拖入资源只预填节点标题 | Agent / Skill 执行器要求 agentId、skillNames、promptTemplate 等 | 资源卡看起来可用,真实运行必失败 | 前端表单 + 后端发布校验 |
|
||||
| A05 | 工具事件读取 name、status | 后端发送 toolName、ok | 工具步骤丢失,失败状态错误 | 后端规范化,前端兼容旧字段 |
|
||||
| A06 | 人工介入读取顶层 prompt / formSchema | 后端放在 pendingInput 内 | 介入卡没有问题文本和表单 | 后端事件扁平化,前端兼容 |
|
||||
| A07 | 文件预览请求单个 artifact 内容接口 | 后端只提供制品列表 | 文件卡可见但无法打开或下载 | 后端新增制品内容接口 |
|
||||
| A08 | 资源 hook 请求 credentials 目录 | 后端没有该路由 | 整个资源加载 Promise.all 可能显示失败 | 后端补只读引用目录,或前端暂停请求 |
|
||||
| A09 | UI 可选“当前草稿试运行”和 mock 场景 | 后端忽略 target / scenario,只运行发布版本 | 用户以为执行草稿,实际执行旧版本 | 前端先隐藏;后端后续可实现草稿快照 |
|
||||
| A10 | 连续输入会创建新的 run | 后端无 conversation / turn 模型 | 不是 ChatDev 式多轮会话 | 二期可选能力 |
|
||||
|
||||
## 3. 必须冻结的总体设计
|
||||
|
||||
### 3.1 两种表示,两个职责
|
||||
|
||||
同一工作流必须保留两份有明确用途的数据,不能让任何一份冒充另一份。
|
||||
|
||||
| 数据 | 责任方 | 用途 | 是否可直接执行 |
|
||||
| --- | --- | --- | --- |
|
||||
| canvasSchemaJson | Coze 前端产生;后端仅作为受限 JSON 文档保存 | 画布恢复、节点位置、Coze 表单内部字段、编辑体验 | 否 |
|
||||
| executionGraph | 前端适配器编译;后端以 WorkflowGraph 校验并版本化 | 发布校验、权限检查、调度、恢复、SSE、审计 | 是 |
|
||||
|
||||
约束:
|
||||
|
||||
1. DeerFlow harness 只接收 executionGraph,绝不解析 Coze StandardNodeType 或 Coze 内部服务字段。
|
||||
2. canvasSchemaJson 不进入 SSE,不进入运行快照,不作为表达式或节点配置来源。
|
||||
3. 发布版本至少冻结 executionGraph;建议同时保存其来源 canvasSchemaJson 的哈希,便于审计与排错。
|
||||
4. Coze 到 WorkflowGraph 的逐节点转换放在前端 adapters/coze-canvas-adapter.ts 或其同级编译器中。后端不耦合 Coze 的数值节点类型。
|
||||
5. 后端应拒绝“只有 canvasSchemaJson、没有合法 executionGraph”的发布请求,不能静默将其当作空图。
|
||||
|
||||
### 3.2 推荐的工作室适配层
|
||||
|
||||
保持现有 /api/workflows 的标准运行时 API 语义,不在其核心请求中混入 Coze 专属字段。新增一层仅面向独立页的 studio facade:
|
||||
|
||||
GET /api/workflows/{workflow_id}/studio-document
|
||||
PUT /api/workflows/{workflow_id}/studio-draft
|
||||
POST /api/workflows/{workflow_id}/studio-validate
|
||||
POST /api/workflows/{workflow_id}/studio-publish
|
||||
|
||||
现有以下端点继续作为运行时和其他客户端的标准接口:
|
||||
|
||||
POST /api/workflows/{workflow_id}/runs
|
||||
GET /api/workflows/{workflow_id}/runs
|
||||
GET /api/workflows/runs/{run_id}/events
|
||||
GET /api/workflows/runs/{run_id}/stream
|
||||
|
||||
这样既避免污染通用 API,也让前端能获得它需要的 camelCase DTO。
|
||||
|
||||
## 4. Studio facade API 契约
|
||||
|
||||
### 4.1 获取文档
|
||||
|
||||
请求:
|
||||
|
||||
GET /api/workflows/{workflow_id}/studio-document
|
||||
|
||||
响应:
|
||||
|
||||
{
|
||||
"workflowId": "wf_001",
|
||||
"name": "周报生成",
|
||||
"draftRevision": 12,
|
||||
"publishedVersionId": "ver_009",
|
||||
"canvasSchemaJson": "{...}",
|
||||
"executionGraph": { "schemaVersion": "1.0", "id": "wf_001", "nodes": [], "edges": [] },
|
||||
"updatedAt": "2026-08-29T10:00:00+08:00"
|
||||
}
|
||||
|
||||
要求:
|
||||
|
||||
- 从现有 definition 的 id、draft_revision、draft_graph、published_version_id 等字段适配生成。
|
||||
- canvasSchemaJson 为空时返回空画布 JSON,而不是 null。
|
||||
- 权限沿用现有 owner/admin 规则。
|
||||
- 不返回 DSN、HTTP 请求头、resume token、完整 agent 配置或任何私密字段。
|
||||
|
||||
### 4.2 保存草稿
|
||||
|
||||
请求:
|
||||
|
||||
PUT /api/workflows/{workflow_id}/studio-draft
|
||||
|
||||
{
|
||||
"expectedRevision": 12,
|
||||
"canvasSchemaJson": "{...}",
|
||||
"executionGraph": {
|
||||
"schemaVersion": "1.0",
|
||||
"id": "wf_001",
|
||||
"nodes": [],
|
||||
"edges": []
|
||||
}
|
||||
}
|
||||
|
||||
响应:
|
||||
|
||||
{ "revision": 13, "updatedAt": "2026-08-29T10:01:00+08:00" }
|
||||
|
||||
规则:
|
||||
|
||||
1. 先做 WorkflowGraph 结构解析;无法解析返回 400,错误码 WORKFLOW_SCHEMA_INVALID。
|
||||
2. 画布 JSON 只做大小、JSON 格式、深度和危险字段控制,不做执行解释。
|
||||
3. canvasSchemaJson 与 executionGraph 必须在同一次 CAS 更新中落库,不能出现画布和执行图不同 revision。
|
||||
4. revision 冲突返回 409:
|
||||
|
||||
{
|
||||
"code": "WORKFLOW_DRAFT_CONFLICT",
|
||||
"message": "草稿版本冲突,请刷新后重试或另存副本",
|
||||
"currentRevision": 13
|
||||
}
|
||||
|
||||
5. 不建议让旧 PUT /draft 同时接受 schemaJson 与 graph。若为灰度必须暂时兼容,兼容逻辑只能位于 facade 路由,且必须在一个发布周期后删除。
|
||||
|
||||
### 4.3 校验与发布
|
||||
|
||||
校验请求:
|
||||
|
||||
POST /api/workflows/{workflow_id}/studio-validate
|
||||
{ "executionGraph": { ... } }
|
||||
|
||||
校验响应:
|
||||
|
||||
{
|
||||
"valid": false,
|
||||
"issues": [
|
||||
{
|
||||
"code": "WORKFLOW_RESOURCE_MISSING",
|
||||
"nodeId": "agent_writer",
|
||||
"field": "config.agentId",
|
||||
"message": "未选择可访问的智能体"
|
||||
}
|
||||
],
|
||||
"graphHash": null
|
||||
}
|
||||
|
||||
发布请求:
|
||||
|
||||
POST /api/workflows/{workflow_id}/studio-publish
|
||||
{ "expectedRevision": 13, "changeNote": "配置报告智能体" }
|
||||
|
||||
发布响应:
|
||||
|
||||
{ "versionId": "ver_010", "publishedAt": "2026-08-29T10:02:00+08:00" }
|
||||
|
||||
发布必须复用标准发布版本表和不可变 graph 快照。facade 只负责 DTO 命名、画布关联和调用标准发布服务,不能复制一套版本逻辑。
|
||||
|
||||
## 5. 后端实施项
|
||||
|
||||
### 5.1 持久化 canvas schema 与草稿原子更新
|
||||
|
||||
现有 workflow_definitions 只有 draft_graph_json。新增独立字段,例如 draft_canvas_schema_json;不要把 Coze 画布塞进 draft_graph_json。
|
||||
|
||||
需要修改:
|
||||
|
||||
| 路径 | 改动 |
|
||||
| --- | --- |
|
||||
| packages/harness/deerflow/persistence/workflows/model.py | WorkflowDefinitionRow 增加 draft_canvas_schema_json;如需审计可增加 published_canvas_schema_hash |
|
||||
| packages/harness/deerflow/persistence/migrations/versions/ | 新增 Alembic migration,存量行默认 {} |
|
||||
| packages/harness/deerflow/persistence/workflows/base.py | save_studio_draft 或扩展 save_draft 的原子接口,输入两份文档和 expected_revision |
|
||||
| packages/harness/deerflow/persistence/workflows/sql.py | 单条 CAS UPDATE 同时更新两份文档、revision、updated_at |
|
||||
| packages/harness/deerflow/persistence/workflows/memory.py | 同步实现,保证测试环境语义一致 |
|
||||
|
||||
禁止:
|
||||
|
||||
- 先保存 canvas、后保存 graph 的两次写入。
|
||||
- 在 SQL store 之外用“先读 revision、再无条件写”的方式实现 CAS。
|
||||
- 把 canvasSchemaJson 输出到运行事件或 run context。
|
||||
|
||||
### 5.2 Studio facade 路由与 DTO
|
||||
|
||||
新增 app/gateway/routers/workflow_studio.py,负责:
|
||||
|
||||
1. studio-document、studio-draft、studio-validate、studio-publish 四个端点。
|
||||
2. camelCase DTO 与现有 persistence snake_case 字段之间的映射。
|
||||
3. 统一错误结构和 owner/admin 鉴权。
|
||||
4. 调用通用 WorkflowGraph 结构校验与 app 层资源校验。
|
||||
|
||||
在 app/gateway/app.py 或当前工作流 router 注册位置挂载该 router。不要让 packages/harness/deerflow 导入 app.gateway。
|
||||
|
||||
前端移交项:
|
||||
|
||||
- api/workflows.ts 改为调用 studio facade。
|
||||
- coze-canvas-adapter.ts 产出 executionGraph,不再只做 JSON.stringify。
|
||||
- 画布新增或修改节点时,同时更新可编译的节点配置。
|
||||
|
||||
### 5.3 发布前资源和权限校验
|
||||
|
||||
现有 validator 支持 resource_checker 钩子,但通用 harness 不应直接导入应用层的 agent、skill、data-source store。应在 app 层构造 checker 后传入。
|
||||
|
||||
在 studio-validate、studio-publish 和标准 publish 入口都执行:
|
||||
|
||||
| 节点 | 必检字段 | 后端校验 |
|
||||
| --- | --- | --- |
|
||||
| agent | agentId、promptTemplate、responseSchema | 智能体存在、调用者可见、提示词非空、输出 schema 合法 |
|
||||
| skill | mode、agentId、skillNames、promptTemplate | 技能存在且启用;agent_skill 必须属于智能体白名单;callable_skill 未开放时明确拒绝 |
|
||||
| sql_read | dataSourceId、queryTemplate、parameters | 数据源可见、只读、表白名单、SQL 解析与 maxRows 上限 |
|
||||
| http | dataSourceId 或 credentialRef、method、path、headers 映射 | 资源可见、方法白名单、URL/内网策略、凭据引用可用 |
|
||||
| code | language、source、timeoutSeconds、output mapping | 沙箱已启用、语言白名单、超时和输出目录限制 |
|
||||
| output | schema、field mapping | 输出字段可解析,输出 schema 可校验 |
|
||||
| subworkflow | workflowId、versionId | 子工作流已发布、调用者有权限、深度限制 |
|
||||
|
||||
校验必须返回 nodeId 与字段路径,便于前端定位 Coze 节点表单。运行时仍保留二次校验,防止资源被禁用或权限在发布后撤销。
|
||||
|
||||
### 5.4 资源目录与凭据引用
|
||||
|
||||
保留现有:
|
||||
|
||||
- app/gateway/routers/workflow_resources.py:Agent、Skill、数据源目录。
|
||||
- app/gateway/routers/workflow_data_sources.py:SQL/HTTP 数据源创建、加密与只读策略。
|
||||
|
||||
补充:
|
||||
|
||||
GET /api/workflows/resources/credentials
|
||||
|
||||
响应只允许:
|
||||
|
||||
{
|
||||
"credentials": [
|
||||
{
|
||||
"credentialId": "http_source_001",
|
||||
"name": "CRM 接口密钥",
|
||||
"type": "http_headers",
|
||||
"sourceKind": "http"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
绝不返回 headers、token、DSN、加密密文或可逆提示。若一期 UI 没有凭据选择器,前端必须停止请求该端点,不能让一个 404 使整个资源面板显示“加载失败”。
|
||||
|
||||
### 5.5 运行目标、输入 schema 与草稿试运行
|
||||
|
||||
一期决策:
|
||||
|
||||
1. POST /runs 只接受发布版本或明确的 versionId。
|
||||
2. 前端隐藏“当前草稿(试运行快照)”和 mock scenario 选择器。
|
||||
3. 后端若收到 target=draft,返回 422 WORKFLOW_DRAFT_RUN_UNSUPPORTED,禁止静默回退到发布版本。
|
||||
|
||||
二期若需要草稿试运行,新增明确的 test-run API,而不是篡改正式 runs:
|
||||
|
||||
POST /api/workflows/{workflow_id}/test-runs
|
||||
|
||||
流程为:CAS 读取草稿 → 生成不可变 test snapshot → 记录 snapshot id → 运行 → 保留期清理。运行记录必须注明 test=true,不能被误认为发布版本运行。
|
||||
|
||||
运行输入:
|
||||
|
||||
- studio-document 响应必须带 inputSchema。
|
||||
- 前端根据 inputSchema 渲染输入,而非永远发送 input.query。
|
||||
- POST /runs 继续使用后端 JSON Schema 验证;错误返回 field/path/expected,前端在表单附近展示。
|
||||
|
||||
### 5.6 SSE 事件契约收敛
|
||||
|
||||
后端事件是前端执行过程唯一事实来源。所有事件须先持久化再广播,seq 单调递增且可 REST 重放。
|
||||
|
||||
规范化以下 payload:
|
||||
|
||||
node.tool.started
|
||||
{
|
||||
"name": "web_search",
|
||||
"toolCallId": "call_001",
|
||||
"title": "联网检索",
|
||||
"args": { "path": "/mnt/user-data/outputs/report.md" }
|
||||
}
|
||||
|
||||
node.tool.finished
|
||||
{
|
||||
"name": "web_search",
|
||||
"toolCallId": "call_001",
|
||||
"status": "succeeded",
|
||||
"summary": "获得 12 条结果"
|
||||
}
|
||||
|
||||
run.awaiting_input
|
||||
{
|
||||
"resumeToken": "one-time-token",
|
||||
"nodeId": "review",
|
||||
"prompt": "请确认是否发布",
|
||||
"formSchema": { "type": "object", "properties": {} },
|
||||
"actions": ["approve", "reject"]
|
||||
}
|
||||
|
||||
artifact.created
|
||||
{
|
||||
"artifactId": "artifact_001",
|
||||
"name": "weekly-report.docx",
|
||||
"kind": "file",
|
||||
"mimeType": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"sizeBytes": 1234
|
||||
}
|
||||
|
||||
规则:
|
||||
|
||||
- 工具字段统一为 name;不要只发 toolName。
|
||||
- 失败工具必须明确 status=failed 与脱敏 error。
|
||||
- pendingInput 可以继续存储在数据库,但对外事件须扁平化为 prompt、formSchema、actions。
|
||||
- agent 节点继续发送安全展示元数据 agent、parallelGroupId、roundIndex;不发送系统提示词、技能白名单、凭据和完整配置。
|
||||
- 不将每个模型 token 当作 aria/live 或持久化事件;继续使用节流后的 node.output.delta。
|
||||
- 事件类型表若仍不含 node.tool.input_delta,应从前端 mock 中删除该依赖,或后端正式实现该事件;不能只有 mock 有。
|
||||
|
||||
涉及路径:
|
||||
|
||||
| 路径 | 改动 |
|
||||
| --- | --- |
|
||||
| packages/harness/deerflow/workflows/nodes/agents.py | on_tool 输出 name、status、最小安全 args |
|
||||
| app/gateway/workflow_executor.py | run.awaiting_input 对外 payload 扁平化 |
|
||||
| packages/harness/deerflow/workflows/events.py | 冻结事件类型与 payload schema 样例 |
|
||||
| app/gateway/routers/workflow_runs.py | REST 重放与 SSE 输出保持同一 envelope |
|
||||
| packages/harness/deerflow/workflows/runtime/engine.py | 保持 node.started 的 agent / parallelGroupId / roundIndex 安全投影 |
|
||||
|
||||
### 5.7 制品内容、预览和下载
|
||||
|
||||
新增:
|
||||
|
||||
GET /api/workflows/runs/{run_id}/artifacts/{artifact_id}
|
||||
GET /api/workflows/runs/{run_id}/artifacts/{artifact_id}/download
|
||||
|
||||
内容接口返回受限预览:
|
||||
|
||||
{
|
||||
"artifactId": "artifact_001",
|
||||
"name": "weekly-report.md",
|
||||
"kind": "markdown",
|
||||
"mimeType": "text/markdown",
|
||||
"text": "# 周报",
|
||||
"sizeBytes": 2048
|
||||
}
|
||||
|
||||
策略:
|
||||
|
||||
1. 每次读取先校验 run owner/admin,再校验 artifact 属于该 run。
|
||||
2. 文本、JSON、表格只返回大小受限预览;大文件返回 truncated=true。
|
||||
3. 二进制文件不塞入 JSON;download 接口以附件或受控 inline 响应流返回。
|
||||
4. 路径只能从已登记的 artifact id 解析,禁止接收任意 sandbox 路径。
|
||||
5. 制品存储生命周期沿用现有 sandbox/artifact 机制;工作流表只存引用与元数据。
|
||||
|
||||
涉及路径:
|
||||
|
||||
- app/gateway/routers/workflow_runs.py:新增两个路由。
|
||||
- packages/harness/deerflow/persistence/workflow_runs/*:按 run_id + artifact_id 查询的 store 方法。
|
||||
- app/gateway/routers/artifacts.py:仅可参考现有 thread artifact 的鉴权和文件响应,不可直接跨 thread 复用。
|
||||
|
||||
### 5.8 取消状态和事件游标
|
||||
|
||||
统一运行状态:
|
||||
|
||||
| 后端状态 | 前端显示 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| queued | 排队中 | 已入队未领取 |
|
||||
| running | 运行中 | 正在执行 |
|
||||
| awaiting_input | 等待人工介入 | 暂停并持有一次性 token |
|
||||
| cancel_requested | 取消中 | 等待运行器确认,非终态 |
|
||||
| completed / failed / cancelled | 终态 | SSE 停止重连 |
|
||||
|
||||
后端工作:
|
||||
|
||||
1. _public_run 输出 lastSeq 时映射 persistence 的 next_event_seq,不能读取不存在的 last_seq。
|
||||
2. 前端 RunStatus 类型和 RunStatusBadge 增加 cancel_requested;后端不应把它伪装为 cancelled。
|
||||
3. list events 空页必须返回请求 cursor,防止前端游标回退。
|
||||
|
||||
### 5.9 连续对话运行(二期,可选)
|
||||
|
||||
当前“对话运行”是单个 run 的视图投影:每次发送创建一个新 run。若业务要 ChatDev 式连续问答,需要独立设计,不能复用 run_id 作为会话 id。
|
||||
|
||||
建议新增:
|
||||
|
||||
workflow_conversations
|
||||
workflow_conversation_turns
|
||||
workflow_runs.conversation_id (nullable)
|
||||
|
||||
接口:
|
||||
|
||||
POST /api/workflows/{workflow_id}/conversations
|
||||
GET /api/workflows/{workflow_id}/conversations
|
||||
POST /api/workflows/conversations/{conversation_id}/turns
|
||||
|
||||
每个 turn 创建一个 run,并记录输入、run id、可展示摘要和制品引用。默认不把完整模型上下文拼接给每个节点;由 graph 明确配置哪些字段、哪些历史摘要可见。
|
||||
|
||||
## 6. 推荐实施顺序
|
||||
|
||||
### 阶段 A:真实保存与发布闭环(阻断项)
|
||||
|
||||
1. 增加 canvas schema 持久化迁移和 store 原子接口。
|
||||
2. 实现 studio facade 四个端点。
|
||||
3. 前端实现 Coze JSON 到 WorkflowGraph 编译,并改接 facade。
|
||||
4. 完成输入/输出、代码、Agent、Skill、SQL、HTTP 的最小节点配置表单。
|
||||
5. 加入发布前 app 层资源检查。
|
||||
|
||||
验收:编辑一个“Input → SQL → Agent → Output”流程,刷新后画布与配置不丢失,发布后执行的是同一版本的标准图。
|
||||
|
||||
### 阶段 B:真实执行过程闭环
|
||||
|
||||
1. 统一工具、人工介入、制品的 SSE payload。
|
||||
2. 实现制品内容与下载接口。
|
||||
3. 修复 lastSeq、cancel_requested。
|
||||
4. 增加跨 REST 重放、SSE 断线重连、人工介入恢复的契约测试。
|
||||
|
||||
验收:报告工作流在 iframe 刷新后能恢复步骤、工具摘要、文件卡和人工介入状态。
|
||||
|
||||
### 阶段 C:草稿试运行与连续会话
|
||||
|
||||
1. 先由产品确认是否真的需要草稿试运行;需要时实现独立 test-run snapshot。
|
||||
2. 再由业务确认是否需要跨 run 的上下文记忆;需要时实现 conversation/turn。
|
||||
|
||||
在阶段 C 前,前端必须隐藏对应入口,禁止用“看似可点、实际忽略”的体验替代能力。
|
||||
|
||||
## 7. 测试与验收
|
||||
|
||||
### 7.1 后端新增测试
|
||||
|
||||
建议新增:
|
||||
|
||||
| 文件 | 重点场景 |
|
||||
| --- | --- |
|
||||
| tests/test_workflow_studio_facade.py | studio DTO、camelCase、CAS、错误结构、owner/admin 鉴权 |
|
||||
| tests/test_workflow_studio_graph_contract.py | Coze 编译产物可被 WorkflowGraph 接受;非法 graph 被拒绝 |
|
||||
| tests/test_workflow_resource_validation.py | Agent/Skill/SQL/HTTP/Code/Output 配置与权限校验 |
|
||||
| tests/test_workflow_event_contract.py | tool、awaiting_input、artifact、parallel agent payload 与 REST/SSE 一致 |
|
||||
| tests/test_workflow_artifact_content.py | 文本预览、二进制下载、越权、路径穿越、超大文件 |
|
||||
| tests/test_workflow_runs.py | target=draft 显式拒绝、cancel_requested、lastSeq |
|
||||
|
||||
保留并扩展:
|
||||
|
||||
- tests/test_workflow_engine.py:并行、循环、节点 started 元数据。
|
||||
- tests/test_workflow_runs.py:运行、取消、恢复、SSE、重放。
|
||||
- tests/test_workflow_data_sources.py:SQL 只读、HTTP allowlist、秘密不回显。
|
||||
|
||||
### 7.2 前后端契约 fixture
|
||||
|
||||
在 backend/tests/fixtures/workflow_studio/ 固化 JSON fixture,并让前端测试复制或通过生成脚本消费:
|
||||
|
||||
1. studio-document。
|
||||
2. 保存成功与 409 冲突。
|
||||
3. 资源缺失校验。
|
||||
4. Agent 工具 started/finished。
|
||||
5. human_input 暂停与恢复。
|
||||
6. markdown/json/table/docx 制品。
|
||||
7. 并行三智能体与取消中状态。
|
||||
|
||||
禁止前后端各自手写近似 mock 事件。fixture 变更必须同时更新双方测试。
|
||||
|
||||
### 7.3 发布准入
|
||||
|
||||
- [ ] Coze 画布和 executionGraph 采用两份数据、一次 CAS 保存。
|
||||
- [ ] 真实后端不再依赖 schemaJson、valid 等 mock 专用字段。
|
||||
- [ ] 所有可拖入资源都能在 graph 中找到真实绑定字段。
|
||||
- [ ] 发布前可定位到节点和字段的资源/权限错误。
|
||||
- [ ] 工具、人工介入、制品在 SSE 与 REST replay 中字段一致。
|
||||
- [ ] 制品内容和下载经过 run 级鉴权,不能按任意路径读取。
|
||||
- [ ] target=draft 不会被静默执行为发布版本。
|
||||
- [ ] typecheck、后端 pytest、ruff 与至少一条 iframe E2E 全部通过。
|
||||
|
||||
## 8. 明确不做的事项
|
||||
|
||||
1. 不把整个 Coze Go 后端或 ChatDev 后端迁入 DeerFlow。
|
||||
2. 不在 DeerFlow harness 中导入 app.gateway 或 Coze 前端类型。
|
||||
3. 不为了兼容 mock 而让标准 WorkflowGraph API 永久接受任意 schemaJson。
|
||||
4. 不向浏览器返回数据库 DSN、HTTP headers、token、resume token、沙箱真实文件路径或模型隐藏思维链。
|
||||
5. 不在未实现草稿快照、连续会话前保留误导性的 UI 入口。
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# ADR 0001:工作流编排运行时边界
|
||||
|
||||
- **状态**:Accepted(阶段 0 冻结;决策 3 已按阶段 2 实现修订,见文末修订记录)
|
||||
- **日期**:2026-08-28
|
||||
- **关联**:`docs/WORKFLOW_STUDIO_BACKEND_DEV_ZH.md`
|
||||
|
||||
## 背景
|
||||
|
||||
需要在 DeerFlow 中支持可持久化、可恢复、可审计的工作流运行,并用 Coze Studio 独立画布作为编辑器。可选路径包括:整包接入 Coze 后端、整包接入 ChatDev 2.0、或在 DeerFlow 内自建运行时。
|
||||
|
||||
## 决策
|
||||
|
||||
1. **Coze Studio 只负责画布与节点配置 UI**
|
||||
不复用其完整后端协议、内置试运行、Trace/Debug。Gateway 仅提供薄兼容层(canvas/save/create/node_template)。
|
||||
|
||||
2. **ChatDev 2.0 只作语义参考**
|
||||
可参考 DAG、并行、循环、动态边、会话执行边界;**不**引入其 WebSocket、SQLite 存储、宿主机 `subprocess` 代码执行、自有 Agent/模型 runtime。
|
||||
|
||||
3. **执行内核 = DeerFlow 自建 DAG 调度器;LangGraph 只作智能体内核**
|
||||
见下方「阶段 2 修订」。智能体 / 技能 / 沙箱 / 模型 / 制品全部复用现有 harness 能力。`deerflow.workflows` 不得导入 `app.*`。
|
||||
|
||||
4. **流式协议 = 持久事件日志 + SSE**
|
||||
先写 `workflow_run_events`,再推 live;断线用 `Last-Event-ID` / `after_seq` 补放。不与 ChatDev WebSocket 并存。
|
||||
|
||||
## 阶段 0 配套确认
|
||||
|
||||
| 议题 | 结论 |
|
||||
|------|------|
|
||||
| SQL 节点驱动 | 首期仅 **MySQL 和/或 PostgreSQL**(与项目实际库一致);不抽象全数据库 |
|
||||
| 凭据存储 | 画布只存 `credentialRef` / `dataSourceId`;明文密钥进服务端凭据存储,不进 JSON / SSE / 日志 |
|
||||
| iframe 域名 | 由 `config.yaml` → `workflows.embed.allowed_origins`(阶段 6 落地配置项);票据绑定 origin + workflow + user + actions,约 60s 一次性兑换 |
|
||||
| 应用库 vs 业务查询库 | DeerFlow 自身业务库 **不得**直接暴露给 SQL 节点;必须管理员登记独立只读数据源 |
|
||||
|
||||
## 阶段 2 修订:工作流级编排不编译成 LangGraph
|
||||
|
||||
阶段 0 原定「标准图编译为 `StateGraph`」。实现阶段 2 时改为**自建 DAG 调度器**(`deerflow/workflows/runtime/engine.py`),LangGraph 退到**单个 `agent` / `skill` 节点内部**继续做智能体内核。
|
||||
|
||||
**原因**:运行状态必须只有一个权威来源。工作流的可恢复性建立在 `workflow_node_runs`(已完成节点的输出)+ `workflow_run_events`(事件日志)之上——重新租约或人工恢复时,执行器从这两张表回放,只跑缺失的节点。若同时用 LangGraph checkpoint 保存工作流级状态,就会出现两份状态需要保持一致(checkpoint 与 node_runs 谁说了算?租约被抢走后 checkpoint 归谁?),恢复语义无法自证。此外 `human_input` 的暂停需要把 `resume_token` 与 `pending_input` 暴露到 HTTP/SSE 层,条件分支需要「未选中的端口整条子图跳过」的显式边状态,循环需要按 `bodyEntry` 精确失效 body 内的 node_runs——这些都要求调度器直接读写运行存储。
|
||||
|
||||
**代价**:调度、并行度、重试、超时、循环上限这些机制要自己实现和测试(`tests/test_workflow_engine.py`、`tests/test_workflow_runs.py`)。**收益**:崩溃恢复、取消、暂停/恢复都只有一条真相路径,且不需要为工作流额外引入一套 checkpoint 生命周期。
|
||||
|
||||
调度规则本身很小:每条边有 `pending` / `active` / `pruned` 三态,节点在「非回边入边都已定型且至少一条 `active`」时就绪,全部 `pruned` 则跳过并继续下推剪枝。`loop` 节点的 `continue` 回边是唯一允许的环,不参与就绪判定。
|
||||
|
||||
## 后果
|
||||
|
||||
- 短期需自建 schema / store / dispatcher / SSE,工作量高于“整包移植”。
|
||||
- 长期只维护一套模型调用、鉴权、沙箱与审计,避免双运行时分叉。
|
||||
- 前端时间线必须与本仓库冻结的事件 envelope(`deerflow.workflows.events`)契约一致。
|
||||
|
||||
## 否决方案
|
||||
|
||||
- 把 Coze `workflow_api` 全量试运行当作执行引擎。
|
||||
- 把 ChatDev `python_executor.py`(宿主机 subprocess)当作代码节点。
|
||||
- 在画布 JSON 或 URL query 中传递 JWT / DB 密码 / API Key。
|
||||
@@ -42,6 +42,7 @@ from deerflow.config.token_usage_config import TokenUsageConfig
|
||||
from deerflow.config.tool_config import ToolConfig, ToolGroupConfig
|
||||
from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_search_config_from_dict
|
||||
from deerflow.config.vision_config import VisionConfig
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.config.y_log_config import YLogConfig
|
||||
|
||||
load_dotenv()
|
||||
@@ -149,6 +150,7 @@ class AppConfig(BaseModel):
|
||||
knowledge_ingest: KnowledgeIngestConfig = Field(default_factory=KnowledgeIngestConfig, description="External knowledge-ingest endpoint configuration")
|
||||
task_deeplink: TaskDeeplinkConfig = Field(default_factory=TaskDeeplinkConfig, description="任务深链(rwfx)目的树闸门与跳转地址配置(前端经 /api/public/config/task-deeplink 读取)")
|
||||
y_log: YLogConfig = Field(default_factory=YLogConfig, description="操作日志上报配置(前端经 /api/public/config/y-log 读取)")
|
||||
workflows: WorkflowConfig = Field(default_factory=WorkflowConfig, description="Workflow Studio limits / iframe origins")
|
||||
sentiment_agent: SentimentAgentConfig = Field(
|
||||
default_factory=SentimentAgentConfig,
|
||||
description="舆情分析虚拟智能体上游代理超时(地址由前端 runtime-config.js 随请求传入)",
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Workflow Studio configuration (limits, egress policy, sandbox, iframe origins)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class WorkflowEmbedConfig(BaseModel):
|
||||
"""iframe ticket / origin policy."""
|
||||
|
||||
allowed_origins: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Allowed parent origins for workflow embed tickets (exact match)",
|
||||
)
|
||||
ticket_ttl_seconds: int = Field(default=60, ge=5, le=600)
|
||||
|
||||
|
||||
class WorkflowHttpPolicyConfig(BaseModel):
|
||||
"""Egress policy for the ``http`` node.
|
||||
|
||||
Default-deny on private address space. ``allowed_hosts`` is an explicit
|
||||
allowlist (exact host or ``.suffix`` match); when it is empty every public
|
||||
host is allowed but private/loopback/link-local targets are still refused,
|
||||
which is what stops SSRF against cloud metadata and internal services.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(default=True)
|
||||
allowed_hosts: list[str] = Field(default_factory=list)
|
||||
blocked_hosts: list[str] = Field(default_factory=list)
|
||||
allow_private_networks: bool = Field(
|
||||
default=False,
|
||||
description="Allow loopback/private/link-local targets (only for trusted intranet deployments)",
|
||||
)
|
||||
allowed_ports: list[int] = Field(default_factory=lambda: [80, 443, 8080, 8443])
|
||||
max_response_bytes: int = Field(default=2_097_152, ge=1024)
|
||||
max_redirects: int = Field(default=0, ge=0, le=5)
|
||||
timeout_seconds: int = Field(default=60, ge=1)
|
||||
|
||||
|
||||
class WorkflowSqlPolicyConfig(BaseModel):
|
||||
"""Read-only policy for the ``sql_read`` node."""
|
||||
|
||||
enabled: bool = Field(default=True)
|
||||
max_rows: int = Field(default=1000, ge=1)
|
||||
statement_timeout_seconds: int = Field(default=30, ge=1)
|
||||
max_cell_chars: int = Field(default=4000, ge=64)
|
||||
|
||||
|
||||
class WorkflowCodeNodeConfig(BaseModel):
|
||||
"""``code`` node execution. Off by default: the local runner is a hardened
|
||||
subprocess, not a container, so operators must opt in knowingly."""
|
||||
|
||||
enabled: bool = Field(default=False)
|
||||
timeout_seconds: int = Field(default=30, ge=1, le=600)
|
||||
max_source_chars: int = Field(default=20_000, ge=64)
|
||||
max_output_chars: int = Field(default=100_000, ge=256)
|
||||
memory_limit_mb: int = Field(default=512, ge=64)
|
||||
allow_network: bool = Field(default=False)
|
||||
|
||||
|
||||
class WorkflowRetentionConfig(BaseModel):
|
||||
event_retention_days: int = Field(default=30, ge=1)
|
||||
run_retention_days: int = Field(default=90, ge=1)
|
||||
|
||||
|
||||
class WorkflowConfig(BaseModel):
|
||||
"""System-level workflow limits and policies."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Master switch for workflow studio APIs")
|
||||
max_steps: int = Field(default=500, ge=1)
|
||||
max_loop_iterations: int = Field(default=20, ge=1)
|
||||
max_parallelism: int = Field(default=16, ge=1)
|
||||
run_timeout_seconds: int = Field(default=3600, ge=1)
|
||||
node_timeout_seconds: int = Field(default=600, ge=1)
|
||||
max_concurrent_runs_per_user: int = Field(default=5, ge=1)
|
||||
lease_ttl_seconds: int = Field(default=60, ge=10)
|
||||
embed: WorkflowEmbedConfig = Field(default_factory=WorkflowEmbedConfig)
|
||||
http: WorkflowHttpPolicyConfig = Field(default_factory=WorkflowHttpPolicyConfig)
|
||||
sql: WorkflowSqlPolicyConfig = Field(default_factory=WorkflowSqlPolicyConfig)
|
||||
code: WorkflowCodeNodeConfig = Field(default_factory=WorkflowCodeNodeConfig)
|
||||
retention: WorkflowRetentionConfig = Field(default_factory=WorkflowRetentionConfig)
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
"""Create workflow_definitions / workflow_versions tables.
|
||||
|
||||
Revision ID: 20260828_01
|
||||
Revises: 20260825_01
|
||||
Create Date: 2026-08-28
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_01"
|
||||
down_revision = "20260825_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_definitions"):
|
||||
op.create_table(
|
||||
"workflow_definitions",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("owner_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="draft"),
|
||||
sa.Column("draft_revision", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("draft_graph_json", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_workflow_definitions_owner_id", "workflow_definitions", ["owner_id"])
|
||||
op.create_index("ix_workflow_definitions_status", "workflow_definitions", ["status"])
|
||||
|
||||
if not inspector.has_table("workflow_versions"):
|
||||
op.create_table(
|
||||
"workflow_versions",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("workflow_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("version_number", sa.Integer(), nullable=False),
|
||||
sa.Column("graph_json", sa.Text(), nullable=False),
|
||||
sa.Column("graph_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("input_schema", sa.Text(), nullable=True),
|
||||
sa.Column("output_schema", sa.Text(), nullable=True),
|
||||
sa.Column("published_by", sa.String(length=64), nullable=True),
|
||||
sa.Column("published_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("change_note", sa.Text(), nullable=False, server_default=""),
|
||||
sa.ForeignKeyConstraint(["workflow_id"], ["workflow_definitions.id"], ondelete="CASCADE"),
|
||||
sa.UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),
|
||||
)
|
||||
op.create_index("ix_workflow_versions_workflow_id", "workflow_versions", ["workflow_id"])
|
||||
op.create_index("ix_workflow_versions_graph_hash", "workflow_versions", ["graph_hash"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if inspector.has_table("workflow_versions"):
|
||||
op.drop_table("workflow_versions")
|
||||
if inspector.has_table("workflow_definitions"):
|
||||
op.drop_table("workflow_definitions")
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
"""Create workflow run / node-run / event / artifact tables.
|
||||
|
||||
Revision ID: 20260828_02
|
||||
Revises: 20260828_01
|
||||
Create Date: 2026-08-28
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_02"
|
||||
down_revision = "20260828_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
|
||||
if not inspector.has_table("workflow_runs"):
|
||||
op.create_table(
|
||||
"workflow_runs",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("workflow_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("workflow_version_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
|
||||
sa.Column("owner_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=128), nullable=True),
|
||||
sa.Column("input_json", sa.Text(), nullable=True),
|
||||
sa.Column("output_json", sa.Text(), nullable=True),
|
||||
sa.Column("error_json", sa.Text(), nullable=True),
|
||||
sa.Column("context_json", sa.Text(), nullable=True),
|
||||
sa.Column("pending_input_json", sa.Text(), nullable=True),
|
||||
sa.Column("resume_token", sa.String(length=64), nullable=True),
|
||||
sa.Column("lease_owner", sa.String(length=64), nullable=True),
|
||||
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
|
||||
sa.Column("version", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("retry_of_run_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),
|
||||
)
|
||||
op.create_index("ix_workflow_runs_workflow_id", "workflow_runs", ["workflow_id"])
|
||||
op.create_index("ix_workflow_runs_workflow_version_id", "workflow_runs", ["workflow_version_id"])
|
||||
op.create_index("ix_workflow_runs_owner_id", "workflow_runs", ["owner_id"])
|
||||
op.create_index("ix_workflow_runs_status", "workflow_runs", ["status"])
|
||||
op.create_index("ix_workflow_runs_resume_token", "workflow_runs", ["resume_token"])
|
||||
|
||||
if not inspector.has_table("workflow_node_runs"):
|
||||
op.create_table(
|
||||
"workflow_node_runs",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("run_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("node_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("node_type", sa.String(length=32), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("iteration", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("is_current", sa.Boolean(), nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("input_snapshot", sa.Text(), nullable=True),
|
||||
sa.Column("output_snapshot", sa.Text(), nullable=True),
|
||||
sa.Column("error_json", sa.Text(), nullable=True),
|
||||
sa.Column("parent_node_run_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("duration_ms", sa.Integer(), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint(
|
||||
"run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"
|
||||
),
|
||||
)
|
||||
op.create_index("ix_workflow_node_runs_run_id", "workflow_node_runs", ["run_id"])
|
||||
op.create_index("ix_workflow_node_runs_status", "workflow_node_runs", ["status"])
|
||||
|
||||
if not inspector.has_table("workflow_run_events"):
|
||||
op.create_table(
|
||||
"workflow_run_events",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("run_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("workflow_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("version_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("seq", sa.Integer(), nullable=False),
|
||||
sa.Column("event_type", sa.String(length=64), nullable=False),
|
||||
sa.Column("node_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("node_run_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("payload_json", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
|
||||
)
|
||||
op.create_index("ix_workflow_run_events_run_id", "workflow_run_events", ["run_id"])
|
||||
op.create_index("ix_workflow_run_events_run_seq", "workflow_run_events", ["run_id", "seq"])
|
||||
|
||||
if not inspector.has_table("workflow_run_artifacts"):
|
||||
op.create_table(
|
||||
"workflow_run_artifacts",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("run_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("node_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("name", sa.String(length=512), nullable=False),
|
||||
sa.Column("mime_type", sa.String(length=191), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("size_bytes", sa.Integer(), nullable=True),
|
||||
sa.Column("preview", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_workflow_run_artifacts_run_id", "workflow_run_artifacts", ["run_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
for table in (
|
||||
"workflow_run_artifacts",
|
||||
"workflow_run_events",
|
||||
"workflow_node_runs",
|
||||
"workflow_runs",
|
||||
):
|
||||
if inspector.has_table(table):
|
||||
op.drop_table(table)
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""Create workflow_data_sources table.
|
||||
|
||||
Revision ID: 20260828_03
|
||||
Revises: 20260828_02
|
||||
Create Date: 2026-08-28
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_03"
|
||||
down_revision = "20260828_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if inspector.has_table("workflow_data_sources"):
|
||||
return
|
||||
op.create_table(
|
||||
"workflow_data_sources",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("name", sa.String(length=191), nullable=False),
|
||||
sa.Column("description", sa.String(length=512), nullable=False, server_default=""),
|
||||
sa.Column("kind", sa.String(length=32), nullable=False, server_default="sql"),
|
||||
sa.Column("driver", sa.String(length=64), nullable=False, server_default=""),
|
||||
sa.Column("secret_ciphertext", sa.Text(), nullable=False),
|
||||
sa.Column("secret_encrypted", sa.Boolean(), nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("masked_target", sa.String(length=512), nullable=False, server_default=""),
|
||||
sa.Column("max_rows", sa.Integer(), nullable=False, server_default="1000"),
|
||||
sa.Column("allowed_tables", sa.Text(), nullable=True),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("owner_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("created_by", sa.String(length=64), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("name", name="uq_workflow_data_sources_name"),
|
||||
)
|
||||
op.create_index("ix_workflow_data_sources_owner_id", "workflow_data_sources", ["owner_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
if sa.inspect(bind).has_table("workflow_data_sources"):
|
||||
op.drop_table("workflow_data_sources")
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""Add workflow counter columns: version allocator + event seq counter.
|
||||
|
||||
Revision ID: 20260828_04
|
||||
Revises: 20260828_03
|
||||
Create Date: 2026-08-28
|
||||
|
||||
Functional-robustness remediation (audit items P1-02 / P1-03):
|
||||
|
||||
* ``workflow_definitions.next_version_number`` — publish allocates the version
|
||||
number by CAS-bumping this counter instead of ``MAX(version_number) + 1``,
|
||||
so two concurrent publishes get adjacent distinct numbers instead of a
|
||||
unique-constraint 500.
|
||||
* ``workflow_runs.next_event_seq`` — the event log draws each event's ``seq``
|
||||
from this counter in the insert's own transaction instead of ``MAX(seq)+1``.
|
||||
|
||||
Both columns carry a ``server_default`` so existing databases get them via the
|
||||
startup ``_ensure_orm_columns_sync`` sweep even without running Alembic; rows
|
||||
keep the default (1 / 0) and the first write self-heals from the existing
|
||||
maximum on unique-conflict.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_04"
|
||||
down_revision = "20260828_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
|
||||
if inspector.has_table("workflow_definitions"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
|
||||
if "next_version_number" not in existing:
|
||||
op.add_column(
|
||||
"workflow_definitions",
|
||||
sa.Column("next_version_number", sa.Integer(), nullable=False, server_default="1"),
|
||||
)
|
||||
|
||||
if inspector.has_table("workflow_runs"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
|
||||
if "next_event_seq" not in existing:
|
||||
op.add_column(
|
||||
"workflow_runs",
|
||||
sa.Column("next_event_seq", sa.Integer(), nullable=False, server_default="0"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
|
||||
if inspector.has_table("workflow_definitions"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
|
||||
if "next_version_number" in existing:
|
||||
op.drop_column("workflow_definitions", "next_version_number")
|
||||
|
||||
if inspector.has_table("workflow_runs"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
|
||||
if "next_event_seq" in existing:
|
||||
op.drop_column("workflow_runs", "next_event_seq")
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
"""Add non-secret HTTP metadata to workflow data sources.
|
||||
|
||||
Revision ID: 20260829_01
|
||||
Revises: 20260828_04
|
||||
Create Date: 2026-08-29
|
||||
|
||||
HTTP request headers remain in the encrypted secret blob. The endpoint and
|
||||
method allowlist are intentionally separate non-secret fields so the canvas
|
||||
can show a reusable interface resource without exposing credentials.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260829_01"
|
||||
down_revision = "20260828_04"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_data_sources"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
|
||||
if "base_url" not in existing:
|
||||
op.add_column(
|
||||
"workflow_data_sources",
|
||||
sa.Column("base_url", sa.String(length=2048), nullable=False, server_default=""),
|
||||
)
|
||||
if "allowed_methods" not in existing:
|
||||
op.add_column(
|
||||
"workflow_data_sources",
|
||||
sa.Column("allowed_methods", sa.Text(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_data_sources"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
|
||||
if "allowed_methods" in existing:
|
||||
op.drop_column("workflow_data_sources", "allowed_methods")
|
||||
if "base_url" in existing:
|
||||
op.drop_column("workflow_data_sources", "base_url")
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
"""Add Coze canvas draft column to workflow_definitions.
|
||||
|
||||
Revision ID: 20260829_02
|
||||
Revises: 20260829_01
|
||||
Create Date: 2026-08-29
|
||||
|
||||
The canvas document (positions, Coze form internals) is stored separately from
|
||||
the executable WorkflowGraph so neither representation can impersonate the
|
||||
other. ``published_canvas_schema_hash`` records which canvas the current
|
||||
published version was compiled from, for audit only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260829_02"
|
||||
down_revision = "20260829_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_definitions"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
|
||||
if "draft_canvas_schema_json" not in existing:
|
||||
op.add_column(
|
||||
"workflow_definitions",
|
||||
sa.Column("draft_canvas_schema_json", sa.Text(), nullable=False, server_default="'{}'"),
|
||||
)
|
||||
if "published_canvas_schema_hash" not in existing:
|
||||
op.add_column(
|
||||
"workflow_definitions",
|
||||
sa.Column("published_canvas_schema_hash", sa.String(length=64), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_definitions"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
|
||||
if "published_canvas_schema_hash" in existing:
|
||||
op.drop_column("workflow_definitions", "published_canvas_schema_hash")
|
||||
if "draft_canvas_schema_json" in existing:
|
||||
op.drop_column("workflow_definitions", "draft_canvas_schema_json")
|
||||
+15
@@ -50,6 +50,14 @@ from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
from deerflow.persistence.light_apps.model import LightAppRow
|
||||
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
@@ -128,6 +136,13 @@ __all__ = [
|
||||
"KnowledgeSourceRow",
|
||||
"KnowledgeTagRow",
|
||||
"LightAppRow",
|
||||
"WorkflowDataSourceRow",
|
||||
"WorkflowDefinitionRow",
|
||||
"WorkflowNodeRunRow",
|
||||
"WorkflowRunArtifactRow",
|
||||
"WorkflowRunEventRow",
|
||||
"WorkflowRunRow",
|
||||
"WorkflowVersionRow",
|
||||
"LlmCallMetricRow",
|
||||
"LlmWikiKnowledgeBaseRow",
|
||||
"LlmWikiConversationDepositRow",
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"""Workflow data-source persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
|
||||
from deerflow.persistence.workflow_data_sources.store import (
|
||||
MemoryWorkflowDataSourceStore,
|
||||
SqlWorkflowDataSourceStore,
|
||||
WorkflowDataSourceStore,
|
||||
make_workflow_data_source_store,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowDataSourceStore",
|
||||
"SqlWorkflowDataSourceStore",
|
||||
"WorkflowDataSourceRow",
|
||||
"WorkflowDataSourceStore",
|
||||
"make_workflow_data_source_store",
|
||||
]
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
"""ORM model for workflow SQL data sources and HTTP credentials."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, UniqueConstraint
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class WorkflowDataSourceRow(Base):
|
||||
"""A read-only database connection usable by ``sql_read`` nodes.
|
||||
|
||||
``secret_ciphertext`` holds the DSN encrypted with the workflow secret key;
|
||||
it is never returned by any API. ``owner_id`` NULL means the data source is
|
||||
shared (admin-managed)."""
|
||||
|
||||
__tablename__ = "workflow_data_sources"
|
||||
__table_args__ = (UniqueConstraint("name", name="uq_workflow_data_sources_name"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(191), nullable=False)
|
||||
description: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False, default="sql")
|
||||
driver: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
base_url: Mapped[str] = mapped_column(
|
||||
String(2048),
|
||||
nullable=False,
|
||||
default="",
|
||||
server_default=sa_text("''"),
|
||||
)
|
||||
# Nullable so the startup column synchronizer can safely patch existing databases.
|
||||
allowed_methods: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
secret_ciphertext: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
secret_encrypted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
masked_target: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
max_rows: Mapped[int] = mapped_column(Integer, nullable=False, default=1000, server_default=sa_text("1000"))
|
||||
allowed_tables: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
owner_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+293
@@ -0,0 +1,293 @@
|
||||
"""Workflow data-source store (SQL + in-memory).
|
||||
|
||||
Secrets are encrypted before they reach the database and only ever decrypted
|
||||
inside :meth:`resolve_dsn`, which the run executor calls. No read path returns
|
||||
the ciphertext.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import uuid
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
|
||||
from deerflow.workflows.security.secret_box import decrypt_secret, encrypt_secret, mask_dsn
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _public_dict(row: WorkflowDataSourceRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"name": row.name,
|
||||
"description": row.description or "",
|
||||
"kind": row.kind,
|
||||
"driver": row.driver or "",
|
||||
"base_url": row.base_url or "",
|
||||
"allowed_methods": list(row.allowed_methods or []),
|
||||
"masked_target": row.masked_target or "",
|
||||
"secret_encrypted": bool(row.secret_encrypted),
|
||||
"max_rows": row.max_rows,
|
||||
"allowed_tables": list(row.allowed_tables or []),
|
||||
"enabled": bool(row.enabled),
|
||||
"owner_id": row.owner_id,
|
||||
"created_by": row.created_by,
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
class WorkflowDataSourceStore(ABC):
|
||||
@abstractmethod
|
||||
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
|
||||
"""Decrypted connection info for the executor. Never expose via HTTP."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SqlWorkflowDataSourceStore(WorkflowDataSourceStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowDataSourceRow).order_by(WorkflowDataSourceRow.name.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
visible = []
|
||||
for row in rows:
|
||||
if row.owner_id is None and include_shared:
|
||||
visible.append(row)
|
||||
elif owner_id is not None and row.owner_id == owner_id:
|
||||
visible.append(row)
|
||||
return [_public_dict(row) for row in visible]
|
||||
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
return _public_dict(row) if row else None
|
||||
|
||||
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
dsn = str(data.get("dsn") or "")
|
||||
kind = str(data.get("kind") or "sql")
|
||||
base_url = str(data.get("base_url") or "")
|
||||
row = WorkflowDataSourceRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
name=str(data["name"])[:191],
|
||||
description=str(data.get("description") or "")[:512],
|
||||
kind=kind,
|
||||
driver=str(
|
||||
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
|
||||
)[:64],
|
||||
base_url=base_url[:2048],
|
||||
allowed_methods=list(data.get("allowed_methods") or []),
|
||||
secret_ciphertext=encrypt_secret(dsn) if dsn else "",
|
||||
secret_encrypted=bool(dsn),
|
||||
masked_target=(base_url if kind == "http" else mask_dsn(dsn))[:512],
|
||||
max_rows=int(data.get("max_rows") or 1000),
|
||||
allowed_tables=list(data.get("allowed_tables") or []),
|
||||
enabled=bool(data.get("enabled", True)),
|
||||
owner_id=data.get("owner_id"),
|
||||
created_by=data.get("created_by"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
out = _public_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
if row is None:
|
||||
return None
|
||||
if data.get("name"):
|
||||
row.name = str(data["name"])[:191]
|
||||
if "description" in data:
|
||||
row.description = str(data.get("description") or "")[:512]
|
||||
if "base_url" in data:
|
||||
row.base_url = str(data.get("base_url") or "")[:2048]
|
||||
if row.kind == "http":
|
||||
row.masked_target = row.base_url[:512]
|
||||
if "allowed_methods" in data:
|
||||
row.allowed_methods = list(data.get("allowed_methods") or [])
|
||||
if data.get("dsn"):
|
||||
dsn = str(data["dsn"])
|
||||
row.secret_ciphertext = encrypt_secret(dsn)
|
||||
row.secret_encrypted = True
|
||||
if row.kind == "http":
|
||||
row.driver = "HTTP API"
|
||||
else:
|
||||
row.masked_target = mask_dsn(dsn)[:512]
|
||||
row.driver = dsn.split("://", 1)[0][:64]
|
||||
if data.get("max_rows") is not None:
|
||||
row.max_rows = int(data["max_rows"])
|
||||
if "allowed_tables" in data:
|
||||
row.allowed_tables = list(data.get("allowed_tables") or [])
|
||||
if "enabled" in data:
|
||||
row.enabled = bool(data["enabled"])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.flush()
|
||||
out = _public_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
if row is None or not row.enabled or not row.secret_ciphertext:
|
||||
return None
|
||||
return {
|
||||
"id": row.id,
|
||||
"dsn": decrypt_secret(row.secret_ciphertext),
|
||||
"kind": row.kind,
|
||||
"base_url": row.base_url or "",
|
||||
"allowed_methods": list(row.allowed_methods or []),
|
||||
"max_rows": row.max_rows,
|
||||
"allowed_tables": list(row.allowed_tables or []),
|
||||
"owner_id": row.owner_id,
|
||||
}
|
||||
|
||||
|
||||
class MemoryWorkflowDataSourceStore(WorkflowDataSourceStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._secrets: dict[str, str] = {}
|
||||
|
||||
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
for row in self._rows.values():
|
||||
if row["owner_id"] is None and include_shared:
|
||||
out.append(row)
|
||||
elif owner_id is not None and row["owner_id"] == owner_id:
|
||||
out.append(row)
|
||||
return [copy.deepcopy(r) for r in sorted(out, key=lambda r: r["name"])]
|
||||
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(source_id)
|
||||
return copy.deepcopy(row) if row else None
|
||||
|
||||
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
source_id = str(data.get("id") or uuid.uuid4())
|
||||
dsn = str(data.get("dsn") or "")
|
||||
kind = str(data.get("kind") or "sql")
|
||||
base_url = str(data.get("base_url") or "")
|
||||
now = _iso(datetime.now(UTC))
|
||||
row = {
|
||||
"id": source_id,
|
||||
"name": str(data["name"]),
|
||||
"description": str(data.get("description") or ""),
|
||||
"kind": kind,
|
||||
"driver": str(
|
||||
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
|
||||
),
|
||||
"base_url": base_url,
|
||||
"allowed_methods": list(data.get("allowed_methods") or []),
|
||||
"masked_target": base_url if kind == "http" else mask_dsn(dsn),
|
||||
"secret_encrypted": bool(dsn),
|
||||
"max_rows": int(data.get("max_rows") or 1000),
|
||||
"allowed_tables": list(data.get("allowed_tables") or []),
|
||||
"enabled": bool(data.get("enabled", True)),
|
||||
"owner_id": data.get("owner_id"),
|
||||
"created_by": data.get("created_by"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._rows[source_id] = row
|
||||
self._secrets[source_id] = dsn
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._rows.get(source_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in (
|
||||
"name",
|
||||
"description",
|
||||
"base_url",
|
||||
"allowed_methods",
|
||||
"max_rows",
|
||||
"allowed_tables",
|
||||
"enabled",
|
||||
):
|
||||
if key in data and data[key] is not None:
|
||||
row[key] = data[key]
|
||||
if "base_url" in data and row["kind"] == "http":
|
||||
row["masked_target"] = str(row["base_url"])
|
||||
if data.get("dsn"):
|
||||
self._secrets[source_id] = str(data["dsn"])
|
||||
if row["kind"] == "http":
|
||||
row["driver"] = "HTTP API"
|
||||
else:
|
||||
row["masked_target"] = mask_dsn(str(data["dsn"]))
|
||||
row["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
self._secrets.pop(source_id, None)
|
||||
return self._rows.pop(source_id, None) is not None
|
||||
|
||||
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(source_id)
|
||||
if row is None or not row["enabled"]:
|
||||
return None
|
||||
return {
|
||||
"id": source_id,
|
||||
"dsn": self._secrets.get(source_id, ""),
|
||||
"kind": row["kind"],
|
||||
"base_url": row.get("base_url", ""),
|
||||
"allowed_methods": list(row.get("allowed_methods") or []),
|
||||
"max_rows": row["max_rows"],
|
||||
"allowed_tables": row["allowed_tables"],
|
||||
"owner_id": row["owner_id"],
|
||||
}
|
||||
|
||||
|
||||
def make_workflow_data_source_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowDataSourceStore()
|
||||
return SqlWorkflowDataSourceStore(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowDataSourceStore",
|
||||
"SqlWorkflowDataSourceStore",
|
||||
"WorkflowDataSourceStore",
|
||||
"make_workflow_data_source_store",
|
||||
]
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Workflow run event-log persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.memory import MemoryWorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_events.sql import SqlWorkflowEventStore
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowEventStore",
|
||||
"SqlWorkflowEventStore",
|
||||
"WorkflowEventStore",
|
||||
"WorkflowRunEventRow",
|
||||
"make_workflow_event_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_event_store(session_factory, run_store=None):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowEventStore(run_store=run_store)
|
||||
return SqlWorkflowEventStore(session_factory)
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""Abstract workflow run event store (durable, replayable)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class WorkflowEventStore(ABC):
|
||||
@abstractmethod
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist one event, assigning the next per-run ``seq``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
"""Retention purge by event age; returns deleted row count."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
"""Delete every event belonging to the given runs (used when the runs
|
||||
themselves are retained away). Returns deleted row count."""
|
||||
raise NotImplementedError
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
"""In-memory workflow event log (tests / no-database deployments)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
|
||||
|
||||
class MemoryWorkflowEventStore(WorkflowEventStore):
|
||||
def __init__(self, *, max_per_run: int = 20_000, run_store: Any = None) -> None:
|
||||
self._events: dict[str, list[dict[str, Any]]] = {}
|
||||
self._max_per_run = max_per_run
|
||||
self._run_store = run_store
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
async with self._lock:
|
||||
rows = self._events.setdefault(run_id, [])
|
||||
event = {
|
||||
"seq": len(rows) + 1 if not rows else rows[-1]["seq"] + 1,
|
||||
"run_id": run_id,
|
||||
"workflow_id": workflow_id,
|
||||
"version_id": version_id,
|
||||
"event_type": event_type,
|
||||
"node_id": node_id,
|
||||
"node_run_id": node_run_id,
|
||||
"payload": payload or {},
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
rows.append(event)
|
||||
if len(rows) > self._max_per_run:
|
||||
del rows[: len(rows) - self._max_per_run]
|
||||
if self._run_store is not None and hasattr(self._run_store, "note_event_seq"):
|
||||
# Keep the run row's lastSeq counter in sync (mirrors the SQL store,
|
||||
# where the counter bump and the insert share one transaction).
|
||||
try:
|
||||
await self._run_store.note_event_seq(run_id, event["seq"])
|
||||
except Exception: # noqa: BLE001 - counter sync is best effort
|
||||
pass
|
||||
return copy.deepcopy(event)
|
||||
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._events.get(run_id, []) if r["seq"] > after_seq]
|
||||
return [copy.deepcopy(r) for r in rows[:limit]]
|
||||
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
rows = self._events.get(run_id, [])
|
||||
return int(rows[-1]["seq"]) if rows else 0
|
||||
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._lock:
|
||||
deleted = 0
|
||||
for run_id, rows in list(self._events.items()):
|
||||
kept: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
created = _parse_dt(row.get("created_at"))
|
||||
if created is not None and created < cutoff:
|
||||
deleted += 1
|
||||
else:
|
||||
kept.append(row)
|
||||
if kept:
|
||||
self._events[run_id] = kept
|
||||
else:
|
||||
del self._events[run_id]
|
||||
return deleted
|
||||
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
if not run_ids:
|
||||
return 0
|
||||
async with self._lock:
|
||||
deleted = 0
|
||||
for run_id in run_ids:
|
||||
rows = self._events.pop(run_id, None)
|
||||
if rows:
|
||||
deleted += len(rows)
|
||||
return deleted
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
return None
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"""ORM model for the durable workflow run event log."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class WorkflowRunEventRow(Base):
|
||||
"""A single persisted event. ``seq`` is monotonic per run and gap-free."""
|
||||
|
||||
__tablename__ = "workflow_run_events"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
|
||||
sa.Index("ix_workflow_run_events_run_seq", "run_id", "seq"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
version_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
seq: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
node_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
# JSON text rather than PortableJSON: payloads can hold bounded deltas.
|
||||
payload_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="{}")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
"""SQLAlchemy-backed workflow event log."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_runs.model import WorkflowRunRow
|
||||
|
||||
_MAX_PAYLOAD_CHARS = 64_000
|
||||
_APPEND_ATTEMPTS = 5
|
||||
|
||||
|
||||
def _dump_payload(payload: dict[str, Any] | None) -> str:
|
||||
text = json.dumps(payload or {}, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
if len(text) > _MAX_PAYLOAD_CHARS:
|
||||
return json.dumps(
|
||||
{"truncated": True, "preview": text[:_MAX_PAYLOAD_CHARS]},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def event_to_dict(row: WorkflowRunEventRow) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(row.payload_json or "{}")
|
||||
except json.JSONDecodeError:
|
||||
payload = {}
|
||||
return {
|
||||
"seq": row.seq,
|
||||
"run_id": row.run_id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"version_id": row.version_id,
|
||||
"event_type": row.event_type,
|
||||
"node_id": row.node_id,
|
||||
"node_run_id": row.node_run_id,
|
||||
"payload": payload if isinstance(payload, dict) else {},
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else None,
|
||||
}
|
||||
|
||||
|
||||
class SqlWorkflowEventStore(WorkflowEventStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
max_retries: int = _APPEND_ATTEMPTS,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist one event, taking its ``seq`` from the run's own counter.
|
||||
|
||||
``MAX(seq) + 1`` let two appenders draw the same number and burn a
|
||||
unique-constraint retry; here the counter increment and the insert share
|
||||
one transaction, and the row write-lock held by the UPDATE serialises
|
||||
concurrent appenders until commit — so seqs are strictly increasing,
|
||||
unique and gap-free per run, in commit order.
|
||||
"""
|
||||
text = _dump_payload(payload)
|
||||
for _ in range(max_retries):
|
||||
async with self._sf() as session:
|
||||
seq = await self._take_seq(session, run_id)
|
||||
row = WorkflowRunEventRow(
|
||||
run_id=run_id,
|
||||
workflow_id=workflow_id,
|
||||
version_id=version_id,
|
||||
seq=seq,
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
node_run_id=node_run_id,
|
||||
payload_json=text,
|
||||
)
|
||||
session.add(row)
|
||||
try:
|
||||
await session.flush()
|
||||
except IntegrityError:
|
||||
# Only reachable for legacy rows whose counter was backfilled
|
||||
# to 0 while events already existed. Re-sync the counter to
|
||||
# MAX(seq) in its own committed transaction, then retry.
|
||||
await session.rollback()
|
||||
await self._resync_counter(run_id)
|
||||
continue
|
||||
out = event_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
raise RuntimeError(f"failed to append workflow event after {max_retries} attempts (run={run_id})")
|
||||
|
||||
async def _take_seq(self, session: Any, run_id: str) -> int:
|
||||
"""Draw the next seq. Falls back to MAX(seq)+1 when the run row is
|
||||
missing (event store used standalone in tests)."""
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow).where(WorkflowRunRow.id == run_id).values(next_event_seq=WorkflowRunRow.next_event_seq + 1)
|
||||
)
|
||||
if result.rowcount:
|
||||
# Our own uncommitted increment; the row lock guards it until commit.
|
||||
current = (
|
||||
await session.execute(select(WorkflowRunRow.next_event_seq).where(WorkflowRunRow.id == run_id))
|
||||
).scalar()
|
||||
return int(current or 1)
|
||||
current = (
|
||||
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
|
||||
).scalar()
|
||||
return int(current or 0) + 1
|
||||
|
||||
async def _resync_counter(self, run_id: str) -> None:
|
||||
"""Point a legacy/stale run counter back at MAX(seq)."""
|
||||
async with self._sf() as session:
|
||||
max_seq = (
|
||||
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
|
||||
).scalar() or 0
|
||||
await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(WorkflowRunRow.id == run_id, WorkflowRunRow.next_event_seq < int(max_seq))
|
||||
.values(next_event_seq=int(max_seq))
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id == run_id, WorkflowRunEventRow.seq > after_seq).order_by(WorkflowRunEventRow.seq.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [event_to_dict(r) for r in rows]
|
||||
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
async with self._sf() as session:
|
||||
value = (await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))).scalar()
|
||||
return int(value or 0)
|
||||
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.created_at < cutoff))
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
if not run_ids:
|
||||
return 0
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id.in_(list(run_ids))))
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
"""Workflow run / node-run / artifact persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunConflictError,
|
||||
WorkflowRunNotFoundError,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.memory import MemoryWorkflowRunStore
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.sql import SqlWorkflowRunStore
|
||||
|
||||
__all__ = [
|
||||
"CANCELLABLE_RUN_STATUSES",
|
||||
"MemoryWorkflowRunStore",
|
||||
"SqlWorkflowRunStore",
|
||||
"TERMINAL_RUN_STATUSES",
|
||||
"WorkflowNodeRunRow",
|
||||
"WorkflowRunArtifactRow",
|
||||
"WorkflowRunConflictError",
|
||||
"WorkflowRunNotFoundError",
|
||||
"WorkflowRunRow",
|
||||
"WorkflowRunStore",
|
||||
"make_workflow_run_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_run_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowRunStore()
|
||||
return SqlWorkflowRunStore(session_factory)
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Abstract workflow run / node-run / artifact store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled")
|
||||
CANCELLABLE_RUN_STATUSES = ("queued", "running", "awaiting_input")
|
||||
|
||||
|
||||
class WorkflowRunNotFoundError(Exception):
|
||||
def __init__(self, run_id: str) -> None:
|
||||
super().__init__(f"workflow run not found: {run_id}")
|
||||
self.run_id = run_id
|
||||
|
||||
|
||||
class WorkflowRunConflictError(Exception):
|
||||
"""Raised when a state transition is not legal for the current status."""
|
||||
|
||||
def __init__(self, run_id: str, status: str) -> None:
|
||||
super().__init__(f"workflow run {run_id} not in a valid state (status={status})")
|
||||
self.run_id = run_id
|
||||
self.status = status
|
||||
|
||||
|
||||
class WorkflowRunStore(ABC):
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
"""Create a run. Returns ``(run, created)``; an existing
|
||||
``idempotency_key`` yields the stored run with ``created=False``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
"""Run ids that are ``queued`` or ``running`` with an expired lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
"""CAS update. When ``lease_owner`` is given the row must still be leased
|
||||
by that owner. Terminal statuses clear the lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Pause atomically: status + token + pending descriptor (+ optional run
|
||||
context) in one conditional update, so a concurrent resume can never be
|
||||
overwritten by a follow-up context write."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
"""Atomically consume a one-shot resume token: ``awaiting_input`` →
|
||||
``queued``. Returns None when the token is wrong/already used."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomic human-input resume in a single conditional update.
|
||||
|
||||
Validates the token, stores ``payload`` as ``context.resume_payload``
|
||||
(replacing the whole run context with ``context`` when given), clears the
|
||||
token/pending descriptor and flips the run to ``queued`` — all under
|
||||
``WHERE status = 'awaiting_input' AND resume_token = :token``. Returns
|
||||
the resumed run on success, ``None`` when the token is wrong, already
|
||||
used, or the run is no longer awaiting input (caller maps that to 409).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Insert-or-update a node run keyed by (run_id, node_id, attempt, iteration)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
"""Mark node runs stale (used when a loop re-enters its body)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
"""Fetch one artifact row scoped to its run (authorization key)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── retention ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
"""Delete finished runs older than the cutoff, with their node runs and
|
||||
artifact rows. Only terminal runs are eligible. Returns deleted run ids
|
||||
(bounded per call so a sweeper can loop)."""
|
||||
raise NotImplementedError
|
||||
+388
@@ -0,0 +1,388 @@
|
||||
"""In-memory workflow run store (tests / no-database deployments)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
class MemoryWorkflowRunStore(WorkflowRunStore):
|
||||
def __init__(self) -> None:
|
||||
self._runs: dict[str, dict[str, Any]] = {}
|
||||
self._node_runs: dict[str, list[dict[str, Any]]] = {}
|
||||
self._artifacts: dict[str, list[dict[str, Any]]] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
async with self._lock:
|
||||
key = data.get("idempotency_key")
|
||||
if key:
|
||||
for run in self._runs.values():
|
||||
if run.get("idempotency_key") == str(key):
|
||||
return copy.deepcopy(run), False
|
||||
now = datetime.now(UTC)
|
||||
run_id = str(data.get("id") or uuid.uuid4())
|
||||
run = {
|
||||
"id": run_id,
|
||||
"workflow_id": str(data["workflow_id"]),
|
||||
"workflow_version_id": str(data["workflow_version_id"]),
|
||||
"status": str(data.get("status") or "queued"),
|
||||
"owner_id": str(data["owner_id"]),
|
||||
"idempotency_key": str(key) if key else None,
|
||||
"input": data.get("input") or {},
|
||||
"output": None,
|
||||
"error": None,
|
||||
"context": data.get("context") or {},
|
||||
"pending_input": None,
|
||||
"resume_token": None,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
"heartbeat_at": None,
|
||||
"attempt": 0,
|
||||
"max_attempts": int(data.get("max_attempts") or 3),
|
||||
"version": 0,
|
||||
"retry_of_run_id": data.get("retry_of_run_id"),
|
||||
"next_event_seq": 0,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"created_at": _iso(now),
|
||||
"updated_at": _iso(now),
|
||||
"_created": now,
|
||||
}
|
||||
self._runs[run_id] = run
|
||||
return copy.deepcopy(run), True
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
return copy.deepcopy(run) if run else None
|
||||
|
||||
async def note_event_seq(self, run_id: str, seq: int) -> None:
|
||||
"""Mirror the SQL counter semantics for no-DB mode: the memory event
|
||||
store reports each handed-out seq so run views can expose lastSeq."""
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is not None and int(seq) > int(run.get("next_event_seq") or 0):
|
||||
run["next_event_seq"] = int(seq)
|
||||
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = sorted(self._runs.values(), key=lambda r: r["_created"], reverse=True)
|
||||
if workflow_id:
|
||||
rows = [r for r in rows if r["workflow_id"] == workflow_id]
|
||||
if owner_id:
|
||||
rows = [r for r in rows if r["owner_id"] == owner_id]
|
||||
if status:
|
||||
rows = [r for r in rows if r["status"] == status]
|
||||
return [copy.deepcopy(r) for r in rows[offset : offset + limit]]
|
||||
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
out: list[str] = []
|
||||
for run in sorted(self._runs.values(), key=lambda r: r["_created"]):
|
||||
if run["status"] == "queued":
|
||||
out.append(run["id"])
|
||||
elif run["status"] == "running" and run["lease_until"]:
|
||||
if datetime.fromisoformat(run["lease_until"]) < now:
|
||||
out.append(run["id"])
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
claimable = run["status"] == "queued" or (run["status"] == "running" and run["lease_until"] and datetime.fromisoformat(run["lease_until"]) < now)
|
||||
if not claimable:
|
||||
return None
|
||||
run.update(
|
||||
status="running",
|
||||
lease_owner=lease_owner,
|
||||
lease_until=_iso(lease_until),
|
||||
heartbeat_at=_iso(now),
|
||||
attempt=run["attempt"] + 1,
|
||||
version=run["version"] + 1,
|
||||
started_at=run["started_at"] or _iso(now),
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["lease_owner"] != lease_owner or run["status"] != "running":
|
||||
return False
|
||||
run["lease_until"] = _iso(lease_until)
|
||||
run["heartbeat_at"] = _iso(datetime.now(UTC))
|
||||
return True
|
||||
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
mapping = {
|
||||
"output_json": "output",
|
||||
"error_json": "error",
|
||||
"context_json": "context",
|
||||
"pending_input_json": "pending_input",
|
||||
}
|
||||
for key, value in fields.items():
|
||||
run[mapping.get(key, key)] = value
|
||||
status = fields.get("status")
|
||||
if status in TERMINAL_RUN_STATUSES:
|
||||
run["lease_owner"] = None
|
||||
run["lease_until"] = None
|
||||
run["resume_token"] = None
|
||||
run["pending_input"] = None
|
||||
run["finished_at"] = run.get("finished_at") or _iso(now)
|
||||
for key in ("started_at", "finished_at"):
|
||||
if isinstance(run.get(key), datetime):
|
||||
run[key] = _iso(run[key])
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(now)
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if run["status"] in ("cancel_requested", "cancelled"):
|
||||
return copy.deepcopy(run)
|
||||
if run["status"] not in CANCELLABLE_RUN_STATUSES:
|
||||
return None
|
||||
run["status"] = "cancel_requested"
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "cancel_requested":
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
run.update(
|
||||
status="cancelled",
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
resume_token=None,
|
||||
pending_input=None,
|
||||
finished_at=_iso(now),
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
run["version"] += 1
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "running":
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
run.update(
|
||||
status="awaiting_input",
|
||||
pending_input=copy.deepcopy(pending_input),
|
||||
resume_token=resume_token,
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
if context is not None:
|
||||
run["context"] = copy.deepcopy(context)
|
||||
run["version"] += 1
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "awaiting_input":
|
||||
return None
|
||||
if not run.get("resume_token") or run["resume_token"] != resume_token:
|
||||
return None
|
||||
run.update(status="queued", resume_token=None, pending_input=None)
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "awaiting_input":
|
||||
return None
|
||||
if not run.get("resume_token") or run["resume_token"] != resume_token:
|
||||
return None
|
||||
base_context = context if context is not None else run.get("context") or {}
|
||||
run.update(
|
||||
context={**base_context, "resume_payload": copy.deepcopy(payload)},
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input=None,
|
||||
)
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(data["run_id"])
|
||||
node_id = str(data["node_id"])
|
||||
attempt = int(data.get("attempt") or 1)
|
||||
iteration = int(data.get("iteration") or 0)
|
||||
rows = self._node_runs.setdefault(run_id, [])
|
||||
existing = next(
|
||||
(r for r in rows if r["node_id"] == node_id and r["attempt"] == attempt and r["iteration"] == iteration),
|
||||
None,
|
||||
)
|
||||
if existing is None:
|
||||
existing = {
|
||||
"id": str(data.get("id") or uuid.uuid4()),
|
||||
"run_id": run_id,
|
||||
"node_id": node_id,
|
||||
"node_type": str(data.get("node_type") or "unknown"),
|
||||
"status": "queued",
|
||||
"attempt": attempt,
|
||||
"iteration": iteration,
|
||||
"is_current": True,
|
||||
"input": None,
|
||||
"output": None,
|
||||
"error": None,
|
||||
"parent_node_run_id": None,
|
||||
"duration_ms": None,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"created_at": _iso(datetime.now(UTC)),
|
||||
}
|
||||
rows.append(existing)
|
||||
for key in (
|
||||
"status",
|
||||
"node_type",
|
||||
"input",
|
||||
"output",
|
||||
"error",
|
||||
"parent_node_run_id",
|
||||
"duration_ms",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
):
|
||||
if key in data and data[key] is not None:
|
||||
existing[key] = _iso(data[key]) if isinstance(data[key], datetime) else data[key]
|
||||
if "is_current" in data:
|
||||
existing["is_current"] = bool(data["is_current"])
|
||||
return copy.deepcopy(existing)
|
||||
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
rows = self._node_runs.get(run_id, [])
|
||||
if current_only:
|
||||
rows = [r for r in rows if r["is_current"]]
|
||||
return [copy.deepcopy(r) for r in rows]
|
||||
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
count = 0
|
||||
for row in self._node_runs.get(run_id, []):
|
||||
if row["node_id"] in node_ids and row["is_current"]:
|
||||
row["is_current"] = False
|
||||
count += 1
|
||||
return count
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = {
|
||||
"id": str(data.get("id") or uuid.uuid4()),
|
||||
"run_id": str(data["run_id"]),
|
||||
"node_id": str(data.get("node_id") or ""),
|
||||
"name": str(data.get("name") or ""),
|
||||
"mime_type": str(data.get("mime_type") or "application/octet-stream"),
|
||||
"path": str(data.get("path") or ""),
|
||||
"size_bytes": data.get("size_bytes"),
|
||||
"preview": data.get("preview"),
|
||||
"created_at": _iso(datetime.now(UTC)),
|
||||
}
|
||||
self._artifacts.setdefault(row["run_id"], []).append(row)
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
return [copy.deepcopy(r) for r in self._artifacts.get(run_id, [])]
|
||||
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
for row in self._artifacts.get(run_id, []):
|
||||
if row["id"] == artifact_id:
|
||||
return copy.deepcopy(row)
|
||||
return None
|
||||
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._lock:
|
||||
deleted: list[str] = []
|
||||
for run_id, run in list(self._runs.items()):
|
||||
if run["status"] not in TERMINAL_RUN_STATUSES:
|
||||
continue
|
||||
finished = _parse_dt(run.get("finished_at"))
|
||||
if finished is None or finished >= cutoff:
|
||||
continue
|
||||
deleted.append(run_id)
|
||||
for run_id in deleted:
|
||||
self._runs.pop(run_id, None)
|
||||
self._node_runs.pop(run_id, None)
|
||||
self._artifacts.pop(run_id, None)
|
||||
return deleted
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
return None
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
"""ORM models for workflow runs and per-node runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class WorkflowRunRow(Base):
|
||||
"""One workflow execution bound to an immutable published version."""
|
||||
|
||||
__tablename__ = "workflow_runs"
|
||||
__table_args__ = (UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
workflow_version_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# queued | running | awaiting_input | completed | failed | cancel_requested | cancelled
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
|
||||
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
input_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
output_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
# Non-sensitive run context (never credentials).
|
||||
context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
# Pending human-input descriptor when status == awaiting_input.
|
||||
pending_input_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
resume_token: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
heartbeat_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3, server_default=sa_text("3"))
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
# Last seq handed out by the event log for this run (0 = none yet). The
|
||||
# event store increments it in the same transaction that inserts the event
|
||||
# row, replacing the MAX(seq)+1 race; "0" backfills existing rows and the
|
||||
# first append self-heals from MAX(seq) on conflict.
|
||||
next_event_seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
retry_of_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class WorkflowNodeRunRow(Base):
|
||||
"""Per-node execution record (also the resume checkpoint for a paused run)."""
|
||||
|
||||
__tablename__ = "workflow_node_runs"
|
||||
__table_args__ = (UniqueConstraint("run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
node_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# queued | running | completed | failed | skipped | cancelled
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default=sa_text("1"))
|
||||
iteration: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
is_current: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
output_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
parent_node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class WorkflowRunArtifactRow(Base):
|
||||
"""Artifact reference produced by a run (bytes live in the sandbox/thread dir)."""
|
||||
|
||||
__tablename__ = "workflow_run_artifacts"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
mime_type: Mapped[str] = mapped_column(String(191), nullable=False, default="application/octet-stream")
|
||||
path: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
preview: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+566
@@ -0,0 +1,566 @@
|
||||
"""SQLAlchemy-backed workflow run store (lease dispatch + resume checkpoints)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
|
||||
_RUN_UPDATABLE = {
|
||||
"status",
|
||||
"output_json",
|
||||
"error_json",
|
||||
"context_json",
|
||||
"pending_input_json",
|
||||
"resume_token",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
}
|
||||
|
||||
# Retention deletes in bounded batches so a long-neglected table cannot produce
|
||||
# one enormous transaction.
|
||||
_PURGE_BATCH = 500
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _load_text_json(raw: str | None) -> Any:
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _dump_text_json(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
|
||||
|
||||
def run_to_dict(row: WorkflowRunRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"workflow_version_id": row.workflow_version_id,
|
||||
"status": row.status,
|
||||
"owner_id": row.owner_id,
|
||||
"idempotency_key": row.idempotency_key,
|
||||
"input": row.input_json or {},
|
||||
"output": row.output_json,
|
||||
"error": row.error_json,
|
||||
"context": row.context_json or {},
|
||||
"pending_input": row.pending_input_json,
|
||||
"resume_token": row.resume_token,
|
||||
"lease_owner": row.lease_owner,
|
||||
"lease_until": _iso(row.lease_until),
|
||||
"heartbeat_at": _iso(row.heartbeat_at),
|
||||
"attempt": row.attempt,
|
||||
"max_attempts": row.max_attempts,
|
||||
"version": row.version,
|
||||
"retry_of_run_id": row.retry_of_run_id,
|
||||
# Last seq handed out by the event log (0 = no events yet).
|
||||
"next_event_seq": int(row.next_event_seq or 0),
|
||||
"started_at": _iso(row.started_at),
|
||||
"finished_at": _iso(row.finished_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
def node_run_to_dict(row: WorkflowNodeRunRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"run_id": row.run_id,
|
||||
"node_id": row.node_id,
|
||||
"node_type": row.node_type,
|
||||
"status": row.status,
|
||||
"attempt": row.attempt,
|
||||
"iteration": row.iteration,
|
||||
"is_current": bool(row.is_current),
|
||||
"input": _load_text_json(row.input_snapshot),
|
||||
"output": _load_text_json(row.output_snapshot),
|
||||
"error": row.error_json,
|
||||
"parent_node_run_id": row.parent_node_run_id,
|
||||
"duration_ms": row.duration_ms,
|
||||
"started_at": _iso(row.started_at),
|
||||
"finished_at": _iso(row.finished_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
def artifact_to_dict(row: WorkflowRunArtifactRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"run_id": row.run_id,
|
||||
"node_id": row.node_id,
|
||||
"name": row.name,
|
||||
"mime_type": row.mime_type,
|
||||
"path": row.path,
|
||||
"size_bytes": row.size_bytes,
|
||||
"preview": row.preview,
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
class SqlWorkflowRunStore(WorkflowRunStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
key = data.get("idempotency_key")
|
||||
if key:
|
||||
existing = await self._get_by_idempotency(str(key))
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
row = WorkflowRunRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
workflow_id=str(data["workflow_id"]),
|
||||
workflow_version_id=str(data["workflow_version_id"]),
|
||||
status=str(data.get("status") or "queued"),
|
||||
owner_id=str(data["owner_id"]),
|
||||
idempotency_key=str(key) if key else None,
|
||||
input_json=data.get("input") or {},
|
||||
context_json=data.get("context") or {},
|
||||
max_attempts=int(data.get("max_attempts") or 3),
|
||||
retry_of_run_id=data.get("retry_of_run_id"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
try:
|
||||
await session.flush()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
if key:
|
||||
existing = await self._get_by_idempotency(str(key))
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
raise
|
||||
out = run_to_dict(row)
|
||||
await session.commit()
|
||||
return out, True
|
||||
|
||||
async def _get_by_idempotency(self, key: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(WorkflowRunRow).where(WorkflowRunRow.idempotency_key == key))).scalar_one_or_none()
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunRow)
|
||||
if workflow_id:
|
||||
stmt = stmt.where(WorkflowRunRow.workflow_id == workflow_id)
|
||||
if owner_id:
|
||||
stmt = stmt.where(WorkflowRunRow.owner_id == owner_id)
|
||||
if status:
|
||||
stmt = stmt.where(WorkflowRunRow.status == status)
|
||||
stmt = stmt.order_by(WorkflowRunRow.created_at.desc()).limit(limit).offset(offset)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [run_to_dict(r) for r in rows]
|
||||
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
stmt = (
|
||||
select(WorkflowRunRow.id)
|
||||
.where(
|
||||
or_(
|
||||
WorkflowRunRow.status == "queued",
|
||||
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
|
||||
)
|
||||
)
|
||||
.order_by(WorkflowRunRow.created_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
return [str(v) for v in (await session.execute(stmt)).scalars().all()]
|
||||
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
or_(
|
||||
WorkflowRunRow.status == "queued",
|
||||
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
|
||||
),
|
||||
)
|
||||
.values(
|
||||
status="running",
|
||||
lease_owner=lease_owner,
|
||||
lease_until=lease_until,
|
||||
heartbeat_at=now,
|
||||
attempt=WorkflowRunRow.attempt + 1,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
started_at=func.coalesce(WorkflowRunRow.started_at, now),
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.lease_owner == lease_owner,
|
||||
WorkflowRunRow.status == "running",
|
||||
)
|
||||
.values(lease_until=lease_until, heartbeat_at=now, updated_at=now)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return False
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
values: dict[str, Any] = {"updated_at": now, "version": WorkflowRunRow.version + 1}
|
||||
for key, value in fields.items():
|
||||
if key in _RUN_UPDATABLE:
|
||||
values[key] = value
|
||||
status = fields.get("status")
|
||||
if status in TERMINAL_RUN_STATUSES:
|
||||
values["lease_owner"] = None
|
||||
values["lease_until"] = None
|
||||
values["resume_token"] = None
|
||||
values["pending_input_json"] = None
|
||||
values.setdefault("finished_at", now)
|
||||
where = [WorkflowRunRow.id == run_id]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status.in_(list(CANCELLABLE_RUN_STATUSES)),
|
||||
)
|
||||
.values(status="cancel_requested", version=WorkflowRunRow.version + 1, updated_at=now)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row and row.status in ("cancel_requested", "cancelled") else None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "cancel_requested"]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(*where)
|
||||
.values(
|
||||
status="cancelled",
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
finished_at=now,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "running"]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
values: dict[str, Any] = {
|
||||
"status": "awaiting_input",
|
||||
"pending_input_json": pending_input,
|
||||
"resume_token": resume_token,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
"version": WorkflowRunRow.version + 1,
|
||||
"updated_at": now,
|
||||
}
|
||||
if context is not None:
|
||||
values["context_json"] = context
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status == "awaiting_input",
|
||||
WorkflowRunRow.resume_token == resume_token,
|
||||
)
|
||||
.values(
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
if context is None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
if row is None:
|
||||
return None
|
||||
context = dict(row.context_json or {})
|
||||
new_context = {**context, "resume_payload": payload}
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status == "awaiting_input",
|
||||
WorkflowRunRow.resume_token == resume_token,
|
||||
)
|
||||
.values(
|
||||
context_json=new_context,
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(data["run_id"])
|
||||
node_id = str(data["node_id"])
|
||||
attempt = int(data.get("attempt") or 1)
|
||||
iteration = int(data.get("iteration") or 0)
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(WorkflowNodeRunRow).where(
|
||||
WorkflowNodeRunRow.run_id == run_id,
|
||||
WorkflowNodeRunRow.node_id == node_id,
|
||||
WorkflowNodeRunRow.attempt == attempt,
|
||||
WorkflowNodeRunRow.iteration == iteration,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = WorkflowNodeRunRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
node_id=node_id,
|
||||
node_type=str(data.get("node_type") or "unknown"),
|
||||
attempt=attempt,
|
||||
iteration=iteration,
|
||||
)
|
||||
session.add(row)
|
||||
row.status = str(data.get("status") or row.status)
|
||||
row.node_type = str(data.get("node_type") or row.node_type)
|
||||
row.is_current = bool(data.get("is_current", True))
|
||||
if "input" in data:
|
||||
row.input_snapshot = _dump_text_json(data.get("input"))
|
||||
if "output" in data:
|
||||
row.output_snapshot = _dump_text_json(data.get("output"))
|
||||
if "error" in data:
|
||||
row.error_json = data.get("error")
|
||||
if data.get("parent_node_run_id"):
|
||||
row.parent_node_run_id = str(data["parent_node_run_id"])
|
||||
if data.get("duration_ms") is not None:
|
||||
row.duration_ms = int(data["duration_ms"])
|
||||
if data.get("started_at") is not None:
|
||||
row.started_at = data["started_at"]
|
||||
if data.get("finished_at") is not None:
|
||||
row.finished_at = data["finished_at"]
|
||||
await session.flush()
|
||||
out = node_run_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id == run_id)
|
||||
if current_only:
|
||||
stmt = stmt.where(WorkflowNodeRunRow.is_current.is_(True))
|
||||
stmt = stmt.order_by(WorkflowNodeRunRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [node_run_to_dict(r) for r in rows]
|
||||
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
if not node_ids:
|
||||
return 0
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowNodeRunRow)
|
||||
.where(
|
||||
WorkflowNodeRunRow.run_id == run_id,
|
||||
WorkflowNodeRunRow.node_id.in_(node_ids),
|
||||
WorkflowNodeRunRow.is_current.is_(True),
|
||||
)
|
||||
.values(is_current=False)
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = WorkflowRunArtifactRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
run_id=str(data["run_id"]),
|
||||
node_id=str(data.get("node_id") or ""),
|
||||
name=str(data.get("name") or "")[:512],
|
||||
mime_type=str(data.get("mime_type") or "application/octet-stream")[:191],
|
||||
path=str(data.get("path") or ""),
|
||||
size_bytes=data.get("size_bytes"),
|
||||
preview=data.get("preview"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
out = artifact_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id == run_id).order_by(WorkflowRunArtifactRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [artifact_to_dict(r) for r in rows]
|
||||
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
stmt = select(WorkflowRunArtifactRow).where(
|
||||
WorkflowRunArtifactRow.id == artifact_id,
|
||||
WorkflowRunArtifactRow.run_id == run_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return artifact_to_dict(row) if row is not None else None
|
||||
|
||||
# ── retention ───────────────────────────────────────────────────────
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._sf() as session:
|
||||
run_ids = (
|
||||
(
|
||||
await session.execute(
|
||||
select(WorkflowRunRow.id)
|
||||
.where(
|
||||
WorkflowRunRow.status.in_(TERMINAL_RUN_STATUSES),
|
||||
WorkflowRunRow.finished_at.is_not(None),
|
||||
WorkflowRunRow.finished_at < cutoff,
|
||||
)
|
||||
.limit(_PURGE_BATCH)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not run_ids:
|
||||
return []
|
||||
ids = list(run_ids)
|
||||
await session.execute(delete(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id.in_(ids)))
|
||||
await session.execute(delete(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id.in_(ids)))
|
||||
await session.execute(delete(WorkflowRunRow).where(WorkflowRunRow.id.in_(ids)))
|
||||
await session.commit()
|
||||
return ids
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
"""Workflow definition / version persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflows.base import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
WorkflowVersionNotFoundError,
|
||||
)
|
||||
from deerflow.persistence.workflows.memory import MemoryWorkflowStore
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
from deerflow.persistence.workflows.sql import WorkflowRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowStore",
|
||||
"WorkflowDefinitionRow",
|
||||
"WorkflowDraftConflictError",
|
||||
"WorkflowNotFoundError",
|
||||
"WorkflowRepository",
|
||||
"WorkflowStore",
|
||||
"WorkflowVersionNotFoundError",
|
||||
"WorkflowVersionRow",
|
||||
"make_workflow_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowStore()
|
||||
return WorkflowRepository(session_factory)
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
"""Abstract workflow definition / version store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class WorkflowDraftConflictError(Exception):
|
||||
"""Raised when ``expected_revision`` does not match the stored draft revision."""
|
||||
|
||||
def __init__(self, workflow_id: str, current_revision: int) -> None:
|
||||
super().__init__(f"draft conflict for {workflow_id}: current={current_revision}")
|
||||
self.workflow_id = workflow_id
|
||||
self.current_revision = current_revision
|
||||
|
||||
|
||||
class WorkflowNotFoundError(Exception):
|
||||
def __init__(self, workflow_id: str) -> None:
|
||||
super().__init__(f"workflow not found: {workflow_id}")
|
||||
self.workflow_id = workflow_id
|
||||
|
||||
|
||||
class WorkflowVersionNotFoundError(Exception):
|
||||
def __init__(self, version_id: str) -> None:
|
||||
super().__init__(f"workflow version not found: {version_id}")
|
||||
self.version_id = version_id
|
||||
|
||||
|
||||
class WorkflowStore(ABC):
|
||||
@abstractmethod
|
||||
async def list_definitions(
|
||||
self,
|
||||
*,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def save_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""CAS update of ``draft_graph_json``. Raises ``WorkflowDraftConflictError``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def save_studio_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
canvas_schema: str = "{}",
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically CAS-update BOTH the execution graph and the canvas document.
|
||||
|
||||
A single conditional UPDATE bumps ``draft_revision`` and writes
|
||||
``draft_graph_json`` + ``draft_canvas_schema_json`` together, so the two
|
||||
representations can never diverge across revisions. Raises
|
||||
``WorkflowDraftConflictError`` on a stale ``expected_revision``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def publish_version(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
graph: dict[str, Any],
|
||||
graph_hash: str,
|
||||
published_by: str | None = None,
|
||||
change_note: str = "",
|
||||
canvas_schema_hash: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_version(self, version_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
|
||||
"""Lightweight list for subworkflow resource catalog."""
|
||||
raise NotImplementedError
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
"""In-memory workflow store for tests / no-DB mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflows.base import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class MemoryWorkflowStore(WorkflowStore):
|
||||
def __init__(self) -> None:
|
||||
self._definitions: dict[str, dict[str, Any]] = {}
|
||||
self._versions: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def list_definitions(
|
||||
self,
|
||||
*,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = list(self._definitions.values())
|
||||
if owner_id:
|
||||
rows = [r for r in rows if r["owner_id"] == owner_id]
|
||||
if status:
|
||||
rows = [r for r in rows if r["status"] == status]
|
||||
elif not include_archived:
|
||||
rows = [r for r in rows if r["status"] != "archived"]
|
||||
rows.sort(key=lambda r: r.get("updated_at") or "", reverse=True)
|
||||
return [
|
||||
{k: v for k, v in r.items() if k not in ("draft_graph", "draft_canvas_schema", "next_version_number")}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
return None
|
||||
out = copy.deepcopy(row)
|
||||
if not include_draft:
|
||||
out.pop("draft_graph", None)
|
||||
return out
|
||||
|
||||
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
|
||||
graph = copy.deepcopy(data.get("draft_graph") or data.get("graph") or {
|
||||
"schemaVersion": "1.0",
|
||||
"id": workflow_id,
|
||||
"name": data.get("name") or "未命名工作流",
|
||||
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
|
||||
"edges": [],
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
"outputSchema": {"type": "object", "properties": {}},
|
||||
"settings": {},
|
||||
})
|
||||
graph.setdefault("id", workflow_id)
|
||||
now = _now()
|
||||
row = {
|
||||
"id": workflow_id,
|
||||
"name": data.get("name") or graph.get("name") or "未命名工作流",
|
||||
"description": data.get("description") or "",
|
||||
"owner_id": data["owner_id"],
|
||||
"status": data.get("status") or "draft",
|
||||
"draft_revision": 0,
|
||||
"next_version_number": 1,
|
||||
"draft_graph": graph,
|
||||
"draft_canvas_schema": str(data.get("draft_canvas_schema") or "{}"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._definitions[workflow_id] = row
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
return None
|
||||
if "name" in data and data["name"] is not None:
|
||||
row["name"] = data["name"]
|
||||
if "description" in data and data["description"] is not None:
|
||||
row["description"] = data["description"]
|
||||
if "status" in data and data["status"] is not None:
|
||||
row["status"] = data["status"]
|
||||
row["updated_at"] = _now()
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
return await self.update_definition(workflow_id, {"status": "archived"})
|
||||
|
||||
async def save_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row["draft_revision"] != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
|
||||
payload = copy.deepcopy(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
row["draft_graph"] = payload
|
||||
row["draft_revision"] = expected_revision + 1
|
||||
if payload.get("name"):
|
||||
row["name"] = str(payload["name"])[:255]
|
||||
row["updated_at"] = _now()
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def save_studio_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
canvas_schema: str = "{}",
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row["draft_revision"] != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
|
||||
payload = copy.deepcopy(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
row["draft_graph"] = payload
|
||||
row["draft_canvas_schema"] = canvas_schema or "{}"
|
||||
row["draft_revision"] = expected_revision + 1
|
||||
if payload.get("name"):
|
||||
row["name"] = str(payload["name"])[:255]
|
||||
row["updated_at"] = _now()
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def publish_version(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
graph: dict[str, Any],
|
||||
graph_hash: str,
|
||||
published_by: str | None = None,
|
||||
change_note: str = "",
|
||||
canvas_schema_hash: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
definition = self._definitions.get(workflow_id)
|
||||
if definition is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
existing = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
|
||||
# Allocate from the definition's own counter (mirrors the SQL store) so
|
||||
# the number never goes backwards even if events are purged/replayed.
|
||||
counter = int(definition.get("next_version_number") or 1)
|
||||
version_number = max(counter, max((v["version_number"] for v in existing), default=0) + 1)
|
||||
definition["next_version_number"] = version_number + 1
|
||||
version_id = f"wv_{uuid.uuid4().hex[:12]}"
|
||||
version = {
|
||||
"id": version_id,
|
||||
"workflow_id": workflow_id,
|
||||
"version_number": version_number,
|
||||
"graph": copy.deepcopy(graph),
|
||||
"graph_hash": graph_hash,
|
||||
"input_schema": graph.get("inputSchema") or graph.get("input_schema") or {},
|
||||
"output_schema": graph.get("outputSchema") or graph.get("output_schema") or {},
|
||||
"published_by": published_by,
|
||||
"published_at": _now(),
|
||||
"change_note": change_note or "",
|
||||
}
|
||||
self._versions[version_id] = version
|
||||
definition["status"] = "active"
|
||||
if canvas_schema_hash:
|
||||
definition["published_canvas_schema_hash"] = str(canvas_schema_hash)[:64]
|
||||
definition["updated_at"] = _now()
|
||||
return copy.deepcopy(version)
|
||||
|
||||
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
rows = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
|
||||
rows.sort(key=lambda r: r["version_number"], reverse=True)
|
||||
return [{k: v for k, v in r.items() if k != "graph"} for r in rows]
|
||||
|
||||
async def get_version(self, version_id: str) -> dict[str, Any] | None:
|
||||
row = self._versions.get(version_id)
|
||||
return copy.deepcopy(row) if row is not None else None
|
||||
|
||||
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
|
||||
by_wf: dict[str, dict[str, Any]] = {}
|
||||
for version in self._versions.values():
|
||||
wf_id = version["workflow_id"]
|
||||
if exclude_workflow_id and wf_id == exclude_workflow_id:
|
||||
continue
|
||||
definition = self._definitions.get(wf_id)
|
||||
if definition is None or definition.get("status") == "archived":
|
||||
continue
|
||||
current = by_wf.get(wf_id)
|
||||
if current is None or version["version_number"] > current["version_number"]:
|
||||
by_wf[wf_id] = {
|
||||
"workflow_id": wf_id,
|
||||
"name": definition["name"],
|
||||
"version_id": version["id"],
|
||||
"version_number": version["version_number"],
|
||||
"graph_hash": version["graph_hash"],
|
||||
"published_at": version["published_at"],
|
||||
}
|
||||
return sorted(by_wf.values(), key=lambda r: r.get("published_at") or "", reverse=True)
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
"""ORM models for workflow definitions and immutable published versions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class WorkflowDefinitionRow(Base):
|
||||
"""Mutable workflow metadata + current draft canvas."""
|
||||
|
||||
__tablename__ = "workflow_definitions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
description: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# draft | active | archived
|
||||
status: Mapped[str] = mapped_column(String(32), default="draft", nullable=False, index=True)
|
||||
draft_revision: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
# Monotonic version allocator. "1" = no version published yet; publish takes
|
||||
# the value then CAS-bumps it, replacing the old MAX(version_number)+1 race.
|
||||
next_version_number: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=1, server_default=sa_text("1")
|
||||
)
|
||||
draft_graph_json: Mapped[str] = mapped_column(PortableLongText(), default="{}", nullable=False)
|
||||
# Coze canvas document (opaque JSON text). Never interpreted by the runtime;
|
||||
# the executable WorkflowGraph lives in draft_graph_json above.
|
||||
draft_canvas_schema_json: Mapped[str] = mapped_column(
|
||||
PortableLongText(), default="{}", server_default=sa_text("'{}'"), nullable=False
|
||||
)
|
||||
# sha256 of the canvas document the current published version was compiled
|
||||
# from (audit trail only; publish always freezes the execution graph).
|
||||
published_canvas_schema_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class WorkflowVersionRow(Base):
|
||||
"""Immutable published graph snapshot. Runs bind to version id."""
|
||||
|
||||
__tablename__ = "workflow_versions"
|
||||
__table_args__ = (UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
workflow_id: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
ForeignKey("workflow_definitions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
graph_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
|
||||
graph_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
input_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
output_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
published_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
change_note: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
+385
@@ -0,0 +1,385 @@
|
||||
"""SQLAlchemy-backed workflow definition / version store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflows.base import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
|
||||
# Concurrent publishers CAS-retry against the workflow's own counter; a handful
|
||||
# of attempts is far more than any realistic contention window needs.
|
||||
_PUBLISH_ATTEMPTS = 6
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _parse_graph(raw: str | dict[str, Any] | None) -> dict[str, Any]:
|
||||
if raw is None:
|
||||
return {}
|
||||
if isinstance(raw, dict):
|
||||
return raw
|
||||
text = str(raw).strip()
|
||||
if not text:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _dump_graph(graph: dict[str, Any]) -> str:
|
||||
return json.dumps(graph, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
|
||||
def _definition_dict(row: WorkflowDefinitionRow, *, include_draft: bool) -> dict[str, Any]:
|
||||
data = {
|
||||
"id": row.id,
|
||||
"name": row.name,
|
||||
"description": row.description or "",
|
||||
"owner_id": row.owner_id,
|
||||
"status": row.status,
|
||||
"draft_revision": row.draft_revision,
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
if include_draft:
|
||||
data["draft_graph"] = _parse_graph(row.draft_graph_json)
|
||||
# Canvas document as stored (opaque JSON text; "{}" when never saved).
|
||||
data["draft_canvas_schema"] = row.draft_canvas_schema_json or "{}"
|
||||
return data
|
||||
|
||||
|
||||
def _version_dict(row: WorkflowVersionRow, *, include_graph: bool = True) -> dict[str, Any]:
|
||||
data = {
|
||||
"id": row.id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"version_number": row.version_number,
|
||||
"graph_hash": row.graph_hash,
|
||||
"input_schema": row.input_schema or {},
|
||||
"output_schema": row.output_schema or {},
|
||||
"published_by": row.published_by,
|
||||
"published_at": _iso(row.published_at),
|
||||
"change_note": row.change_note or "",
|
||||
}
|
||||
if include_graph:
|
||||
data["graph"] = _parse_graph(row.graph_json)
|
||||
return data
|
||||
|
||||
|
||||
class WorkflowRepository(WorkflowStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_definitions(
|
||||
self,
|
||||
*,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowDefinitionRow)
|
||||
if owner_id:
|
||||
stmt = stmt.where(WorkflowDefinitionRow.owner_id == owner_id)
|
||||
if status:
|
||||
stmt = stmt.where(WorkflowDefinitionRow.status == status)
|
||||
elif not include_archived:
|
||||
stmt = stmt.where(WorkflowDefinitionRow.status != "archived")
|
||||
stmt = stmt.order_by(WorkflowDefinitionRow.updated_at.desc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_definition_dict(row, include_draft=False) for row in result.scalars()]
|
||||
|
||||
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
return _definition_dict(row, include_draft=include_draft) if row is not None else None
|
||||
|
||||
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
|
||||
graph = data.get("draft_graph") or data.get("graph") or {
|
||||
"schemaVersion": "1.0",
|
||||
"id": workflow_id,
|
||||
"name": data.get("name") or "未命名工作流",
|
||||
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
|
||||
"edges": [],
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
"outputSchema": {"type": "object", "properties": {}},
|
||||
"settings": {},
|
||||
}
|
||||
if isinstance(graph, dict) and not graph.get("id"):
|
||||
graph = {**graph, "id": workflow_id}
|
||||
if isinstance(graph, dict) and data.get("name") and not graph.get("name"):
|
||||
graph = {**graph, "name": data["name"]}
|
||||
row = WorkflowDefinitionRow(
|
||||
id=workflow_id,
|
||||
name=data.get("name") or graph.get("name") or "未命名工作流",
|
||||
description=data.get("description") or "",
|
||||
owner_id=data["owner_id"],
|
||||
status=data.get("status") or "draft",
|
||||
draft_revision=0,
|
||||
draft_graph_json=_dump_graph(graph),
|
||||
draft_canvas_schema_json=str(data.get("draft_canvas_schema") or "{}"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
out = _definition_dict(row, include_draft=True)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if row is None:
|
||||
return None
|
||||
if "name" in data and data["name"] is not None:
|
||||
row.name = data["name"]
|
||||
if "description" in data and data["description"] is not None:
|
||||
row.description = data["description"]
|
||||
if "status" in data and data["status"] is not None:
|
||||
row.status = data["status"]
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.flush()
|
||||
out = _definition_dict(row, include_draft=True)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
return await self.update_definition(workflow_id, {"status": "archived"})
|
||||
|
||||
async def save_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by # audit reserved for later
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row.draft_revision != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
|
||||
payload = dict(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
if row.name and not payload.get("name"):
|
||||
payload["name"] = row.name
|
||||
# Conditional UPDATE on the revision itself: two sessions that both
|
||||
# read the same revision cannot both win — the loser's WHERE clause
|
||||
# matches zero rows and surfaces a deterministic conflict instead of
|
||||
# silently overwriting the other editor's draft.
|
||||
values: dict[str, Any] = {
|
||||
"draft_graph_json": _dump_graph(payload),
|
||||
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
|
||||
"updated_at": datetime.now(UTC),
|
||||
}
|
||||
if payload.get("name"):
|
||||
values["name"] = str(payload["name"])[:255]
|
||||
result = await session.execute(
|
||||
update(WorkflowDefinitionRow)
|
||||
.where(
|
||||
WorkflowDefinitionRow.id == workflow_id,
|
||||
WorkflowDefinitionRow.draft_revision == expected_revision,
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
current = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if current is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
|
||||
await session.commit()
|
||||
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
assert fresh is not None
|
||||
return _definition_dict(fresh, include_draft=True)
|
||||
|
||||
async def save_studio_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
canvas_schema: str = "{}",
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by # audit reserved for later
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row.draft_revision != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
|
||||
payload = dict(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
if row.name and not payload.get("name"):
|
||||
payload["name"] = row.name
|
||||
# One conditional UPDATE writes graph + canvas + revision together:
|
||||
# the two representations are always from the same save, and a
|
||||
# concurrent editor loses deterministically instead of interleaving
|
||||
# a new canvas onto someone else's graph.
|
||||
values: dict[str, Any] = {
|
||||
"draft_graph_json": _dump_graph(payload),
|
||||
"draft_canvas_schema_json": canvas_schema or "{}",
|
||||
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
|
||||
"updated_at": datetime.now(UTC),
|
||||
}
|
||||
if payload.get("name"):
|
||||
values["name"] = str(payload["name"])[:255]
|
||||
result = await session.execute(
|
||||
update(WorkflowDefinitionRow)
|
||||
.where(
|
||||
WorkflowDefinitionRow.id == workflow_id,
|
||||
WorkflowDefinitionRow.draft_revision == expected_revision,
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
current = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if current is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
|
||||
await session.commit()
|
||||
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
assert fresh is not None
|
||||
return _definition_dict(fresh, include_draft=True)
|
||||
|
||||
async def publish_version(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
graph: dict[str, Any],
|
||||
graph_hash: str,
|
||||
published_by: str | None = None,
|
||||
change_note: str = "",
|
||||
canvas_schema_hash: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Allocate the version number from the workflow's own counter.
|
||||
|
||||
``MAX(version_number) + 1`` races between concurrent publishes; here the
|
||||
number is claimed by CAS-bumping ``next_version_number`` inside the same
|
||||
transaction that inserts the version row, and the unique
|
||||
``(workflow_id, version_number)`` constraint is the final safety net.
|
||||
Concurrent publishers get adjacent, distinct numbers.
|
||||
"""
|
||||
last_error: Exception | None = None
|
||||
for _ in range(_PUBLISH_ATTEMPTS):
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
definition = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if definition is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
counter_seen = int(definition.next_version_number or 1)
|
||||
# Legacy rows got the column auto-added with server_default 1
|
||||
# while versions already exist — never allocate below the max.
|
||||
latest = (
|
||||
await session.execute(
|
||||
select(func.max(WorkflowVersionRow.version_number)).where(
|
||||
WorkflowVersionRow.workflow_id == workflow_id
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
number = max(counter_seen, int(latest) + 1)
|
||||
claimed = await session.execute(
|
||||
update(WorkflowDefinitionRow)
|
||||
.where(
|
||||
WorkflowDefinitionRow.id == workflow_id,
|
||||
WorkflowDefinitionRow.next_version_number == counter_seen,
|
||||
)
|
||||
.values(next_version_number=number + 1)
|
||||
)
|
||||
if claimed.rowcount == 0:
|
||||
# Another publish landed first — release nothing (the
|
||||
# counter bump never happened) and retry with fresh state.
|
||||
await session.rollback()
|
||||
last_error = None
|
||||
continue
|
||||
version = WorkflowVersionRow(
|
||||
id=f"wv_{uuid.uuid4().hex[:12]}",
|
||||
workflow_id=workflow_id,
|
||||
version_number=number,
|
||||
graph_json=_dump_graph(graph),
|
||||
graph_hash=graph_hash,
|
||||
input_schema=graph.get("inputSchema") or graph.get("input_schema") or {},
|
||||
output_schema=graph.get("outputSchema") or graph.get("output_schema") or {},
|
||||
published_by=published_by,
|
||||
change_note=change_note or "",
|
||||
)
|
||||
definition.status = "active"
|
||||
definition.updated_at = datetime.now(UTC)
|
||||
if canvas_schema_hash:
|
||||
definition.published_canvas_schema_hash = canvas_schema_hash[:64]
|
||||
session.add(version)
|
||||
await session.flush()
|
||||
out = _version_dict(version, include_graph=True)
|
||||
await session.commit()
|
||||
return out
|
||||
except IntegrityError as exc: # pragma: no cover - CAS makes this near-impossible
|
||||
last_error = exc
|
||||
continue
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
raise RuntimeError(f"failed to allocate a workflow version number for {workflow_id}")
|
||||
|
||||
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(WorkflowVersionRow)
|
||||
.where(WorkflowVersionRow.workflow_id == workflow_id)
|
||||
.order_by(WorkflowVersionRow.version_number.desc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_version_dict(row, include_graph=False) for row in result.scalars()]
|
||||
|
||||
async def get_version(self, version_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowVersionRow, version_id)
|
||||
return _version_dict(row, include_graph=True) if row is not None else None
|
||||
|
||||
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(WorkflowVersionRow, WorkflowDefinitionRow)
|
||||
.join(WorkflowDefinitionRow, WorkflowDefinitionRow.id == WorkflowVersionRow.workflow_id)
|
||||
.where(WorkflowDefinitionRow.status != "archived")
|
||||
.order_by(WorkflowVersionRow.published_at.desc())
|
||||
)
|
||||
if exclude_workflow_id:
|
||||
stmt = stmt.where(WorkflowVersionRow.workflow_id != exclude_workflow_id)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for version, definition in result.all():
|
||||
# Latest version per workflow only.
|
||||
if version.workflow_id in seen:
|
||||
continue
|
||||
seen.add(version.workflow_id)
|
||||
out.append(
|
||||
{
|
||||
"workflow_id": definition.id,
|
||||
"name": definition.name,
|
||||
"version_id": version.id,
|
||||
"version_number": version.version_number,
|
||||
"graph_hash": version.graph_hash,
|
||||
"published_at": _iso(version.published_at),
|
||||
}
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Workflow Studio protocol package (phase 0 — frozen contracts).
|
||||
|
||||
Execution, persistence, and Gateway routers land in later phases.
|
||||
This package must never import ``app.*``.
|
||||
"""
|
||||
|
||||
from deerflow.workflows.errors import (
|
||||
ALL_WORKFLOW_ERROR_CODES,
|
||||
WorkflowError,
|
||||
WorkflowErrorBody,
|
||||
WorkflowErrorCode,
|
||||
)
|
||||
from deerflow.workflows.events import (
|
||||
ALL_WORKFLOW_EVENT_TYPES,
|
||||
TERMINAL_EVENT_TYPES,
|
||||
WorkflowEventEnvelope,
|
||||
WorkflowEventSink,
|
||||
WorkflowEventType,
|
||||
)
|
||||
from deerflow.workflows.schemas import (
|
||||
ALL_NODE_TYPES,
|
||||
ALL_RUN_STATUSES,
|
||||
SCHEMA_VERSION,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
ArtifactRef,
|
||||
NodeResult,
|
||||
NodeType,
|
||||
ResumeRunRequest,
|
||||
RunStatus,
|
||||
StartRunRequest,
|
||||
WorkflowEdge,
|
||||
WorkflowGraph,
|
||||
WorkflowGraphSettings,
|
||||
WorkflowNode,
|
||||
)
|
||||
from deerflow.workflows.validator import assert_publishable, validate_workflow_graph
|
||||
|
||||
__all__ = [
|
||||
"ALL_NODE_TYPES",
|
||||
"ALL_RUN_STATUSES",
|
||||
"ALL_WORKFLOW_ERROR_CODES",
|
||||
"ALL_WORKFLOW_EVENT_TYPES",
|
||||
"SCHEMA_VERSION",
|
||||
"TERMINAL_EVENT_TYPES",
|
||||
"TERMINAL_RUN_STATUSES",
|
||||
"ArtifactRef",
|
||||
"NodeResult",
|
||||
"NodeType",
|
||||
"ResumeRunRequest",
|
||||
"RunStatus",
|
||||
"StartRunRequest",
|
||||
"WorkflowEdge",
|
||||
"WorkflowError",
|
||||
"WorkflowErrorBody",
|
||||
"WorkflowErrorCode",
|
||||
"WorkflowEventEnvelope",
|
||||
"WorkflowEventSink",
|
||||
"WorkflowEventType",
|
||||
"WorkflowGraph",
|
||||
"WorkflowGraphSettings",
|
||||
"WorkflowNode",
|
||||
"assert_publishable",
|
||||
"validate_workflow_graph",
|
||||
]
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Stable workflow error codes and exception types (protocol v1).
|
||||
|
||||
User-visible ``message`` must never include stack traces, SQL passwords,
|
||||
full response bodies, or internal host paths. Full exceptions go to
|
||||
controlled server logs only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
WorkflowErrorCode = Literal[
|
||||
# graph / publish
|
||||
"WORKFLOW_SCHEMA_INVALID",
|
||||
"WORKFLOW_GRAPH_CYCLE",
|
||||
"WORKFLOW_NODE_NOT_FOUND",
|
||||
"WORKFLOW_EDGE_INVALID",
|
||||
"WORKFLOW_EXPRESSION_INVALID",
|
||||
"WORKFLOW_EXPRESSION_UPSTREAM_REQUIRED",
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"WORKFLOW_RESOURCE_FORBIDDEN",
|
||||
"WORKFLOW_LIMIT_EXCEEDED",
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
"WORKFLOW_DRAFT_CONFLICT",
|
||||
"WORKFLOW_VERSION_NOT_FOUND",
|
||||
# run lifecycle
|
||||
"WORKFLOW_RUN_NOT_FOUND",
|
||||
"WORKFLOW_RUN_CONFLICT",
|
||||
"WORKFLOW_RUN_NOT_RESUMABLE",
|
||||
"WORKFLOW_RESUME_TOKEN_INVALID",
|
||||
"WORKFLOW_CANCELLED",
|
||||
"WORKFLOW_TIMEOUT",
|
||||
"WORKFLOW_MAX_STEPS",
|
||||
"WORKFLOW_MAX_LOOP_ITERATIONS",
|
||||
"WORKFLOW_BUDGET_EXCEEDED",
|
||||
# node categories
|
||||
"WORKFLOW_INPUT_INVALID",
|
||||
"WORKFLOW_AGENT_FAILED",
|
||||
"WORKFLOW_SKILL_FAILED",
|
||||
"WORKFLOW_HTTP_POLICY_DENIED",
|
||||
"WORKFLOW_HTTP_FAILED",
|
||||
"WORKFLOW_SQL_POLICY_DENIED",
|
||||
"WORKFLOW_SQL_FAILED",
|
||||
"WORKFLOW_CODE_SANDBOX_FAILED",
|
||||
"WORKFLOW_TRANSFORM_FAILED",
|
||||
"WORKFLOW_CONDITION_FAILED",
|
||||
"WORKFLOW_MERGE_FAILED",
|
||||
"WORKFLOW_HUMAN_INPUT_INVALID",
|
||||
"WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
"WORKFLOW_LOOP_FAILED",
|
||||
# auth / embed
|
||||
"WORKFLOW_FORBIDDEN",
|
||||
"WORKFLOW_EMBED_TICKET_INVALID",
|
||||
"WORKFLOW_EMBED_ORIGIN_DENIED",
|
||||
# generic
|
||||
"WORKFLOW_INTERNAL",
|
||||
]
|
||||
|
||||
ALL_WORKFLOW_ERROR_CODES: tuple[WorkflowErrorCode, ...] = (
|
||||
"WORKFLOW_SCHEMA_INVALID",
|
||||
"WORKFLOW_GRAPH_CYCLE",
|
||||
"WORKFLOW_NODE_NOT_FOUND",
|
||||
"WORKFLOW_EDGE_INVALID",
|
||||
"WORKFLOW_EXPRESSION_INVALID",
|
||||
"WORKFLOW_EXPRESSION_UPSTREAM_REQUIRED",
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"WORKFLOW_RESOURCE_FORBIDDEN",
|
||||
"WORKFLOW_LIMIT_EXCEEDED",
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
"WORKFLOW_DRAFT_CONFLICT",
|
||||
"WORKFLOW_VERSION_NOT_FOUND",
|
||||
"WORKFLOW_RUN_NOT_FOUND",
|
||||
"WORKFLOW_RUN_CONFLICT",
|
||||
"WORKFLOW_RUN_NOT_RESUMABLE",
|
||||
"WORKFLOW_RESUME_TOKEN_INVALID",
|
||||
"WORKFLOW_CANCELLED",
|
||||
"WORKFLOW_TIMEOUT",
|
||||
"WORKFLOW_MAX_STEPS",
|
||||
"WORKFLOW_MAX_LOOP_ITERATIONS",
|
||||
"WORKFLOW_BUDGET_EXCEEDED",
|
||||
"WORKFLOW_INPUT_INVALID",
|
||||
"WORKFLOW_AGENT_FAILED",
|
||||
"WORKFLOW_SKILL_FAILED",
|
||||
"WORKFLOW_HTTP_POLICY_DENIED",
|
||||
"WORKFLOW_HTTP_FAILED",
|
||||
"WORKFLOW_SQL_POLICY_DENIED",
|
||||
"WORKFLOW_SQL_FAILED",
|
||||
"WORKFLOW_CODE_SANDBOX_FAILED",
|
||||
"WORKFLOW_TRANSFORM_FAILED",
|
||||
"WORKFLOW_CONDITION_FAILED",
|
||||
"WORKFLOW_MERGE_FAILED",
|
||||
"WORKFLOW_HUMAN_INPUT_INVALID",
|
||||
"WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
"WORKFLOW_LOOP_FAILED",
|
||||
"WORKFLOW_FORBIDDEN",
|
||||
"WORKFLOW_EMBED_TICKET_INVALID",
|
||||
"WORKFLOW_EMBED_ORIGIN_DENIED",
|
||||
"WORKFLOW_INTERNAL",
|
||||
)
|
||||
|
||||
|
||||
class WorkflowErrorBody(BaseModel):
|
||||
"""Wire-format error body shared by API responses and ``run.failed`` events."""
|
||||
|
||||
code: WorkflowErrorCode
|
||||
message: str
|
||||
retryable: bool = False
|
||||
node_id: str | None = Field(default=None, alias="nodeId")
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class WorkflowError(Exception):
|
||||
"""Raised by validators / node executors; convertible to :class:`WorkflowErrorBody`."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
code: WorkflowErrorCode,
|
||||
message: str,
|
||||
*,
|
||||
retryable: bool = False,
|
||||
node_id: str | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.retryable = retryable
|
||||
self.node_id = node_id
|
||||
self.details = details or {}
|
||||
|
||||
def to_body(self) -> WorkflowErrorBody:
|
||||
return WorkflowErrorBody(
|
||||
code=self.code,
|
||||
message=self.message,
|
||||
retryable=self.retryable,
|
||||
node_id=self.node_id,
|
||||
details=self.details,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ALL_WORKFLOW_ERROR_CODES",
|
||||
"WorkflowError",
|
||||
"WorkflowErrorBody",
|
||||
"WorkflowErrorCode",
|
||||
]
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Workflow SSE event protocol v1.0.
|
||||
|
||||
Invariant: **persist first, then publish**. ``seq`` is monotonic per run.
|
||||
Heartbeat comments (``: heartbeat``) are live-only and never stored.
|
||||
|
||||
Canonical ``data`` payloads (frozen contract — REST replay and SSE must agree):
|
||||
|
||||
node.tool.started {"name": "web_search", "toolCallId": "call_001",
|
||||
"title": "联网检索", "args": {"path": "..."}}
|
||||
node.tool.finished {"name": "web_search", "toolCallId": "call_001",
|
||||
"status": "succeeded" | "failed", "summary": "获得 12 条结果"}
|
||||
run.awaiting_input {"resumeToken": "one-time-token", "prompt": "请确认是否发布",
|
||||
"formSchema": {...}, "actions": ["approve", "reject"]}
|
||||
artifact.created {"artifactId": "artifact_001", "name": "weekly-report.docx",
|
||||
"kind": "file" | "markdown" | "json" | "table",
|
||||
"mimeType": "...", "sizeBytes": 1234, "preview": "..."}
|
||||
node.started {"nodeType": ..., "name": ..., "attempt": 1,
|
||||
"agent": {...}, "parallelGroupId": "...", "roundIndex": 2}
|
||||
|
||||
Rules: tool identity is ``name`` (never ``toolName``); outcomes are ``status``
|
||||
(never ``ok``); human-input descriptors are flattened (never nested under
|
||||
``pendingInput``); artifacts never carry filesystem paths; agent nodes expose
|
||||
display metadata only (no system prompts, skill whitelists or credentials).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Literal, Protocol, runtime_checkable
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
SCHEMA_VERSION = "1.0"
|
||||
|
||||
WorkflowEventType = Literal[
|
||||
"run.created",
|
||||
"run.queued",
|
||||
"run.started",
|
||||
"node.queued",
|
||||
"node.started",
|
||||
"node.progress",
|
||||
"node.output.delta",
|
||||
"node.tool.started",
|
||||
"node.tool.finished",
|
||||
"artifact.created",
|
||||
"node.completed",
|
||||
"node.failed",
|
||||
"run.awaiting_input",
|
||||
"run.resumed",
|
||||
"run.cancel_requested",
|
||||
"run.cancelled",
|
||||
"run.completed",
|
||||
"run.failed",
|
||||
]
|
||||
|
||||
ALL_WORKFLOW_EVENT_TYPES: tuple[WorkflowEventType, ...] = (
|
||||
"run.created",
|
||||
"run.queued",
|
||||
"run.started",
|
||||
"node.queued",
|
||||
"node.started",
|
||||
"node.progress",
|
||||
"node.output.delta",
|
||||
"node.tool.started",
|
||||
"node.tool.finished",
|
||||
"artifact.created",
|
||||
"node.completed",
|
||||
"node.failed",
|
||||
"run.awaiting_input",
|
||||
"run.resumed",
|
||||
"run.cancel_requested",
|
||||
"run.cancelled",
|
||||
"run.completed",
|
||||
"run.failed",
|
||||
)
|
||||
|
||||
TERMINAL_EVENT_TYPES: frozenset[WorkflowEventType] = frozenset({"run.completed", "run.failed", "run.cancelled"})
|
||||
|
||||
_KIND_BY_MIME: tuple[tuple[tuple[str, ...], str], ...] = (
|
||||
(("text/markdown", "text/x-markdown"), "markdown"),
|
||||
(("application/json", "text/json", "application/javascript"), "json"),
|
||||
(("text/csv", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), "table"),
|
||||
)
|
||||
|
||||
|
||||
def artifact_kind(mime_type: str) -> str:
|
||||
"""Coarse wire-level artifact category: file | markdown | json | table."""
|
||||
lowered = (mime_type or "").lower()
|
||||
for prefixes, kind in _KIND_BY_MIME:
|
||||
if any(lowered.startswith(prefix) for prefix in prefixes):
|
||||
return kind
|
||||
return "file"
|
||||
|
||||
|
||||
class WorkflowEventEnvelope(BaseModel):
|
||||
"""Common envelope for every persisted / SSE workflow event."""
|
||||
|
||||
schema_version: Literal["1.0"] = Field(default=SCHEMA_VERSION, alias="schemaVersion")
|
||||
run_id: str = Field(alias="runId")
|
||||
workflow_id: str = Field(alias="workflowId")
|
||||
version_id: str = Field(alias="versionId")
|
||||
seq: int = 0
|
||||
event_type: WorkflowEventType = Field(alias="event")
|
||||
node_id: str | None = Field(default=None, alias="nodeId")
|
||||
node_run_id: str | None = Field(default=None, alias="nodeRunId")
|
||||
timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
data: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
def to_sse_dict(self) -> dict[str, Any]:
|
||||
"""JSON object for the SSE ``data:`` line (camelCase wire fields)."""
|
||||
return {
|
||||
"schemaVersion": self.schema_version,
|
||||
"runId": self.run_id,
|
||||
"workflowId": self.workflow_id,
|
||||
"versionId": self.version_id,
|
||||
"seq": self.seq,
|
||||
"event": self.event_type,
|
||||
"nodeId": self.node_id,
|
||||
"nodeRunId": self.node_run_id,
|
||||
"timestamp": self.timestamp.isoformat() if self.timestamp else None,
|
||||
"data": self.data,
|
||||
}
|
||||
|
||||
def to_sse_frame(self) -> str:
|
||||
"""Full SSE frame including ``id`` / ``event`` / ``data`` lines."""
|
||||
import json
|
||||
|
||||
payload = json.dumps(self.to_sse_dict(), ensure_ascii=False, separators=(",", ":"))
|
||||
return f"id: {self.seq}\nevent: {self.event_type}\ndata: {payload}\n\n"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class WorkflowEventSink(Protocol):
|
||||
"""Persist-then-publish sink used by the future executor."""
|
||||
|
||||
async def emit(
|
||||
self,
|
||||
event_type: WorkflowEventType,
|
||||
*,
|
||||
data: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> WorkflowEventEnvelope:
|
||||
"""Persist event (assign ``seq``), then fan out to live subscribers."""
|
||||
...
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ALL_WORKFLOW_EVENT_TYPES",
|
||||
"SCHEMA_VERSION",
|
||||
"TERMINAL_EVENT_TYPES",
|
||||
"WorkflowEventEnvelope",
|
||||
"WorkflowEventSink",
|
||||
"WorkflowEventType",
|
||||
"artifact_kind",
|
||||
]
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Safe expression / template evaluation for workflow node configs.
|
||||
|
||||
Two surfaces, both intentionally tiny:
|
||||
|
||||
* ``render_template("… {{ nodes.a.data.title }} …", state)`` — string
|
||||
interpolation over a whitelisted state tree.
|
||||
* ``evaluate_condition("nodes.a.data.score > 0.5", state)`` — boolean
|
||||
predicate over the same tree.
|
||||
|
||||
Neither uses :func:`eval`. The condition grammar is a single comparison or a
|
||||
chain joined by ``and`` / ``or``; operands are either a state path or a JSON
|
||||
literal. Anything else raises ``WORKFLOW_EXPRESSION_INVALID`` so a malformed
|
||||
graph fails loudly at publish time instead of silently at run time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
_TEMPLATE_RE = re.compile(r"\{\{\s*([^{}]+?)\s*\}\}")
|
||||
_ROOT_KEYS = ("inputs", "nodes", "run", "loop", "env")
|
||||
_PATH_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_\-]*|\[\d+\])*$")
|
||||
_COMPARATORS = ("==", "!=", ">=", "<=", ">", "<", " in ", " contains ")
|
||||
_MAX_RENDER_CHARS = 200_000
|
||||
|
||||
|
||||
def _invalid(expr: str, reason: str) -> WorkflowError:
|
||||
return WorkflowError(
|
||||
"WORKFLOW_EXPRESSION_INVALID",
|
||||
f"表达式不合法:{reason}",
|
||||
details={"expression": expr[:200]},
|
||||
)
|
||||
|
||||
|
||||
def resolve_path(path: str, state: dict[str, Any]) -> Any:
|
||||
"""Read ``a.b[0].c`` from the state tree. Missing segments yield ``None``."""
|
||||
expr = path.strip()
|
||||
if not _PATH_RE.match(expr):
|
||||
raise _invalid(expr, "路径格式不合法")
|
||||
root = expr.split(".", 1)[0].split("[", 1)[0]
|
||||
if root not in _ROOT_KEYS:
|
||||
raise _invalid(expr, f"根节点必须是 {'/'.join(_ROOT_KEYS)} 之一")
|
||||
current: Any = state
|
||||
for segment in re.findall(r"[A-Za-z_][A-Za-z0-9_\-]*|\[\d+\]", expr):
|
||||
if current is None:
|
||||
return None
|
||||
if segment.startswith("["):
|
||||
index = int(segment[1:-1])
|
||||
if not isinstance(current, (list, tuple)) or index >= len(current):
|
||||
return None
|
||||
current = current[index]
|
||||
elif isinstance(current, dict):
|
||||
current = current.get(segment)
|
||||
else:
|
||||
return None
|
||||
return current
|
||||
|
||||
|
||||
def _stringify(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
return json.dumps(value, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def render_template(template: str, state: dict[str, Any]) -> str:
|
||||
"""Replace every ``{{ path }}`` with its stringified state value."""
|
||||
if not template:
|
||||
return ""
|
||||
|
||||
def _sub(match: re.Match[str]) -> str:
|
||||
return _stringify(resolve_path(match.group(1), state))
|
||||
|
||||
rendered = _TEMPLATE_RE.sub(_sub, template)
|
||||
if len(rendered) > _MAX_RENDER_CHARS:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_LIMIT_EXCEEDED",
|
||||
"模板渲染结果超出长度上限",
|
||||
details={"limit": _MAX_RENDER_CHARS},
|
||||
)
|
||||
return rendered
|
||||
|
||||
|
||||
def render_value(value: Any, state: dict[str, Any]) -> Any:
|
||||
"""Recursively render templates inside dict / list / str structures.
|
||||
|
||||
A string that is exactly one ``{{ path }}`` keeps the resolved value's
|
||||
native type instead of being stringified.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
match = _TEMPLATE_RE.fullmatch(value.strip())
|
||||
if match:
|
||||
return resolve_path(match.group(1), state)
|
||||
return render_template(value, state)
|
||||
if isinstance(value, dict):
|
||||
return {k: render_value(v, state) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [render_value(v, state) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def _operand(token: str, state: dict[str, Any]) -> Any:
|
||||
text = token.strip()
|
||||
if not text:
|
||||
raise _invalid(token, "操作数为空")
|
||||
match = _TEMPLATE_RE.fullmatch(text)
|
||||
if match:
|
||||
return resolve_path(match.group(1), state)
|
||||
lowered = text.lower()
|
||||
if lowered in ("true", "false"):
|
||||
return lowered == "true"
|
||||
if lowered in ("null", "none"):
|
||||
return None
|
||||
if (text[0], text[-1]) in (('"', '"'), ("'", "'")) and len(text) >= 2:
|
||||
return text[1:-1]
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if _PATH_RE.match(text) and text.split(".", 1)[0].split("[", 1)[0] in _ROOT_KEYS:
|
||||
return resolve_path(text, state)
|
||||
return text
|
||||
|
||||
|
||||
def _truthy(value: Any) -> bool:
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() not in ("", "false", "0", "null", "none")
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _compare(left: Any, op: str, right: Any) -> bool:
|
||||
if op == "==":
|
||||
return left == right
|
||||
if op == "!=":
|
||||
return left != right
|
||||
if op == "in":
|
||||
try:
|
||||
return left in right # type: ignore[operator]
|
||||
except TypeError:
|
||||
return False
|
||||
if op == "contains":
|
||||
try:
|
||||
return right in left # type: ignore[operator]
|
||||
except TypeError:
|
||||
return False
|
||||
try:
|
||||
if op == ">":
|
||||
return left > right
|
||||
if op == "<":
|
||||
return left < right
|
||||
if op == ">=":
|
||||
return left >= right
|
||||
if op == "<=":
|
||||
return left <= right
|
||||
except TypeError:
|
||||
return False
|
||||
raise _invalid(op, f"不支持的比较运算符 {op}")
|
||||
|
||||
|
||||
def _split_top_level(expr: str, keyword: str) -> list[str] | None:
|
||||
"""Split on ``keyword`` outside quotes / braces; None when absent."""
|
||||
parts: list[str] = []
|
||||
depth = 0
|
||||
quote: str | None = None
|
||||
buffer: list[str] = []
|
||||
index = 0
|
||||
pattern = f" {keyword} "
|
||||
while index < len(expr):
|
||||
char = expr[index]
|
||||
if quote:
|
||||
buffer.append(char)
|
||||
if char == quote:
|
||||
quote = None
|
||||
index += 1
|
||||
continue
|
||||
if char in ('"', "'"):
|
||||
quote = char
|
||||
buffer.append(char)
|
||||
index += 1
|
||||
continue
|
||||
if char in "[{(":
|
||||
depth += 1
|
||||
elif char in "]})":
|
||||
depth -= 1
|
||||
if depth == 0 and expr[index : index + len(pattern)] == pattern:
|
||||
parts.append("".join(buffer))
|
||||
buffer = []
|
||||
index += len(pattern)
|
||||
continue
|
||||
buffer.append(char)
|
||||
index += 1
|
||||
if not parts:
|
||||
return None
|
||||
parts.append("".join(buffer))
|
||||
return parts
|
||||
|
||||
|
||||
def evaluate_condition(expression: str, state: dict[str, Any]) -> bool:
|
||||
"""Evaluate a boolean predicate without ``eval``."""
|
||||
expr = (expression or "").strip()
|
||||
if not expr:
|
||||
return False
|
||||
|
||||
or_parts = _split_top_level(expr, "or")
|
||||
if or_parts:
|
||||
return any(evaluate_condition(part, state) for part in or_parts)
|
||||
and_parts = _split_top_level(expr, "and")
|
||||
if and_parts:
|
||||
return all(evaluate_condition(part, state) for part in and_parts)
|
||||
|
||||
if expr.startswith("not "):
|
||||
return not evaluate_condition(expr[4:], state)
|
||||
if expr.startswith("(") and expr.endswith(")"):
|
||||
return evaluate_condition(expr[1:-1], state)
|
||||
|
||||
for comparator in _COMPARATORS:
|
||||
op = comparator.strip()
|
||||
index = expr.find(comparator)
|
||||
if index <= 0:
|
||||
continue
|
||||
left = _operand(expr[:index], state)
|
||||
right = _operand(expr[index + len(comparator) :], state)
|
||||
return _compare(left, op, right)
|
||||
|
||||
return _truthy(_operand(expr, state))
|
||||
|
||||
|
||||
_NODES_IDENT_RE = re.compile(r"(?:\{\{\s*)?nodes\.([A-Za-z_][A-Za-z0-9_\-]*)")
|
||||
|
||||
|
||||
def collect_referenced_nodes(value: Any) -> set[str]:
|
||||
"""Node ids referenced through ``nodes.<id>`` anywhere in a config tree.
|
||||
|
||||
Covers both ``{{ nodes.x.data }}`` templates and bare condition paths
|
||||
such as ``nodes.x.data.score > 0.5``.
|
||||
"""
|
||||
found: set[str] = set()
|
||||
if isinstance(value, str):
|
||||
found.update(_NODES_IDENT_RE.findall(value))
|
||||
elif isinstance(value, dict):
|
||||
for item in value.values():
|
||||
found |= collect_referenced_nodes(item)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
found |= collect_referenced_nodes(item)
|
||||
return found
|
||||
|
||||
|
||||
__all__ = [
|
||||
"collect_referenced_nodes",
|
||||
"evaluate_condition",
|
||||
"render_template",
|
||||
"render_value",
|
||||
"resolve_path",
|
||||
]
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
"""Node executors for the workflow runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.workflows.nodes.agents import (
|
||||
AgentNodeExecutor,
|
||||
SkillNodeExecutor,
|
||||
SubworkflowNodeExecutor,
|
||||
)
|
||||
from deerflow.workflows.nodes.base import ACTIVE_PORTS, NodeExecutor, NodeRegistry
|
||||
from deerflow.workflows.nodes.core import (
|
||||
ConditionNodeExecutor,
|
||||
LoopNodeExecutor,
|
||||
MergeNodeExecutor,
|
||||
OutputNodeExecutor,
|
||||
StartNodeExecutor,
|
||||
TransformNodeExecutor,
|
||||
)
|
||||
from deerflow.workflows.nodes.human import HumanInputNodeExecutor
|
||||
from deerflow.workflows.nodes.integrations import (
|
||||
CodeNodeExecutor,
|
||||
HttpNodeExecutor,
|
||||
SqlReadNodeExecutor,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ACTIVE_PORTS",
|
||||
"AgentNodeExecutor",
|
||||
"CodeNodeExecutor",
|
||||
"ConditionNodeExecutor",
|
||||
"HttpNodeExecutor",
|
||||
"HumanInputNodeExecutor",
|
||||
"LoopNodeExecutor",
|
||||
"MergeNodeExecutor",
|
||||
"NodeExecutor",
|
||||
"NodeRegistry",
|
||||
"OutputNodeExecutor",
|
||||
"SkillNodeExecutor",
|
||||
"SqlReadNodeExecutor",
|
||||
"StartNodeExecutor",
|
||||
"SubworkflowNodeExecutor",
|
||||
"TransformNodeExecutor",
|
||||
"build_default_registry",
|
||||
]
|
||||
|
||||
|
||||
def build_default_registry(config: WorkflowConfig | None = None) -> NodeRegistry:
|
||||
"""Registry with every executable node type wired up."""
|
||||
cfg = config or WorkflowConfig()
|
||||
registry = NodeRegistry()
|
||||
registry.register("start", StartNodeExecutor())
|
||||
registry.register("output", OutputNodeExecutor())
|
||||
registry.register("transform", TransformNodeExecutor())
|
||||
registry.register("condition", ConditionNodeExecutor())
|
||||
registry.register("merge", MergeNodeExecutor())
|
||||
registry.register("loop", LoopNodeExecutor())
|
||||
registry.register("human_input", HumanInputNodeExecutor())
|
||||
registry.register("http", HttpNodeExecutor(cfg))
|
||||
registry.register("sql_read", SqlReadNodeExecutor(cfg))
|
||||
registry.register("code", CodeNodeExecutor(cfg))
|
||||
registry.register("agent", AgentNodeExecutor())
|
||||
registry.register("skill", SkillNodeExecutor())
|
||||
registry.register("subworkflow", SubworkflowNodeExecutor())
|
||||
return registry
|
||||
@@ -0,0 +1,361 @@
|
||||
"""LLM-backed nodes: ``agent``, ``skill``, ``subworkflow``.
|
||||
|
||||
These are the only nodes that call the DeerFlow LangGraph lead agent. The agent
|
||||
invocation itself is injected (``deps.run_agent``) so the harness keeps its
|
||||
import firewall; this module owns prompt rendering, streaming fan-out, JSON
|
||||
response coercion, and error typing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.expressions import render_template, render_value
|
||||
from deerflow.workflows.nodes.base import require_dep
|
||||
from deerflow.workflows.runtime.context import RunContext
|
||||
from deerflow.workflows.schemas import ArtifactRef, NodeResult, WorkflowNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_JSON_BLOCK = re.compile(r"```(?:json)?\s*(.+?)\s*```", re.DOTALL)
|
||||
_DELTA_FLUSH_SECONDS = 0.05
|
||||
_DELTA_FLUSH_CHARS = 1024
|
||||
|
||||
|
||||
class _DeltaCoalescer:
|
||||
"""Merge model tokens into 50ms / 1KB event payloads (§9.5)."""
|
||||
|
||||
def __init__(self, ctx: RunContext, node_id: str) -> None:
|
||||
self._ctx = ctx
|
||||
self._node_id = node_id
|
||||
self._parts: list[str] = []
|
||||
self._chars = 0
|
||||
self._task: asyncio.Task | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def push(self, chunk: str) -> None:
|
||||
if not chunk:
|
||||
return
|
||||
async with self._lock:
|
||||
self._parts.append(chunk)
|
||||
self._chars += len(chunk)
|
||||
if self._chars >= _DELTA_FLUSH_CHARS:
|
||||
await self._flush_unlocked()
|
||||
return
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._delayed())
|
||||
|
||||
async def _delayed(self) -> None:
|
||||
await asyncio.sleep(_DELTA_FLUSH_SECONDS)
|
||||
async with self._lock:
|
||||
await self._flush_unlocked()
|
||||
|
||||
async def _flush_unlocked(self) -> None:
|
||||
if not self._parts:
|
||||
return
|
||||
text = "".join(self._parts)
|
||||
self._parts.clear()
|
||||
self._chars = 0
|
||||
await self._ctx.emit_event(
|
||||
"node.output.delta",
|
||||
data={"channel": "answer", "delta": text, "text": text},
|
||||
node_id=self._node_id,
|
||||
)
|
||||
|
||||
async def flush(self) -> None:
|
||||
task = self._task
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
async with self._lock:
|
||||
await self._flush_unlocked()
|
||||
|
||||
|
||||
def _extract_json(text: str) -> Any:
|
||||
candidate = text.strip()
|
||||
match = _JSON_BLOCK.search(candidate)
|
||||
if match:
|
||||
candidate = match.group(1).strip()
|
||||
else:
|
||||
start = min((i for i in (candidate.find("{"), candidate.find("[")) if i >= 0), default=-1)
|
||||
if start > 0:
|
||||
candidate = candidate[start:]
|
||||
try:
|
||||
return json.loads(candidate)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
_SAFE_ARG_STRING_CHARS = 160
|
||||
_SAFE_ARG_MAX_KEYS = 12
|
||||
|
||||
|
||||
def _safe_tool_args(args: Any) -> dict[str, Any]:
|
||||
"""Minimal safe projection of tool-call args for the event log.
|
||||
|
||||
Scalars only (strings truncated), no nested containers, bounded key count —
|
||||
enough for the canvas to label the step without leaking full payloads such
|
||||
as entire documents, prompts or credentials that a tool may carry."""
|
||||
if not isinstance(args, dict):
|
||||
return {}
|
||||
out: dict[str, Any] = {}
|
||||
for key, value in list(args.items())[:_SAFE_ARG_MAX_KEYS]:
|
||||
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
value = value[:_SAFE_ARG_STRING_CHARS]
|
||||
out[str(key)[:64]] = value
|
||||
return out
|
||||
|
||||
|
||||
class _AgentInvocation:
|
||||
"""Shared plumbing for ``agent`` and ``skill`` (agent_skill mode)."""
|
||||
|
||||
async def _invoke(
|
||||
self,
|
||||
node: WorkflowNode,
|
||||
ctx: RunContext,
|
||||
*,
|
||||
agent_id: str,
|
||||
prompt: str,
|
||||
skill_names: list[str],
|
||||
response_mode: str,
|
||||
response_schema: dict[str, Any],
|
||||
thread_mode: str,
|
||||
failure_code: str,
|
||||
) -> NodeResult:
|
||||
run_agent = require_dep(ctx.deps.run_agent, node=node, what="智能体执行")
|
||||
coalescer = _DeltaCoalescer(ctx, node.id)
|
||||
|
||||
async def on_delta(chunk: str) -> None:
|
||||
await coalescer.push(chunk)
|
||||
|
||||
async def on_tool(event: dict[str, Any]) -> None:
|
||||
phase = str(event.get("phase") or "started")
|
||||
name = str(event.get("name") or "")
|
||||
call_id = str(event.get("id") or "")
|
||||
if phase == "started":
|
||||
await ctx.emit_event(
|
||||
"node.tool.started",
|
||||
data={
|
||||
"name": name,
|
||||
"toolCallId": call_id,
|
||||
"title": str(event.get("title") or name),
|
||||
"args": _safe_tool_args(event.get("args")),
|
||||
},
|
||||
node_id=node.id,
|
||||
)
|
||||
return
|
||||
ok = event.get("ok")
|
||||
await ctx.emit_event(
|
||||
"node.tool.finished",
|
||||
data={
|
||||
"name": name,
|
||||
"toolCallId": call_id,
|
||||
"status": "succeeded" if ok in (True, None) else "failed",
|
||||
"summary": event.get("summary"),
|
||||
},
|
||||
node_id=node.id,
|
||||
)
|
||||
|
||||
try:
|
||||
try:
|
||||
outcome = await run_agent(
|
||||
run_id=ctx.run_id,
|
||||
node_id=node.id,
|
||||
owner_id=ctx.owner_id,
|
||||
agent_id=agent_id,
|
||||
prompt=prompt,
|
||||
skill_names=skill_names,
|
||||
thread_mode=thread_mode,
|
||||
cancel=ctx.cancel,
|
||||
on_delta=on_delta,
|
||||
on_tool=on_tool,
|
||||
)
|
||||
except WorkflowError as exc:
|
||||
exc.node_id = exc.node_id or node.id
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - agent failures are expected
|
||||
logger.warning("workflow agent node %s failed", node.id, exc_info=True)
|
||||
raise WorkflowError(
|
||||
failure_code,
|
||||
"智能体执行失败",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"reason": type(exc).__name__},
|
||||
) from exc
|
||||
finally:
|
||||
await coalescer.flush()
|
||||
|
||||
text = str(outcome.get("text") or "")
|
||||
data: dict[str, Any] = {"text": text}
|
||||
if response_mode == "json":
|
||||
payload = outcome.get("json")
|
||||
if payload is None:
|
||||
payload = _extract_json(text)
|
||||
if payload is None:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
"智能体未返回可解析的 JSON",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"preview": text[:500]},
|
||||
)
|
||||
required = response_schema.get("required") if isinstance(response_schema, dict) else None
|
||||
if isinstance(required, list) and isinstance(payload, dict):
|
||||
missing = [str(k) for k in required if payload.get(str(k)) in (None, "")]
|
||||
if missing:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
f"智能体返回缺少字段:{'、'.join(missing)}",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"missing": missing},
|
||||
)
|
||||
data["json"] = payload
|
||||
|
||||
artifacts: list[ArtifactRef] = []
|
||||
for raw in outcome.get("artifacts") or []:
|
||||
if not isinstance(raw, dict) or not raw.get("name"):
|
||||
continue
|
||||
ref = await ctx.register_artifact(
|
||||
node,
|
||||
name=str(raw.get("name")),
|
||||
mime_type=str(raw.get("mime_type") or raw.get("mimeType") or "application/octet-stream"),
|
||||
path=str(raw.get("path") or ""),
|
||||
size_bytes=raw.get("size_bytes") or raw.get("sizeBytes"),
|
||||
preview=raw.get("preview"),
|
||||
)
|
||||
if ref is not None:
|
||||
artifacts.append(ref)
|
||||
|
||||
return NodeResult(
|
||||
data=data,
|
||||
messages=[m for m in (outcome.get("messages") or []) if isinstance(m, dict)],
|
||||
artifacts=artifacts,
|
||||
metadata={
|
||||
"agentId": agent_id,
|
||||
"threadId": outcome.get("thread_id"),
|
||||
"usage": outcome.get("usage") or {},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class AgentNodeExecutor(_AgentInvocation):
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
agent_id = str(node.config.get("agentId") or node.config.get("agent_id") or "")
|
||||
if not agent_id:
|
||||
raise WorkflowError("WORKFLOW_AGENT_FAILED", "智能体节点未选择智能体", node_id=node.id)
|
||||
template = str(node.config.get("promptTemplate") or node.config.get("prompt_template") or "")
|
||||
prompt = render_template(template, ctx.state()).strip()
|
||||
if not prompt:
|
||||
raise WorkflowError("WORKFLOW_AGENT_FAILED", "智能体节点提示词为空", node_id=node.id)
|
||||
schema = node.config.get("responseSchema") or node.config.get("response_schema") or {}
|
||||
return await self._invoke(
|
||||
node,
|
||||
ctx,
|
||||
agent_id=agent_id,
|
||||
prompt=prompt,
|
||||
skill_names=[str(s) for s in (node.config.get("skillNames") or node.config.get("skill_names") or [])],
|
||||
response_mode=str(node.config.get("responseMode") or node.config.get("response_mode") or "text"),
|
||||
response_schema=schema if isinstance(schema, dict) else {},
|
||||
thread_mode=str(node.config.get("threadMode") or node.config.get("thread_mode") or "isolated_per_node"),
|
||||
failure_code="WORKFLOW_AGENT_FAILED",
|
||||
)
|
||||
|
||||
|
||||
class SkillNodeExecutor(_AgentInvocation):
|
||||
"""``agent_skill`` runs the agent with a restricted skill set.
|
||||
``callable_skill`` (direct script invocation) is not executable yet."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
mode = str(node.config.get("mode") or "agent_skill")
|
||||
if mode != "agent_skill":
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SKILL_FAILED",
|
||||
"callable_skill 模式暂未开放执行,请改用 agent_skill",
|
||||
node_id=node.id,
|
||||
details={"mode": mode},
|
||||
)
|
||||
agent_id = str(node.config.get("agentId") or node.config.get("agent_id") or "")
|
||||
skill_names = [str(s) for s in (node.config.get("skillNames") or node.config.get("skill_names") or [])]
|
||||
if not agent_id or not skill_names:
|
||||
raise WorkflowError("WORKFLOW_SKILL_FAILED", "技能节点需要同时指定智能体与技能", node_id=node.id)
|
||||
template = str(node.config.get("promptTemplate") or node.config.get("prompt_template") or "")
|
||||
prompt = render_template(template, ctx.state()).strip() or f"请使用技能 {skill_names[0]} 完成任务。"
|
||||
return await self._invoke(
|
||||
node,
|
||||
ctx,
|
||||
agent_id=agent_id,
|
||||
prompt=prompt,
|
||||
skill_names=skill_names,
|
||||
response_mode="text",
|
||||
response_schema={},
|
||||
thread_mode="isolated_per_node",
|
||||
failure_code="WORKFLOW_SKILL_FAILED",
|
||||
)
|
||||
|
||||
|
||||
class SubworkflowNodeExecutor:
|
||||
"""Runs a published child workflow synchronously and folds in its output."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
run_subworkflow = require_dep(ctx.deps.run_subworkflow, node=node, what="子工作流执行")
|
||||
workflow_id = str(node.config.get("workflowId") or node.config.get("workflow_id") or "")
|
||||
version_id = str(node.config.get("versionId") or node.config.get("version_id") or "")
|
||||
if not workflow_id or not version_id:
|
||||
raise WorkflowError("WORKFLOW_SUBWORKFLOW_FAILED", "子工作流节点缺少 workflowId/versionId", node_id=node.id)
|
||||
mapping = node.config.get("inputMapping") or node.config.get("input_mapping") or {}
|
||||
state = ctx.state()
|
||||
inputs = {str(key): render_value(template, state) for key, template in mapping.items()} if isinstance(mapping, dict) else {}
|
||||
await ctx.emit_event(
|
||||
"node.progress",
|
||||
data={"phase": "subworkflow.start", "workflowId": workflow_id, "versionId": version_id},
|
||||
node_id=node.id,
|
||||
)
|
||||
try:
|
||||
outcome = await run_subworkflow(
|
||||
parent_run_id=ctx.run_id,
|
||||
node_id=node.id,
|
||||
owner_id=ctx.owner_id,
|
||||
workflow_id=workflow_id,
|
||||
version_id=version_id,
|
||||
inputs=inputs,
|
||||
cancel=ctx.cancel,
|
||||
)
|
||||
except WorkflowError as exc:
|
||||
exc.node_id = exc.node_id or node.id
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
"子工作流执行失败",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"reason": type(exc).__name__},
|
||||
) from exc
|
||||
|
||||
status = str(outcome.get("status") or "completed")
|
||||
if status != "completed":
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
f"子工作流未成功完成({status})",
|
||||
node_id=node.id,
|
||||
details={"childRunId": outcome.get("run_id"), "status": status},
|
||||
)
|
||||
output = outcome.get("output")
|
||||
return NodeResult(
|
||||
data=output if isinstance(output, dict) else {"value": output},
|
||||
metadata={"childRunId": outcome.get("run_id"), "workflowId": workflow_id},
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["AgentNodeExecutor", "SkillNodeExecutor", "SubworkflowNodeExecutor"]
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Node executor protocol and registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.ports import ACTIVE_PORTS
|
||||
from deerflow.workflows.runtime.context import RunContext
|
||||
from deerflow.workflows.schemas import NodeResult, NodeType, WorkflowNode
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class NodeExecutor(Protocol):
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult: ...
|
||||
|
||||
|
||||
class NodeRegistry:
|
||||
"""Maps node type → executor. Unknown types fail with a typed error."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._executors: dict[str, NodeExecutor] = {}
|
||||
|
||||
def register(self, node_type: NodeType | str, executor: NodeExecutor) -> None:
|
||||
self._executors[str(node_type)] = executor
|
||||
|
||||
def get(self, node_type: str) -> NodeExecutor:
|
||||
executor = self._executors.get(node_type)
|
||||
if executor is None:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
f"节点类型暂不支持执行:{node_type}",
|
||||
details={"nodeType": node_type},
|
||||
)
|
||||
return executor
|
||||
|
||||
def supported(self) -> list[str]:
|
||||
return sorted(self._executors)
|
||||
|
||||
|
||||
def require_dep(value: Any, *, node: WorkflowNode, what: str) -> Any:
|
||||
if value is None:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
f"运行环境缺少{what}能力,无法执行该节点",
|
||||
node_id=node.id,
|
||||
details={"capability": what},
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
__all__ = ["ACTIVE_PORTS", "NodeExecutor", "NodeRegistry", "require_dep"]
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Deterministic control / data nodes: start, output, transform, condition, merge, loop."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.expressions import evaluate_condition, render_template, render_value
|
||||
from deerflow.workflows.nodes.base import ACTIVE_PORTS
|
||||
from deerflow.workflows.runtime.context import RunContext
|
||||
from deerflow.workflows.schema_validation import schema_issues
|
||||
from deerflow.workflows.schemas import NodeResult, WorkflowNode
|
||||
|
||||
DEFAULT_PORT = "else"
|
||||
LOOP_CONTINUE_PORT = "continue"
|
||||
LOOP_DONE_PORT = "done"
|
||||
|
||||
|
||||
def _required_keys(schema: dict[str, Any]) -> list[str]:
|
||||
required = schema.get("required")
|
||||
if isinstance(required, list):
|
||||
return [str(k) for k in required]
|
||||
return []
|
||||
|
||||
|
||||
class StartNodeExecutor:
|
||||
"""Validates run inputs against the graph input schema and exposes them."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
schema = ctx.graph.input_schema or {}
|
||||
missing = [key for key in _required_keys(schema) if ctx.inputs.get(key) in (None, "")]
|
||||
if missing:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_INPUT_INVALID",
|
||||
f"缺少必填输入:{'、'.join(missing)}",
|
||||
node_id=node.id,
|
||||
details={"missing": missing},
|
||||
)
|
||||
# Full JSON Schema pass (types / enums / nested objects) beyond the
|
||||
# friendly missing-key check above.
|
||||
issues = schema_issues(ctx.inputs, schema)
|
||||
if issues:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_INPUT_INVALID",
|
||||
"输入不符合 schema:" + ";".join(i["message"] for i in issues[:3]),
|
||||
node_id=node.id,
|
||||
details={"errors": issues},
|
||||
)
|
||||
declared = node.config.get("fields")
|
||||
data = dict(ctx.inputs)
|
||||
if isinstance(declared, list) and declared:
|
||||
names = {str(f.get("name")) for f in declared if isinstance(f, dict) and f.get("name")}
|
||||
data = {k: v for k, v in data.items() if k in names} or data
|
||||
return NodeResult(data=data)
|
||||
|
||||
|
||||
class OutputNodeExecutor:
|
||||
"""Projects the final run output. Empty mapping = merge all upstream data."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
mapping = node.config.get("mapping") or {}
|
||||
state = ctx.state()
|
||||
if isinstance(mapping, dict) and mapping:
|
||||
data = {str(key): render_value(template, state) for key, template in mapping.items()}
|
||||
else:
|
||||
data = {}
|
||||
for result in ctx.node_results.values():
|
||||
payload = result.get("data")
|
||||
if isinstance(payload, dict):
|
||||
data.update(payload)
|
||||
schema = ctx.graph.output_schema or {}
|
||||
missing = [key for key in _required_keys(schema) if data.get(key) in (None, "")]
|
||||
if missing:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
f"输出缺少必填字段:{'、'.join(missing)}",
|
||||
node_id=node.id,
|
||||
details={"missing": missing},
|
||||
)
|
||||
issues = schema_issues(data, schema)
|
||||
if issues:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
"输出不符合 schema:" + ";".join(i["message"] for i in issues[:3]),
|
||||
node_id=node.id,
|
||||
details={"errors": issues},
|
||||
)
|
||||
return NodeResult(data=data, metadata={"isRunOutput": True})
|
||||
|
||||
|
||||
class TransformNodeExecutor:
|
||||
"""Pure data reshaping. Supported ops: set / pick / omit / json_parse / join."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
state = ctx.state()
|
||||
data: dict[str, Any] = {}
|
||||
template = node.config.get("template")
|
||||
if isinstance(template, str) and template:
|
||||
data["text"] = render_template(template, state)
|
||||
|
||||
for raw in node.config.get("operations") or []:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
op = str(raw.get("op") or "set")
|
||||
target = str(raw.get("target") or raw.get("name") or "value")
|
||||
try:
|
||||
if op == "set":
|
||||
data[target] = render_value(raw.get("value"), state)
|
||||
elif op == "pick":
|
||||
source = render_value(raw.get("source"), state)
|
||||
keys = [str(k) for k in (raw.get("keys") or [])]
|
||||
data[target] = {k: source.get(k) for k in keys} if isinstance(source, dict) else None
|
||||
elif op == "omit":
|
||||
source = render_value(raw.get("source"), state)
|
||||
keys = {str(k) for k in (raw.get("keys") or [])}
|
||||
data[target] = {k: v for k, v in source.items() if k not in keys} if isinstance(source, dict) else None
|
||||
elif op == "json_parse":
|
||||
text = render_value(raw.get("source"), state)
|
||||
data[target] = json.loads(text) if isinstance(text, str) and text.strip() else None
|
||||
elif op == "join":
|
||||
items = render_value(raw.get("source"), state)
|
||||
separator = str(raw.get("separator") or "\n")
|
||||
data[target] = separator.join(str(i) for i in items) if isinstance(items, list) else ""
|
||||
else:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_TRANSFORM_FAILED",
|
||||
f"不支持的转换操作:{op}",
|
||||
node_id=node.id,
|
||||
details={"op": op},
|
||||
)
|
||||
except WorkflowError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - surface as typed node failure
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_TRANSFORM_FAILED",
|
||||
f"转换操作 {op} 执行失败",
|
||||
node_id=node.id,
|
||||
details={"op": op, "target": target, "reason": type(exc).__name__},
|
||||
) from exc
|
||||
return NodeResult(data=data)
|
||||
|
||||
|
||||
class ConditionNodeExecutor:
|
||||
"""Picks the first matching branch; activates only that outgoing port."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
state = ctx.state()
|
||||
branches = node.config.get("branches") or []
|
||||
if not isinstance(branches, list) or not branches:
|
||||
raise WorkflowError("WORKFLOW_CONDITION_FAILED", "条件节点未配置分支", node_id=node.id)
|
||||
evaluated: list[dict[str, Any]] = []
|
||||
chosen: str | None = None
|
||||
for raw in branches:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
name = str(raw.get("name") or "")
|
||||
expression = str(raw.get("expression") or "")
|
||||
matched = evaluate_condition(expression, state)
|
||||
evaluated.append({"name": name, "matched": matched})
|
||||
if matched and chosen is None:
|
||||
chosen = name
|
||||
if chosen is None:
|
||||
chosen = str(node.config.get("defaultBranch") or node.config.get("default_branch") or DEFAULT_PORT)
|
||||
return NodeResult(
|
||||
data={"branch": chosen, "evaluated": evaluated},
|
||||
metadata={ACTIVE_PORTS: [chosen]},
|
||||
)
|
||||
|
||||
|
||||
class MergeNodeExecutor:
|
||||
"""Collects upstream results per strategy (all / first_success / any_success)."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
strategy = str(node.config.get("strategy") or "all")
|
||||
sources = [str(s) for s in (node.config.get("sources") or [])]
|
||||
if not sources:
|
||||
sources = [edge.source for edge in ctx.graph.edges if edge.target == node.id]
|
||||
collected: dict[str, Any] = {}
|
||||
succeeded: list[str] = []
|
||||
for source in sources:
|
||||
result = ctx.node_results.get(source)
|
||||
if result is None:
|
||||
continue
|
||||
collected[source] = result.get("data")
|
||||
succeeded.append(source)
|
||||
if strategy in ("first_success", "any_success") and not succeeded:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_MERGE_FAILED",
|
||||
"合并节点没有任何成功的上游结果",
|
||||
node_id=node.id,
|
||||
details={"strategy": strategy, "sources": sources},
|
||||
)
|
||||
if strategy == "first_success":
|
||||
first = succeeded[0]
|
||||
return NodeResult(data={"selected": first, "value": collected[first]})
|
||||
return NodeResult(data={"branches": collected, "succeeded": succeeded})
|
||||
|
||||
|
||||
class LoopNodeExecutor:
|
||||
"""Iteration gate. Activates ``continue`` (back-edge) or ``done``.
|
||||
|
||||
Iteration accounting lives in ``ctx.loop_state`` so the cap is enforced even
|
||||
across a resume, and the engine invalidates body node results before the
|
||||
next pass.
|
||||
"""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
iteration = int(ctx.loop_state.get(node.id, 0))
|
||||
configured = int(node.config.get("maxIterations") or node.config.get("max_iterations") or 1)
|
||||
cap = min(configured, ctx.graph.settings.max_loop_iterations)
|
||||
expression = str(node.config.get("continueWhen") or node.config.get("continue_when") or "")
|
||||
should_continue = bool(expression) and evaluate_condition(expression, ctx.state())
|
||||
next_iteration = iteration + 1
|
||||
if should_continue and next_iteration >= cap:
|
||||
should_continue = False
|
||||
port = LOOP_CONTINUE_PORT if should_continue else LOOP_DONE_PORT
|
||||
if should_continue:
|
||||
ctx.loop_state[node.id] = next_iteration
|
||||
return NodeResult(
|
||||
data={
|
||||
"iteration": iteration,
|
||||
"nextIteration": next_iteration if should_continue else iteration,
|
||||
"maxIterations": cap,
|
||||
"continue": should_continue,
|
||||
},
|
||||
metadata={ACTIVE_PORTS: [port]},
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ConditionNodeExecutor",
|
||||
"LoopNodeExecutor",
|
||||
"MergeNodeExecutor",
|
||||
"OutputNodeExecutor",
|
||||
"StartNodeExecutor",
|
||||
"TransformNodeExecutor",
|
||||
]
|
||||
@@ -0,0 +1,77 @@
|
||||
"""``human_input`` node: suspends the run and validates the resumed payload."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.expressions import render_template
|
||||
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
|
||||
from deerflow.workflows.schemas import NodeResult, WorkflowNode
|
||||
|
||||
DEFAULT_ACTIONS = ("submit",)
|
||||
|
||||
|
||||
def _form_schema(node: WorkflowNode) -> dict[str, Any]:
|
||||
schema = node.config.get("formSchema") or node.config.get("form_schema") or {}
|
||||
return schema if isinstance(schema, dict) else {}
|
||||
|
||||
|
||||
def _actions(node: WorkflowNode) -> list[str]:
|
||||
raw = node.config.get("actions")
|
||||
actions = [str(a) for a in raw] if isinstance(raw, list) and raw else list(DEFAULT_ACTIONS)
|
||||
return actions
|
||||
|
||||
|
||||
class HumanInputNodeExecutor:
|
||||
"""First pass raises :class:`WorkflowPaused`; the resume pass consumes the
|
||||
payload the API stored on the context and validates it against the form."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
payload = ctx.resume_payload
|
||||
if not payload or str(payload.get("nodeId") or "") != node.id:
|
||||
raise WorkflowPaused(
|
||||
node.id,
|
||||
{
|
||||
"nodeId": node.id,
|
||||
"name": node.name,
|
||||
"prompt": render_template(str(node.config.get("prompt") or ""), ctx.state()),
|
||||
"formSchema": _form_schema(node),
|
||||
"actions": _actions(node),
|
||||
},
|
||||
)
|
||||
|
||||
action = str(payload.get("action") or "submit")
|
||||
allowed = _actions(node)
|
||||
if action not in allowed:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_HUMAN_INPUT_INVALID",
|
||||
f"不支持的操作:{action}",
|
||||
node_id=node.id,
|
||||
details={"allowed": allowed},
|
||||
)
|
||||
values = payload.get("values")
|
||||
values = values if isinstance(values, dict) else {}
|
||||
schema = _form_schema(node)
|
||||
required = schema.get("required")
|
||||
if isinstance(required, list) and action != "cancel":
|
||||
missing = [str(key) for key in required if values.get(str(key)) in (None, "")]
|
||||
if missing:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_HUMAN_INPUT_INVALID",
|
||||
f"缺少必填项:{'、'.join(missing)}",
|
||||
node_id=node.id,
|
||||
details={"missing": missing},
|
||||
)
|
||||
if action == "cancel":
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_CANCELLED",
|
||||
"用户在人工节点选择终止运行",
|
||||
node_id=node.id,
|
||||
)
|
||||
# One-shot: a later human node in the same run must pause again.
|
||||
ctx.resume_payload = None
|
||||
return NodeResult(data={"action": action, "values": values})
|
||||
|
||||
|
||||
__all__ = ["HumanInputNodeExecutor"]
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
"""External-effect nodes: ``http``, ``sql_read``, ``code``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.expressions import render_template, render_value
|
||||
from deerflow.workflows.runtime.code_runner import run_python_snippet
|
||||
from deerflow.workflows.runtime.context import RunContext
|
||||
from deerflow.workflows.runtime.sql_runner import execute_read_query
|
||||
from deerflow.workflows.schemas import NodeResult, WorkflowNode
|
||||
from deerflow.workflows.security.http_policy import enforce_http_target, enforce_redirect
|
||||
from deerflow.workflows.security.sql_policy import (
|
||||
assert_parameters_bound,
|
||||
assert_read_only,
|
||||
assert_tables_allowed,
|
||||
enforce_limit,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Response headers safe to echo back into the event log.
|
||||
_SAFE_RESPONSE_HEADERS = ("content-type", "content-length", "date", "etag", "location")
|
||||
_REDACTED_REQUEST_HEADERS = ("authorization", "cookie", "x-api-key", "proxy-authorization")
|
||||
|
||||
|
||||
class HttpNodeExecutor:
|
||||
def __init__(self, config: WorkflowConfig) -> None:
|
||||
self._config = config
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
policy = self._config.http
|
||||
state = ctx.state()
|
||||
url = render_template(str(node.config.get("url") or ""), state).strip()
|
||||
if not url:
|
||||
raise WorkflowError("WORKFLOW_HTTP_FAILED", "HTTP 节点缺少 URL", node_id=node.id)
|
||||
method = str(node.config.get("method") or "GET").upper()
|
||||
headers = {str(k): str(v) for k, v in (render_value(node.config.get("headers") or {}, state) or {}).items()}
|
||||
query = render_value(node.config.get("query") or {}, state) or {}
|
||||
body = render_value(node.config.get("body"), state)
|
||||
|
||||
credential_ref = node.config.get("credentialRef") or node.config.get("credential_ref")
|
||||
if credential_ref:
|
||||
headers.update(await self._credential_headers(str(credential_ref), node, ctx))
|
||||
|
||||
try:
|
||||
target = await enforce_http_target(url, policy)
|
||||
except WorkflowError as exc:
|
||||
exc.node_id = exc.node_id or node.id
|
||||
raise
|
||||
|
||||
timeout = min(
|
||||
int(node.config.get("timeoutSeconds") or node.config.get("timeout_seconds") or policy.timeout_seconds),
|
||||
policy.timeout_seconds,
|
||||
)
|
||||
max_bytes = min(
|
||||
int(node.config.get("maxResponseBytes") or node.config.get("max_response_bytes") or policy.max_response_bytes),
|
||||
policy.max_response_bytes,
|
||||
)
|
||||
allow_redirects = bool(node.config.get("allowRedirects") or node.config.get("allow_redirects"))
|
||||
max_hops = policy.max_redirects if allow_redirects else 0
|
||||
|
||||
await ctx.emit_event(
|
||||
"node.progress",
|
||||
data={
|
||||
"phase": "http.request",
|
||||
"method": method,
|
||||
"host": target["host"],
|
||||
"headers": [k for k in headers if k.lower() not in _REDACTED_REQUEST_HEADERS],
|
||||
},
|
||||
node_id=node.id,
|
||||
)
|
||||
|
||||
import httpx
|
||||
|
||||
current_url = url
|
||||
hops = 0
|
||||
raw = b""
|
||||
truncated = False
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
|
||||
while True:
|
||||
ctx.cancel.raise_if_cancelled(node.id)
|
||||
try:
|
||||
request = client.build_request(
|
||||
method,
|
||||
current_url,
|
||||
headers=headers,
|
||||
params=query or None,
|
||||
json=body if isinstance(body, (dict, list)) else None,
|
||||
content=body if isinstance(body, (str, bytes)) else None,
|
||||
)
|
||||
response = await client.send(request, stream=True)
|
||||
except httpx.HTTPError as exc:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_HTTP_FAILED",
|
||||
"HTTP 请求失败",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"reason": type(exc).__name__},
|
||||
) from exc
|
||||
|
||||
if response.status_code in (301, 302, 303, 307, 308) and hops < max_hops:
|
||||
location = response.headers.get("location") or ""
|
||||
await response.aclose()
|
||||
if not location:
|
||||
break
|
||||
current_url = await enforce_redirect(location, current_url, policy)
|
||||
hops += 1
|
||||
continue
|
||||
|
||||
chunks: list[bytes] = []
|
||||
size = 0
|
||||
try:
|
||||
async for chunk in response.aiter_bytes():
|
||||
size += len(chunk)
|
||||
if size > max_bytes:
|
||||
truncated = True
|
||||
break
|
||||
chunks.append(chunk)
|
||||
finally:
|
||||
await response.aclose()
|
||||
raw = b"".join(chunks)
|
||||
break
|
||||
|
||||
text_body = raw.decode("utf-8", errors="replace")
|
||||
parsed: Any = None
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if "json" in content_type.lower():
|
||||
try:
|
||||
parsed = json.loads(text_body)
|
||||
except json.JSONDecodeError:
|
||||
parsed = None
|
||||
|
||||
data = {
|
||||
"status": response.status_code,
|
||||
"ok": 200 <= response.status_code < 300,
|
||||
"headers": {k: v for k, v in response.headers.items() if k.lower() in _SAFE_RESPONSE_HEADERS},
|
||||
"json": parsed,
|
||||
"text": None if parsed is not None else text_body,
|
||||
"truncated": truncated,
|
||||
"redirects": hops,
|
||||
}
|
||||
if not data["ok"]:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_HTTP_FAILED",
|
||||
f"HTTP 请求返回 {response.status_code}",
|
||||
retryable=response.status_code >= 500 or response.status_code == 429,
|
||||
node_id=node.id,
|
||||
details={"status": response.status_code, "preview": text_body[:500]},
|
||||
)
|
||||
return NodeResult(data=data, metadata={"host": target["host"]})
|
||||
|
||||
async def _credential_headers(self, ref: str, node: WorkflowNode, ctx: RunContext) -> dict[str, str]:
|
||||
if ctx.deps.resolve_credential is None:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"运行环境未配置凭据解析能力",
|
||||
node_id=node.id,
|
||||
)
|
||||
credential = await ctx.deps.resolve_credential(ref)
|
||||
if not credential:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"引用的凭据不存在或无权访问",
|
||||
node_id=node.id,
|
||||
details={"credentialRef": ref},
|
||||
)
|
||||
headers = credential.get("headers")
|
||||
return {str(k): str(v) for k, v in headers.items()} if isinstance(headers, dict) else {}
|
||||
|
||||
|
||||
class SqlReadNodeExecutor:
|
||||
def __init__(self, config: WorkflowConfig) -> None:
|
||||
self._config = config
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
policy = self._config.sql
|
||||
if not policy.enabled:
|
||||
raise WorkflowError("WORKFLOW_SQL_POLICY_DENIED", "SQL 节点已被系统禁用", node_id=node.id)
|
||||
data_source_id = str(node.config.get("dataSourceId") or node.config.get("data_source_id") or "")
|
||||
if not data_source_id:
|
||||
raise WorkflowError("WORKFLOW_SQL_FAILED", "SQL 节点未选择数据源", node_id=node.id)
|
||||
if ctx.deps.resolve_data_source is None:
|
||||
raise WorkflowError("WORKFLOW_RESOURCE_MISSING", "运行环境未配置数据源解析能力", node_id=node.id)
|
||||
source = await ctx.deps.resolve_data_source(data_source_id)
|
||||
if not source or not source.get("dsn"):
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"数据源不存在或无权访问",
|
||||
node_id=node.id,
|
||||
details={"dataSourceId": data_source_id},
|
||||
)
|
||||
|
||||
statement = assert_read_only(str(node.config.get("statement") or ""))
|
||||
parameters = render_value(node.config.get("parameters") or {}, ctx.state()) or {}
|
||||
if not isinstance(parameters, dict):
|
||||
raise WorkflowError("WORKFLOW_SQL_FAILED", "SQL 参数必须是对象", node_id=node.id)
|
||||
assert_parameters_bound(statement, parameters)
|
||||
assert_tables_allowed(statement, list(source.get("allowed_tables") or []))
|
||||
max_rows = min(
|
||||
int(node.config.get("maxRows") or node.config.get("max_rows") or policy.max_rows),
|
||||
policy.max_rows,
|
||||
int(source.get("max_rows") or policy.max_rows),
|
||||
)
|
||||
statement = enforce_limit(statement, max_rows)
|
||||
|
||||
await ctx.emit_event(
|
||||
"node.progress",
|
||||
data={"phase": "sql.query", "dataSourceId": data_source_id, "maxRows": max_rows},
|
||||
node_id=node.id,
|
||||
)
|
||||
try:
|
||||
result = await execute_read_query(
|
||||
dsn=str(source["dsn"]),
|
||||
statement=statement,
|
||||
parameters={str(k): v for k, v in parameters.items()},
|
||||
max_rows=max_rows,
|
||||
timeout_seconds=policy.statement_timeout_seconds,
|
||||
max_cell_chars=policy.max_cell_chars,
|
||||
)
|
||||
except WorkflowError as exc:
|
||||
exc.node_id = exc.node_id or node.id
|
||||
raise
|
||||
return NodeResult(data=result)
|
||||
|
||||
|
||||
class CodeNodeExecutor:
|
||||
def __init__(self, config: WorkflowConfig) -> None:
|
||||
self._config = config
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
policy = self._config.code
|
||||
if not policy.enabled:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_CODE_SANDBOX_FAILED",
|
||||
"代码节点未启用(需管理员在 config.yaml 中开启 workflows.code.enabled)",
|
||||
node_id=node.id,
|
||||
)
|
||||
source = str(node.config.get("source") or "")
|
||||
if not source.strip():
|
||||
raise WorkflowError("WORKFLOW_CODE_SANDBOX_FAILED", "代码节点没有源代码", node_id=node.id)
|
||||
if len(source) > policy.max_source_chars:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_LIMIT_EXCEEDED",
|
||||
f"代码长度超过上限 {policy.max_source_chars}",
|
||||
node_id=node.id,
|
||||
)
|
||||
inputs = render_value(node.config.get("inputs") or {}, ctx.state()) or {}
|
||||
timeout = min(
|
||||
int(node.config.get("timeoutSeconds") or node.config.get("timeout_seconds") or policy.timeout_seconds),
|
||||
policy.timeout_seconds,
|
||||
)
|
||||
allow_network = bool((node.config.get("allowNetwork") or node.config.get("allow_network")) and policy.allow_network)
|
||||
try:
|
||||
outcome = await run_python_snippet(
|
||||
source=source,
|
||||
inputs=inputs if isinstance(inputs, dict) else {"value": inputs},
|
||||
timeout_seconds=timeout,
|
||||
max_output_chars=policy.max_output_chars,
|
||||
memory_limit_mb=policy.memory_limit_mb,
|
||||
allow_network=allow_network,
|
||||
)
|
||||
except WorkflowError as exc:
|
||||
exc.node_id = exc.node_id or node.id
|
||||
raise
|
||||
return NodeResult(
|
||||
data=outcome["result"],
|
||||
metadata={"stdout": outcome["stdout"][-4000:], "stderr": outcome["stderr"][-2000:]},
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["CodeNodeExecutor", "HttpNodeExecutor", "SqlReadNodeExecutor"]
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Leaf constants shared by the scheduler and node executors.
|
||||
|
||||
Kept dependency-free so the engine and the node package can both import it
|
||||
without forming a cycle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Metadata key a node sets to restrict which outgoing ports the engine activates.
|
||||
ACTIVE_PORTS = "activePorts"
|
||||
|
||||
# Reserved port names.
|
||||
LOOP_CONTINUE_PORT = "continue"
|
||||
LOOP_DONE_PORT = "done"
|
||||
CONDITION_DEFAULT_PORT = "else"
|
||||
|
||||
__all__ = ["ACTIVE_PORTS", "CONDITION_DEFAULT_PORT", "LOOP_CONTINUE_PORT", "LOOP_DONE_PORT"]
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
"""Workflow execution runtime (scheduler, context, event sink)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.workflows.runtime.context import (
|
||||
CancelToken,
|
||||
RunContext,
|
||||
WorkflowPaused,
|
||||
WorkflowRuntimeDeps,
|
||||
)
|
||||
from deerflow.workflows.runtime.engine import EngineResult, WorkflowEngine
|
||||
from deerflow.workflows.runtime.sink import PersistingWorkflowEventSink, envelope_from_row
|
||||
|
||||
__all__ = [
|
||||
"CancelToken",
|
||||
"EngineResult",
|
||||
"PersistingWorkflowEventSink",
|
||||
"RunContext",
|
||||
"WorkflowEngine",
|
||||
"WorkflowPaused",
|
||||
"WorkflowRuntimeDeps",
|
||||
"envelope_from_row",
|
||||
]
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
"""Constrained Python execution for the ``code`` node.
|
||||
|
||||
The snippet runs in a **separate interpreter process** with:
|
||||
|
||||
* a scrubbed environment (no API keys, tokens, or DSNs inherited),
|
||||
* a fresh temporary working directory that is deleted afterwards,
|
||||
* an address-space cap and no core dumps (POSIX ``setrlimit``),
|
||||
* sockets disabled unless the node explicitly allows network,
|
||||
* a hard wall-clock timeout with process-group kill,
|
||||
* bounded stdout/stderr capture.
|
||||
|
||||
This is process isolation, not a container: a determined snippet can still read
|
||||
world-readable files on the host. That is why ``workflows.code.enabled``
|
||||
defaults to ``false`` — turn it on only where the container sandbox or an OS
|
||||
level jail already confines the backend process.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_RUNNER_TEMPLATE = """
|
||||
import json, os, sys
|
||||
|
||||
_ALLOW_NETWORK = {allow_network}
|
||||
if not _ALLOW_NETWORK:
|
||||
import socket
|
||||
|
||||
def _blocked(*_args, **_kwargs):
|
||||
raise PermissionError("network access is disabled for this code node")
|
||||
|
||||
socket.socket = _blocked
|
||||
socket.create_connection = _blocked
|
||||
socket.getaddrinfo = _blocked
|
||||
|
||||
try:
|
||||
import resource
|
||||
|
||||
_limit = {memory_bytes}
|
||||
resource.setrlimit(resource.RLIMIT_AS, (_limit, _limit))
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
inputs = json.loads(os.environ.get("WORKFLOW_NODE_INPUT") or "{{}}")
|
||||
output = {{}}
|
||||
_globals = {{"__name__": "__workflow_code__", "inputs": inputs, "output": output}}
|
||||
|
||||
with open({source_path!r}, "r", encoding="utf-8") as handle:
|
||||
_source = handle.read()
|
||||
|
||||
exec(compile(_source, "<workflow-code>", "exec"), _globals)
|
||||
|
||||
_result = _globals.get("output")
|
||||
if not isinstance(_result, dict):
|
||||
_result = {{"value": _result}}
|
||||
sys.stdout.write("\\n__WORKFLOW_RESULT__" + json.dumps(_result, ensure_ascii=False, default=str))
|
||||
"""
|
||||
|
||||
|
||||
def _safe_env() -> dict[str, str]:
|
||||
env = {
|
||||
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
||||
"LANG": os.environ.get("LANG", "C.UTF-8"),
|
||||
"PYTHONIOENCODING": "utf-8",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"HOME": "",
|
||||
}
|
||||
if sys.platform == "win32":
|
||||
for key in ("SYSTEMROOT", "COMSPEC", "TEMP", "TMP"):
|
||||
if os.environ.get(key):
|
||||
env[key] = os.environ[key]
|
||||
return env
|
||||
|
||||
|
||||
async def run_python_snippet(
|
||||
*,
|
||||
source: str,
|
||||
inputs: dict[str, Any],
|
||||
timeout_seconds: int,
|
||||
max_output_chars: int,
|
||||
memory_limit_mb: int,
|
||||
allow_network: bool,
|
||||
) -> dict[str, Any]:
|
||||
workdir = Path(tempfile.mkdtemp(prefix="wf-code-"))
|
||||
try:
|
||||
source_path = workdir / "snippet.py"
|
||||
source_path.write_text(source, encoding="utf-8")
|
||||
runner_path = workdir / "runner.py"
|
||||
runner_path.write_text(
|
||||
_RUNNER_TEMPLATE.format(
|
||||
allow_network=bool(allow_network),
|
||||
memory_bytes=max(64, int(memory_limit_mb)) * 1024 * 1024,
|
||||
source_path=str(source_path),
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
env = _safe_env()
|
||||
env["WORKFLOW_NODE_INPUT"] = json.dumps(inputs, ensure_ascii=False, default=str)
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-I",
|
||||
"-S",
|
||||
str(runner_path),
|
||||
cwd=str(workdir),
|
||||
env=env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
start_new_session=sys.platform != "win32",
|
||||
)
|
||||
try:
|
||||
stdout_bytes, stderr_bytes = await asyncio.wait_for(process.communicate(), timeout=timeout_seconds)
|
||||
except TimeoutError as exc:
|
||||
_terminate(process)
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"代码节点执行超时({timeout_seconds}s)",
|
||||
details={"timeoutSeconds": timeout_seconds},
|
||||
) from exc
|
||||
|
||||
stdout = stdout_bytes.decode("utf-8", errors="replace")
|
||||
stderr = stderr_bytes.decode("utf-8", errors="replace")[:max_output_chars]
|
||||
if process.returncode != 0:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_CODE_SANDBOX_FAILED",
|
||||
"代码节点执行失败",
|
||||
details={"exitCode": process.returncode, "stderr": stderr[-2000:]},
|
||||
)
|
||||
|
||||
marker = "__WORKFLOW_RESULT__"
|
||||
index = stdout.rfind(marker)
|
||||
result: dict[str, Any] = {}
|
||||
logs = stdout
|
||||
if index >= 0:
|
||||
logs = stdout[:index]
|
||||
try:
|
||||
parsed = json.loads(stdout[index + len(marker) :])
|
||||
result = parsed if isinstance(parsed, dict) else {"value": parsed}
|
||||
except json.JSONDecodeError:
|
||||
result = {}
|
||||
return {
|
||||
"result": result,
|
||||
"stdout": logs[:max_output_chars],
|
||||
"stderr": stderr,
|
||||
}
|
||||
finally:
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
|
||||
def _terminate(process: asyncio.subprocess.Process) -> None:
|
||||
try:
|
||||
if sys.platform != "win32":
|
||||
os.killpg(os.getpgid(process.pid), 9)
|
||||
else:
|
||||
process.kill()
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
logger.debug("code node process already gone", exc_info=True)
|
||||
|
||||
|
||||
__all__ = ["run_python_snippet"]
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
"""Run-scoped execution context shared by the engine and node executors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.events import WorkflowEventType
|
||||
from deerflow.workflows.schemas import ArtifactRef, WorkflowGraph, WorkflowNode
|
||||
|
||||
EmitFn = Callable[..., Awaitable[Any]]
|
||||
|
||||
|
||||
def _public_artifact_payload(ref: ArtifactRef) -> dict[str, Any]:
|
||||
"""Wire payload for ``artifact.created``.
|
||||
|
||||
The sandbox/host ``path`` stays internal: clients address artifacts by id
|
||||
through the run-scoped content endpoints, never by filesystem path."""
|
||||
from deerflow.workflows.events import artifact_kind
|
||||
|
||||
return {
|
||||
"artifactId": ref.artifact_id,
|
||||
"name": ref.name,
|
||||
"kind": artifact_kind(ref.mime_type),
|
||||
"mimeType": ref.mime_type,
|
||||
"sizeBytes": ref.size_bytes,
|
||||
"preview": ref.preview,
|
||||
}
|
||||
|
||||
|
||||
class WorkflowPaused(Exception): # noqa: N818 - control-flow signal, not an error
|
||||
"""Raised by ``human_input`` to suspend the run until a resume arrives."""
|
||||
|
||||
def __init__(self, node_id: str, pending_input: dict[str, Any]) -> None:
|
||||
super().__init__(f"workflow paused at {node_id}")
|
||||
self.node_id = node_id
|
||||
self.pending_input = pending_input
|
||||
|
||||
|
||||
class CancelToken:
|
||||
"""Cooperative cancellation shared between the executor and node code."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._event = asyncio.Event()
|
||||
self._reason = "cancelled"
|
||||
|
||||
def cancel(self, reason: str = "cancelled") -> None:
|
||||
self._reason = reason
|
||||
self._event.set()
|
||||
|
||||
@property
|
||||
def cancelled(self) -> bool:
|
||||
return self._event.is_set()
|
||||
|
||||
async def wait(self) -> None:
|
||||
await self._event.wait()
|
||||
|
||||
def raise_if_cancelled(self, node_id: str | None = None) -> None:
|
||||
if self._event.is_set():
|
||||
raise WorkflowError("WORKFLOW_CANCELLED", "运行已被取消", node_id=node_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowRuntimeDeps:
|
||||
"""Injected capabilities. The harness never imports the app layer, so every
|
||||
integration point (agents, HTTP egress, SQL, sandbox, subworkflows) arrives
|
||||
as an async callable supplied by the caller. ``None`` means "not available",
|
||||
which node executors turn into a typed ``WORKFLOW_RESOURCE_MISSING``."""
|
||||
|
||||
run_agent: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_http: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_sql: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_code: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_subworkflow: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
save_artifact: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
resolve_credential: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
|
||||
resolve_data_source: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunContext:
|
||||
"""Everything a node executor is allowed to see."""
|
||||
|
||||
run_id: str
|
||||
workflow_id: str
|
||||
version_id: str
|
||||
owner_id: str
|
||||
graph: WorkflowGraph
|
||||
inputs: dict[str, Any]
|
||||
deps: WorkflowRuntimeDeps
|
||||
cancel: CancelToken
|
||||
emit: EmitFn
|
||||
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
loop_state: dict[str, int] = field(default_factory=dict)
|
||||
resume_payload: dict[str, Any] | None = None
|
||||
node_run_ids: dict[str, str] = field(default_factory=dict)
|
||||
current_node_id: str | None = None
|
||||
env: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def state(self) -> dict[str, Any]:
|
||||
"""Whitelisted tree visible to expressions. Credentials never appear."""
|
||||
return {
|
||||
"inputs": self.inputs,
|
||||
"nodes": self.node_results,
|
||||
"run": {"id": self.run_id, "workflowId": self.workflow_id, "versionId": self.version_id},
|
||||
"loop": dict(self.loop_state),
|
||||
"env": self.env,
|
||||
}
|
||||
|
||||
async def emit_event(
|
||||
self,
|
||||
event_type: WorkflowEventType,
|
||||
*,
|
||||
data: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
) -> None:
|
||||
node = node_id or self.current_node_id
|
||||
await self.emit(
|
||||
event_type,
|
||||
data=data or {},
|
||||
node_id=node,
|
||||
node_run_id=self.node_run_ids.get(node) if node else None,
|
||||
)
|
||||
|
||||
async def register_artifact(
|
||||
self,
|
||||
node: WorkflowNode,
|
||||
*,
|
||||
name: str,
|
||||
mime_type: str = "application/octet-stream",
|
||||
path: str = "",
|
||||
size_bytes: int | None = None,
|
||||
preview: str | None = None,
|
||||
) -> ArtifactRef | None:
|
||||
"""Persist an artifact reference and emit ``artifact.created``."""
|
||||
if self.deps.save_artifact is None:
|
||||
return None
|
||||
row = await self.deps.save_artifact(
|
||||
run_id=self.run_id,
|
||||
node_id=node.id,
|
||||
name=name,
|
||||
mime_type=mime_type,
|
||||
path=path,
|
||||
size_bytes=size_bytes,
|
||||
preview=preview,
|
||||
)
|
||||
ref = ArtifactRef(
|
||||
artifactId=str(row.get("id") or ""),
|
||||
name=str(row.get("name") or name),
|
||||
mimeType=str(row.get("mime_type") or mime_type),
|
||||
path=str(row.get("path") or path),
|
||||
sizeBytes=row.get("size_bytes"),
|
||||
preview=row.get("preview"),
|
||||
)
|
||||
await self.emit_event("artifact.created", data=_public_artifact_payload(ref), node_id=node.id)
|
||||
return ref
|
||||
|
||||
|
||||
__all__ = ["CancelToken", "RunContext", "WorkflowPaused", "WorkflowRuntimeDeps"]
|
||||
+531
@@ -0,0 +1,531 @@
|
||||
"""Persisted DAG scheduler for workflow runs.
|
||||
|
||||
Scheduling model
|
||||
----------------
|
||||
Every edge carries a state: ``pending`` → ``active`` (its source chose this
|
||||
port) or ``pruned`` (the source chose a different port). A node is *ready* when
|
||||
none of its non-back incoming edges is ``pending`` and at least one is
|
||||
``active``; a node whose incoming edges all resolved to ``pruned`` is *skipped*
|
||||
and prunes its own outgoing edges in turn. That single rule gives us fan-out
|
||||
parallelism, condition branching, and merge joins without a topological
|
||||
pre-pass, and it terminates because each superstep resolves at least one edge.
|
||||
|
||||
Loops are the one cycle we allow: the back edge (``loop`` node, port
|
||||
``continue``) is excluded from readiness gating, and activating it clears the
|
||||
body's results so the next iteration re-runs them. Every re-entry is capped by
|
||||
``settings.max_loop_iterations``.
|
||||
|
||||
Durability: node results are written to the run store as they complete, so a
|
||||
resumed or re-leased run replays only what is missing. ``human_input`` raises
|
||||
:class:`WorkflowPaused`, which the caller turns into ``awaiting_input``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Protocol
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.ports import ACTIVE_PORTS, LOOP_CONTINUE_PORT
|
||||
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
|
||||
from deerflow.workflows.schemas import NodeResult, WorkflowEdge, WorkflowGraph, WorkflowNode
|
||||
|
||||
|
||||
class ExecutorRegistry(Protocol):
|
||||
"""Structural view of ``deerflow.workflows.nodes.NodeRegistry``.
|
||||
|
||||
Declared here rather than imported so the scheduler has no dependency on
|
||||
the node package (which depends on the runtime context).
|
||||
"""
|
||||
|
||||
def get(self, node_type: str) -> Any: ...
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PENDING = "pending"
|
||||
ACTIVE = "active"
|
||||
PRUNED = "pruned"
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineResult:
|
||||
output: dict[str, Any] = field(default_factory=dict)
|
||||
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
loop_state: dict[str, int] = field(default_factory=dict)
|
||||
steps: int = 0
|
||||
|
||||
|
||||
def _is_back_edge(edge: WorkflowEdge, nodes: dict[str, WorkflowNode]) -> bool:
|
||||
source = nodes.get(edge.source)
|
||||
return bool(source and source.type == "loop" and (edge.source_port or "") == LOOP_CONTINUE_PORT)
|
||||
|
||||
|
||||
def _node_started_event_data(node: WorkflowNode, node_attempt: int) -> dict[str, Any]:
|
||||
"""Safe presentation metadata for workflow clients.
|
||||
|
||||
This stays intentionally small: UI clients need stable agent identity and
|
||||
an optional explicit parallel group, but never prompt templates, tools,
|
||||
credentials, or the full node config. A missing group is valid; clients
|
||||
may still use their backwards-compatible name-based presentation.
|
||||
"""
|
||||
|
||||
data: dict[str, Any] = {
|
||||
"nodeType": node.type,
|
||||
"nodeKind": node.type,
|
||||
"name": node.name,
|
||||
"attempt": node_attempt,
|
||||
}
|
||||
group_id = node.config.get("parallelGroupId") or node.config.get("parallel_group_id")
|
||||
if isinstance(group_id, str) and group_id.strip():
|
||||
data["parallelGroupId"] = group_id.strip()
|
||||
round_index = node.config.get("roundIndex")
|
||||
if round_index is None:
|
||||
round_index = node.config.get("round_index")
|
||||
if isinstance(round_index, int) and round_index >= 0:
|
||||
data["roundIndex"] = round_index
|
||||
|
||||
if node.type == "agent":
|
||||
agent_id = node.config.get("agentId") or node.config.get("agent_id")
|
||||
if isinstance(agent_id, str) and agent_id.strip():
|
||||
agent: dict[str, str] = {
|
||||
"agentId": agent_id.strip(),
|
||||
"name": str(
|
||||
node.config.get("agentName")
|
||||
or node.config.get("agent_name")
|
||||
or node.name
|
||||
or agent_id
|
||||
),
|
||||
}
|
||||
avatar_url = node.config.get("agentAvatarUrl") or node.config.get("agent_avatar_url")
|
||||
if isinstance(avatar_url, str) and avatar_url.strip():
|
||||
agent["avatarUrl"] = avatar_url.strip()
|
||||
data["agent"] = agent
|
||||
return data
|
||||
|
||||
|
||||
class WorkflowEngine:
|
||||
def __init__(self, registry: ExecutorRegistry, *, run_store: WorkflowRunStore | None = None) -> None:
|
||||
self._registry = registry
|
||||
self._store = run_store
|
||||
|
||||
async def run(
|
||||
self,
|
||||
ctx: RunContext,
|
||||
*,
|
||||
preloaded: dict[str, dict[str, Any]] | None = None,
|
||||
loop_state: dict[str, int] | None = None,
|
||||
attempt: int = 1,
|
||||
) -> EngineResult:
|
||||
graph: WorkflowGraph = ctx.graph
|
||||
nodes = graph.node_map()
|
||||
outgoing: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
|
||||
incoming: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
|
||||
for edge in graph.edges:
|
||||
outgoing[edge.source].append(edge)
|
||||
incoming[edge.target].append(edge)
|
||||
|
||||
edge_state: dict[str, str] = {edge.id: PENDING for edge in graph.edges}
|
||||
done: set[str] = set()
|
||||
skipped: set[str] = set()
|
||||
|
||||
ctx.loop_state.update(loop_state or {})
|
||||
if preloaded:
|
||||
ctx.node_results.update(preloaded)
|
||||
done |= set(preloaded)
|
||||
# Re-derive edge states from the ports each replayed node selected.
|
||||
for node_id in list(done):
|
||||
self._apply_ports(
|
||||
node_id,
|
||||
ctx.node_results.get(node_id, {}).get("metadata") or {},
|
||||
outgoing,
|
||||
edge_state,
|
||||
)
|
||||
|
||||
semaphore = asyncio.Semaphore(max(1, graph.settings.max_parallelism))
|
||||
steps = 0
|
||||
max_steps = max(1, graph.settings.max_steps)
|
||||
|
||||
while True:
|
||||
ctx.cancel.raise_if_cancelled()
|
||||
self._propagate_skips(nodes, incoming, outgoing, edge_state, done, skipped)
|
||||
ready = [node_id for node_id in nodes if node_id not in done and node_id not in skipped and self._readiness(node_id, incoming, edge_state, nodes) == "ready"]
|
||||
if not ready:
|
||||
break
|
||||
|
||||
steps += len(ready)
|
||||
if steps > max_steps:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_MAX_STEPS",
|
||||
f"运行步数超过上限 {max_steps}",
|
||||
details={"maxSteps": max_steps},
|
||||
)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(self._run_one(nodes[node_id], ctx, semaphore, attempt) for node_id in ready),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
paused: WorkflowPaused | None = None
|
||||
failure: BaseException | None = None
|
||||
for node_id, outcome in zip(ready, results, strict=True):
|
||||
if isinstance(outcome, WorkflowPaused):
|
||||
paused = paused or outcome
|
||||
continue
|
||||
if isinstance(outcome, BaseException):
|
||||
failure = failure or outcome
|
||||
continue
|
||||
ctx.node_results[node_id] = outcome
|
||||
done.add(node_id)
|
||||
metadata = outcome.get("metadata") or {}
|
||||
self._apply_ports(node_id, metadata, outgoing, edge_state)
|
||||
if nodes[node_id].type == "loop":
|
||||
await self._maybe_reset_loop_body(nodes[node_id], ctx, nodes, outgoing, edge_state, done, skipped)
|
||||
if failure is not None:
|
||||
raise failure
|
||||
if paused is not None:
|
||||
raise paused
|
||||
|
||||
return EngineResult(
|
||||
output=self._collect_output(nodes, ctx),
|
||||
node_results=dict(ctx.node_results),
|
||||
loop_state=dict(ctx.loop_state),
|
||||
steps=steps,
|
||||
)
|
||||
|
||||
# ── scheduling helpers ──────────────────────────────────────────────
|
||||
def _readiness(
|
||||
self,
|
||||
node_id: str,
|
||||
incoming: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
nodes: dict[str, WorkflowNode],
|
||||
) -> str:
|
||||
edges = incoming.get(node_id) or []
|
||||
if not edges:
|
||||
return "ready"
|
||||
back = [e for e in edges if _is_back_edge(e, nodes)]
|
||||
if any(edge_state[e.id] == ACTIVE for e in back):
|
||||
return "ready"
|
||||
normal = [e for e in edges if not _is_back_edge(e, nodes)]
|
||||
if not normal:
|
||||
return "blocked"
|
||||
if any(edge_state[e.id] == PENDING for e in normal):
|
||||
return "blocked"
|
||||
if any(edge_state[e.id] == ACTIVE for e in normal):
|
||||
return "ready"
|
||||
return "skip"
|
||||
|
||||
def _apply_ports(
|
||||
self,
|
||||
node_id: str,
|
||||
metadata: dict[str, Any],
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
) -> None:
|
||||
active_ports = metadata.get(ACTIVE_PORTS)
|
||||
allowed = {str(p) for p in active_ports} if isinstance(active_ports, list) else None
|
||||
for edge in outgoing.get(node_id) or []:
|
||||
port = edge.source_port or ""
|
||||
if allowed is None or not port or port in allowed:
|
||||
edge_state[edge.id] = ACTIVE
|
||||
else:
|
||||
edge_state[edge.id] = PRUNED
|
||||
|
||||
def _propagate_skips(
|
||||
self,
|
||||
nodes: dict[str, WorkflowNode],
|
||||
incoming: dict[str, list[WorkflowEdge]],
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
done: set[str],
|
||||
skipped: set[str],
|
||||
) -> None:
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for node_id in nodes:
|
||||
if node_id in done or node_id in skipped:
|
||||
continue
|
||||
if self._readiness(node_id, incoming, edge_state, nodes) != "skip":
|
||||
continue
|
||||
skipped.add(node_id)
|
||||
for edge in outgoing.get(node_id) or []:
|
||||
edge_state[edge.id] = PRUNED
|
||||
changed = True
|
||||
|
||||
async def _maybe_reset_loop_body(
|
||||
self,
|
||||
loop_node: WorkflowNode,
|
||||
ctx: RunContext,
|
||||
nodes: dict[str, WorkflowNode],
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
done: set[str],
|
||||
skipped: set[str],
|
||||
) -> None:
|
||||
back_edges = [e for e in outgoing.get(loop_node.id) or [] if _is_back_edge(e, nodes) and edge_state[e.id] == ACTIVE]
|
||||
if not back_edges:
|
||||
return
|
||||
body_entry = str(loop_node.config.get("bodyEntry") or loop_node.config.get("body_entry") or "")
|
||||
if not body_entry or body_entry not in nodes:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_LOOP_FAILED",
|
||||
"循环节点的 bodyEntry 不存在",
|
||||
node_id=loop_node.id,
|
||||
details={"bodyEntry": body_entry},
|
||||
)
|
||||
body = self._loop_body(body_entry, loop_node.id, outgoing, nodes)
|
||||
for node_id in body:
|
||||
done.discard(node_id)
|
||||
skipped.discard(node_id)
|
||||
ctx.node_results.pop(node_id, None)
|
||||
for edge in outgoing.get(node_id) or []:
|
||||
if not _is_back_edge(edge, nodes):
|
||||
edge_state[edge.id] = PENDING
|
||||
# The back edge stays ACTIVE so the body entry is reachable again; the
|
||||
# loop node prunes it itself once it selects the ``done`` port.
|
||||
if self._store is not None and body:
|
||||
try:
|
||||
await self._store.invalidate_node_runs(ctx.run_id, sorted(body))
|
||||
except Exception: # noqa: BLE001 - bookkeeping must not kill the run
|
||||
logger.warning("failed to invalidate node runs for loop reset", exc_info=True)
|
||||
await ctx.emit_event(
|
||||
"node.progress",
|
||||
data={
|
||||
"phase": "loop.iteration",
|
||||
"iteration": ctx.loop_state.get(loop_node.id, 0),
|
||||
"bodyEntry": body_entry,
|
||||
},
|
||||
node_id=loop_node.id,
|
||||
)
|
||||
|
||||
def _loop_body(
|
||||
self,
|
||||
body_entry: str,
|
||||
loop_id: str,
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
nodes: dict[str, WorkflowNode],
|
||||
) -> set[str]:
|
||||
"""Nodes reachable from ``body_entry`` up to (and including) the loop node."""
|
||||
body: set[str] = set()
|
||||
stack = [body_entry]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
if current in body:
|
||||
continue
|
||||
body.add(current)
|
||||
if current == loop_id:
|
||||
continue
|
||||
for edge in outgoing.get(current) or []:
|
||||
if not _is_back_edge(edge, nodes):
|
||||
stack.append(edge.target)
|
||||
return body
|
||||
|
||||
# ── node execution ──────────────────────────────────────────────────
|
||||
async def _run_one(
|
||||
self,
|
||||
node: WorkflowNode,
|
||||
ctx: RunContext,
|
||||
semaphore: asyncio.Semaphore,
|
||||
attempt: int,
|
||||
) -> dict[str, Any]:
|
||||
async with semaphore:
|
||||
ctx.cancel.raise_if_cancelled(node.id)
|
||||
retry_cfg = node.config.get("retry") if isinstance(node.config.get("retry"), dict) else {}
|
||||
max_attempts = max(1, min(5, int(retry_cfg.get("maxAttempts") or 1)))
|
||||
backoff = max(0.0, float(retry_cfg.get("backoffSeconds") or 0.5))
|
||||
on_error = str(node.config.get("onError") or "fail")
|
||||
timeout = self._node_timeout(node, ctx)
|
||||
|
||||
last_error: WorkflowError | None = None
|
||||
for node_attempt in range(1, max_attempts + 1):
|
||||
node_run = await self._begin_node_run(ctx, node, attempt, node_attempt)
|
||||
started = time.monotonic()
|
||||
await ctx.emit_event(
|
||||
"node.started",
|
||||
data=_node_started_event_data(node, node_attempt),
|
||||
node_id=node.id,
|
||||
)
|
||||
try:
|
||||
executor = self._registry.get(node.type)
|
||||
result: NodeResult = await asyncio.wait_for(executor.execute(node, ctx), timeout=timeout)
|
||||
except WorkflowPaused:
|
||||
await self._finish_node_run(ctx, node_run, status="queued", output=None)
|
||||
raise
|
||||
except TimeoutError as exc:
|
||||
last_error = WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"节点执行超时({timeout}s)",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"timeoutSeconds": timeout},
|
||||
)
|
||||
await self._fail_node_run(ctx, node_run, last_error, started)
|
||||
if node_attempt < max_attempts:
|
||||
await asyncio.sleep(backoff * node_attempt)
|
||||
continue
|
||||
if on_error == "continue":
|
||||
return self._error_result(last_error)
|
||||
raise last_error from exc
|
||||
except WorkflowError as exc:
|
||||
last_error = exc
|
||||
exc.node_id = exc.node_id or node.id
|
||||
await self._fail_node_run(ctx, node_run, exc, started)
|
||||
if exc.retryable and node_attempt < max_attempts:
|
||||
await asyncio.sleep(backoff * node_attempt)
|
||||
continue
|
||||
if on_error == "continue":
|
||||
return self._error_result(exc)
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - unexpected node bug
|
||||
logger.exception("workflow node %s crashed", node.id)
|
||||
last_error = WorkflowError(
|
||||
"WORKFLOW_INTERNAL",
|
||||
"节点执行发生内部错误",
|
||||
node_id=node.id,
|
||||
details={"reason": type(exc).__name__},
|
||||
)
|
||||
await self._fail_node_run(ctx, node_run, last_error, started)
|
||||
if on_error == "continue":
|
||||
return self._error_result(last_error)
|
||||
raise last_error from exc
|
||||
|
||||
payload = result.model_dump(by_alias=False)
|
||||
duration = int((time.monotonic() - started) * 1000)
|
||||
await self._finish_node_run(ctx, node_run, status="completed", output=payload, duration_ms=duration)
|
||||
await ctx.emit_event(
|
||||
"node.completed",
|
||||
data={
|
||||
"nodeType": node.type,
|
||||
"durationMs": duration,
|
||||
"data": payload.get("data"),
|
||||
"warnings": payload.get("warnings") or [],
|
||||
},
|
||||
node_id=node.id,
|
||||
)
|
||||
return payload
|
||||
|
||||
raise last_error or WorkflowError("WORKFLOW_INTERNAL", "节点执行失败", node_id=node.id)
|
||||
|
||||
def _node_timeout(self, node: WorkflowNode, ctx: RunContext) -> int:
|
||||
configured = node.config.get("timeoutSeconds") or node.config.get("timeout_seconds")
|
||||
graph_limit = ctx.graph.settings.node_timeout_seconds
|
||||
try:
|
||||
value = int(configured) if configured else graph_limit
|
||||
except (TypeError, ValueError):
|
||||
value = graph_limit
|
||||
return max(1, min(value, graph_limit))
|
||||
|
||||
def _error_result(self, error: WorkflowError) -> dict[str, Any]:
|
||||
return {
|
||||
"data": {},
|
||||
"messages": [],
|
||||
"artifacts": [],
|
||||
"metadata": {"failed": True, "error": error.to_body().model_dump(by_alias=True)},
|
||||
"warnings": [error.message],
|
||||
}
|
||||
|
||||
async def _begin_node_run(self, ctx: RunContext, node: WorkflowNode, run_attempt: int, node_attempt: int) -> dict[str, Any] | None:
|
||||
if self._store is None:
|
||||
return None
|
||||
try:
|
||||
row = await self._store.record_node_run(
|
||||
{
|
||||
"run_id": ctx.run_id,
|
||||
"node_id": node.id,
|
||||
"node_type": node.type,
|
||||
"status": "running",
|
||||
"attempt": node_attempt,
|
||||
"iteration": int(ctx.loop_state.get(node.id, 0)),
|
||||
"started_at": datetime.now(UTC),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("failed to record node run start for %s", node.id, exc_info=True)
|
||||
return None
|
||||
ctx.node_run_ids[node.id] = str(row.get("id") or "")
|
||||
return row
|
||||
|
||||
async def _finish_node_run(
|
||||
self,
|
||||
ctx: RunContext,
|
||||
node_run: dict[str, Any] | None,
|
||||
*,
|
||||
status: str,
|
||||
output: dict[str, Any] | None,
|
||||
duration_ms: int | None = None,
|
||||
) -> None:
|
||||
if self._store is None or node_run is None:
|
||||
return
|
||||
try:
|
||||
await self._store.record_node_run(
|
||||
{
|
||||
"run_id": ctx.run_id,
|
||||
"node_id": node_run["node_id"],
|
||||
"node_type": node_run["node_type"],
|
||||
"attempt": node_run["attempt"],
|
||||
"iteration": node_run["iteration"],
|
||||
"status": status,
|
||||
"output": output,
|
||||
"duration_ms": duration_ms,
|
||||
"finished_at": datetime.now(UTC),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("failed to record node run finish", exc_info=True)
|
||||
|
||||
async def _fail_node_run(
|
||||
self,
|
||||
ctx: RunContext,
|
||||
node_run: dict[str, Any] | None,
|
||||
error: WorkflowError,
|
||||
started: float,
|
||||
) -> None:
|
||||
duration = int((time.monotonic() - started) * 1000)
|
||||
body = error.to_body().model_dump(by_alias=True)
|
||||
if self._store is not None and node_run is not None:
|
||||
try:
|
||||
await self._store.record_node_run(
|
||||
{
|
||||
"run_id": ctx.run_id,
|
||||
"node_id": node_run["node_id"],
|
||||
"node_type": node_run["node_type"],
|
||||
"attempt": node_run["attempt"],
|
||||
"iteration": node_run["iteration"],
|
||||
"status": "failed",
|
||||
"error": body,
|
||||
"duration_ms": duration,
|
||||
"finished_at": datetime.now(UTC),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("failed to record node run failure", exc_info=True)
|
||||
await ctx.emit_event(
|
||||
"node.failed",
|
||||
data={"error": body, "durationMs": duration},
|
||||
node_id=error.node_id,
|
||||
)
|
||||
|
||||
def _collect_output(self, nodes: dict[str, WorkflowNode], ctx: RunContext) -> dict[str, Any]:
|
||||
output: dict[str, Any] = {}
|
||||
for node_id, node in nodes.items():
|
||||
if node.type != "output":
|
||||
continue
|
||||
result = ctx.node_results.get(node_id)
|
||||
if isinstance(result, dict) and isinstance(result.get("data"), dict):
|
||||
output.update(result["data"])
|
||||
return output
|
||||
|
||||
|
||||
__all__ = ["EngineResult", "WorkflowEngine"]
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Persist-then-publish event sink for workflow runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.workflows.events import WorkflowEventEnvelope, WorkflowEventType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PublishFn = Callable[[dict[str, Any]], Awaitable[None]]
|
||||
|
||||
|
||||
class PersistingWorkflowEventSink:
|
||||
"""Assigns ``seq`` in the database first, then fans out to live listeners.
|
||||
|
||||
A persistence failure never aborts the run, but the event is **dropped
|
||||
rather than published**: SSE clients tail by cursor and rebuild from the
|
||||
durable log, so a live-only frame with a made-up seq would render once and
|
||||
then vanish on reconnect (a "ghost event"). Callers can see the drop via
|
||||
``degraded_count`` / the ``eventPersistenceDegraded`` marker on the returned
|
||||
envelope; the stream endpoint synthesises a closing frame from the run row
|
||||
when a terminal event could not be logged.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: WorkflowEventStore,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
publish: PublishFn | None = None,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._run_id = run_id
|
||||
self._workflow_id = workflow_id
|
||||
self._version_id = version_id
|
||||
self._publish = publish
|
||||
self.degraded_count = 0
|
||||
|
||||
async def emit(
|
||||
self,
|
||||
event_type: WorkflowEventType,
|
||||
*,
|
||||
data: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> WorkflowEventEnvelope:
|
||||
payload = data or {}
|
||||
seq = 0
|
||||
degraded = False
|
||||
try:
|
||||
row = await self._store.append(
|
||||
run_id=self._run_id,
|
||||
workflow_id=self._workflow_id,
|
||||
version_id=self._version_id,
|
||||
event_type=event_type,
|
||||
payload=payload,
|
||||
node_id=node_id,
|
||||
node_run_id=node_run_id,
|
||||
)
|
||||
seq = int(row.get("seq") or 0)
|
||||
except Exception as exc: # noqa: BLE001 - the run must survive a log outage
|
||||
logger.warning(
|
||||
"workflow event not persisted (dropped, not published): run=%s event=%s node=%s error=%s:%s",
|
||||
self._run_id,
|
||||
event_type,
|
||||
node_id,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
self.degraded_count += 1
|
||||
degraded = True
|
||||
|
||||
envelope = WorkflowEventEnvelope(
|
||||
runId=self._run_id,
|
||||
workflowId=self._workflow_id,
|
||||
versionId=self._version_id,
|
||||
seq=seq,
|
||||
event=event_type,
|
||||
nodeId=node_id,
|
||||
nodeRunId=node_run_id,
|
||||
timestamp=datetime.now(UTC),
|
||||
data={**payload, "eventPersistenceDegraded": True} if degraded else payload,
|
||||
)
|
||||
if degraded:
|
||||
return envelope
|
||||
if self._publish is not None:
|
||||
try:
|
||||
await self._publish(envelope.to_sse_dict())
|
||||
except Exception: # noqa: BLE001 - live fan-out is best effort
|
||||
logger.debug("workflow live publish failed", exc_info=True)
|
||||
return envelope
|
||||
|
||||
|
||||
def envelope_from_row(row: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Turn a stored event row into the SSE wire object (replay path)."""
|
||||
return {
|
||||
"schemaVersion": "1.0",
|
||||
"runId": row.get("run_id"),
|
||||
"workflowId": row.get("workflow_id"),
|
||||
"versionId": row.get("version_id"),
|
||||
"seq": int(row.get("seq") or 0),
|
||||
"event": row.get("event_type"),
|
||||
"nodeId": row.get("node_id"),
|
||||
"nodeRunId": row.get("node_run_id"),
|
||||
"timestamp": row.get("created_at"),
|
||||
"data": row.get("payload") or {},
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["PersistingWorkflowEventSink", "envelope_from_row"]
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
"""Read-only query execution for the ``sql_read`` node.
|
||||
|
||||
Engines are cached per DSN with a tiny pool because a workflow typically hits
|
||||
the same reporting database many times in one run. Values are bound as
|
||||
parameters and every cell is truncated before it can reach an event payload.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_engines: dict[str, AsyncEngine] = {}
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
|
||||
async def _get_engine(dsn: str) -> AsyncEngine:
|
||||
async with _lock:
|
||||
engine = _engines.get(dsn)
|
||||
if engine is None:
|
||||
engine = create_async_engine(
|
||||
dsn,
|
||||
pool_pre_ping=True,
|
||||
pool_size=2,
|
||||
max_overflow=2,
|
||||
pool_recycle=1800,
|
||||
)
|
||||
_engines[dsn] = engine
|
||||
return engine
|
||||
|
||||
|
||||
async def dispose_engines() -> None:
|
||||
async with _lock:
|
||||
for engine in _engines.values():
|
||||
await engine.dispose()
|
||||
_engines.clear()
|
||||
|
||||
|
||||
def _coerce(value: Any, max_chars: int) -> Any:
|
||||
if value is None or isinstance(value, (bool, int, float)):
|
||||
return value
|
||||
if isinstance(value, Decimal):
|
||||
return float(value)
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, (bytes, bytearray, memoryview)):
|
||||
return f"<binary {len(bytes(value))} bytes>"
|
||||
text_value = str(value)
|
||||
return text_value if len(text_value) <= max_chars else text_value[:max_chars] + "…"
|
||||
|
||||
|
||||
async def execute_read_query(
|
||||
*,
|
||||
dsn: str,
|
||||
statement: str,
|
||||
parameters: dict[str, Any],
|
||||
max_rows: int,
|
||||
timeout_seconds: int,
|
||||
max_cell_chars: int = 4000,
|
||||
) -> dict[str, Any]:
|
||||
engine = await _get_engine(dsn)
|
||||
|
||||
async def _run() -> dict[str, Any]:
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text(statement), parameters)
|
||||
columns = list(result.keys())
|
||||
rows = result.fetchmany(max_rows)
|
||||
truncated = bool(result.fetchone())
|
||||
return {
|
||||
"columns": columns,
|
||||
"rows": [{col: _coerce(val, max_cell_chars) for col, val in zip(columns, row, strict=False)} for row in rows],
|
||||
"rowCount": len(rows),
|
||||
"truncated": truncated,
|
||||
}
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(_run(), timeout=timeout_seconds)
|
||||
except TimeoutError as exc:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"数据库查询超时({timeout_seconds}s)",
|
||||
retryable=True,
|
||||
details={"timeoutSeconds": timeout_seconds},
|
||||
) from exc
|
||||
except WorkflowError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - never leak DSN/credentials upward
|
||||
logger.warning("workflow sql_read failed", exc_info=True)
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SQL_FAILED",
|
||||
"数据库查询执行失败",
|
||||
retryable=True,
|
||||
details={"reason": type(exc).__name__},
|
||||
) from exc
|
||||
|
||||
|
||||
__all__ = ["dispose_engines", "execute_read_query"]
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""Load frozen workflow sample graphs shipped with the harness."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.workflows.schemas import WorkflowGraph
|
||||
|
||||
_SAMPLES_DIR = Path(__file__).resolve().parent
|
||||
_NODES_DIR = _SAMPLES_DIR / "nodes"
|
||||
|
||||
|
||||
def samples_dir() -> Path:
|
||||
return _SAMPLES_DIR
|
||||
|
||||
|
||||
def nodes_dir() -> Path:
|
||||
return _NODES_DIR
|
||||
|
||||
|
||||
@lru_cache(maxsize=32)
|
||||
def load_sample_graph(name: str = "report_generation") -> WorkflowGraph:
|
||||
"""Load and validate a sample graph by stem name (without ``.json``)."""
|
||||
path = _SAMPLES_DIR / f"{name}.json"
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"workflow sample not found: {path}")
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
return WorkflowGraph.model_validate(raw)
|
||||
|
||||
|
||||
@lru_cache(maxsize=64)
|
||||
def load_node_sample(node_type: str) -> dict:
|
||||
"""Load a minimal single-node fixture (raw dict, not a full graph)."""
|
||||
path = _NODES_DIR / f"{node_type}.json"
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"node sample not found: {path}")
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def list_node_samples() -> list[str]:
|
||||
if not _NODES_DIR.is_dir():
|
||||
return []
|
||||
return sorted(p.stem for p in _NODES_DIR.glob("*.json"))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"list_node_samples",
|
||||
"load_node_sample",
|
||||
"load_sample_graph",
|
||||
"nodes_dir",
|
||||
"samples_dir",
|
||||
]
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"runId": "run_01",
|
||||
"workflowId": "wf_report_generation",
|
||||
"versionId": "wv_03",
|
||||
"seq": 42,
|
||||
"event": "node.output.delta",
|
||||
"nodeId": "agent_write",
|
||||
"nodeRunId": "nr_09",
|
||||
"timestamp": "2026-08-26T10:00:00+08:00",
|
||||
"data": {
|
||||
"channel": "answer",
|
||||
"delta": "第一段内容"
|
||||
}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"id": "agent_1",
|
||||
"type": "agent",
|
||||
"name": "智能体",
|
||||
"config": {
|
||||
"agentId": "report-writer",
|
||||
"promptTemplate": "根据材料撰写:{{ nodes.merge_1.data }}",
|
||||
"responseMode": "text",
|
||||
"threadMode": "isolated_per_node",
|
||||
"timeoutSeconds": 600,
|
||||
"skillNames": []
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"id": "code_1",
|
||||
"type": "code",
|
||||
"name": "代码(沙箱)",
|
||||
"config": {
|
||||
"language": "python",
|
||||
"source": "def main(inputs):\n rows = inputs.get(\"rows\") or []\n return {\"count\": len(rows), \"titles\": [r.get(\"title\") for r in rows[:10]]}\n",
|
||||
"timeoutSeconds": 60,
|
||||
"allowNetwork": false
|
||||
}
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"id": "cond_1",
|
||||
"type": "condition",
|
||||
"name": "条件",
|
||||
"config": {
|
||||
"branches": [
|
||||
{
|
||||
"name": "passed",
|
||||
"expression": "{{ nodes.agent_review.data.passed }} == true"
|
||||
},
|
||||
{
|
||||
"name": "rejected",
|
||||
"expression": "{{ nodes.agent_review.data.passed }} == false"
|
||||
}
|
||||
],
|
||||
"defaultBranch": "rejected"
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"id": "http_1",
|
||||
"type": "http",
|
||||
"name": "HTTP",
|
||||
"config": {
|
||||
"method": "GET",
|
||||
"url": "https://api.example.internal/items/{{ inputs.taskId }}",
|
||||
"headers": { "Accept": "application/json" },
|
||||
"query": {},
|
||||
"body": null,
|
||||
"credentialRef": "cred_task_api",
|
||||
"timeoutSeconds": 30,
|
||||
"maxResponseBytes": 1048576,
|
||||
"allowRedirects": false
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"id": "human_1",
|
||||
"type": "human_input",
|
||||
"name": "人工介入",
|
||||
"config": {
|
||||
"formSchema": {
|
||||
"type": "object",
|
||||
"required": ["decision"],
|
||||
"properties": {
|
||||
"decision": {
|
||||
"type": "string",
|
||||
"enum": ["approve", "revise"]
|
||||
},
|
||||
"comment": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"prompt": "请确认是否放行当前报告草稿。",
|
||||
"actions": ["approve", "revise"]
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"id": "loop_1",
|
||||
"type": "loop",
|
||||
"name": "受限循环",
|
||||
"config": {
|
||||
"maxIterations": 2,
|
||||
"continueWhen": "{{ nodes.agent_review.data.passed }} == false",
|
||||
"bodyEntry": "agent_write"
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"id": "merge_1",
|
||||
"type": "merge",
|
||||
"name": "合并",
|
||||
"config": {
|
||||
"strategy": "all",
|
||||
"sources": ["sql_1", "http_1", "agent_collect"]
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"id": "output_1",
|
||||
"type": "output",
|
||||
"name": "输出",
|
||||
"config": {
|
||||
"mapping": {
|
||||
"reportMarkdown": "{{ nodes.agent_write.data.answer }}"
|
||||
},
|
||||
"llmRepair": false
|
||||
}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"id": "skill_1",
|
||||
"type": "skill",
|
||||
"name": "技能(agent_skill)",
|
||||
"config": {
|
||||
"mode": "agent_skill",
|
||||
"agentId": "report-collector",
|
||||
"skillNames": ["web-research"],
|
||||
"promptTemplate": "使用指定技能完成:{{ inputs.topic }}",
|
||||
"timeoutSeconds": 300
|
||||
}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"id": "sql_1",
|
||||
"type": "sql_read",
|
||||
"name": "SQL 只读",
|
||||
"config": {
|
||||
"dataSourceId": "ds_report_readonly",
|
||||
"statement": "SELECT id, title FROM report_source WHERE task_id = :task_id",
|
||||
"parameters": {
|
||||
"task_id": "{{ inputs.taskId }}"
|
||||
},
|
||||
"maxRows": 1000
|
||||
}
|
||||
}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"id": "start_1",
|
||||
"type": "start",
|
||||
"name": "开始",
|
||||
"config": {}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"id": "sub_1",
|
||||
"type": "subworkflow",
|
||||
"name": "子工作流",
|
||||
"config": {
|
||||
"workflowId": "wf_collect_pack",
|
||||
"versionId": "wv_01",
|
||||
"inputMapping": {
|
||||
"topic": "{{ inputs.topic }}",
|
||||
"taskId": "{{ inputs.taskId }}"
|
||||
}
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"id": "transform_1",
|
||||
"type": "transform",
|
||||
"name": "模板/转换",
|
||||
"config": {
|
||||
"operations": [
|
||||
{
|
||||
"op": "map",
|
||||
"from": "{{ nodes.sql_1.data.rows }}",
|
||||
"to": "rows"
|
||||
},
|
||||
{
|
||||
"op": "template",
|
||||
"template": "# {{ inputs.topic }}\n\n共 {{ nodes.sql_1.metadata.rowCount }} 行。",
|
||||
"to": "markdown"
|
||||
}
|
||||
],
|
||||
"template": null
|
||||
}
|
||||
}
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"id": "wf_report_generation",
|
||||
"name": "专题报告生成",
|
||||
"description": "阶段0验收基线:并行取数 → 合并 → 整理 → 撰写 → 审核 → 条件退回/输出",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"required": ["topic", "taskId"],
|
||||
"properties": {
|
||||
"topic": { "type": "string", "description": "报告主题" },
|
||||
"taskId": { "type": "string", "description": "业务任务 ID" },
|
||||
"dateRange": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"start": { "type": "string" },
|
||||
"end": { "type": "string" }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"outputSchema": {
|
||||
"type": "object",
|
||||
"required": ["reportMarkdown", "passed"],
|
||||
"properties": {
|
||||
"reportMarkdown": { "type": "string" },
|
||||
"passed": { "type": "boolean" },
|
||||
"artifacts": { "type": "array" },
|
||||
"issues": { "type": "array" }
|
||||
}
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"id": "start_1",
|
||||
"type": "start",
|
||||
"name": "开始",
|
||||
"config": {}
|
||||
},
|
||||
{
|
||||
"id": "sql_1",
|
||||
"type": "sql_read",
|
||||
"name": "查询内部结构化数据",
|
||||
"config": {
|
||||
"dataSourceId": "ds_report_readonly",
|
||||
"statement": "SELECT id, title, content, updated_at FROM report_source WHERE task_id = :task_id ORDER BY updated_at DESC",
|
||||
"parameters": {
|
||||
"task_id": "{{ inputs.taskId }}"
|
||||
},
|
||||
"maxRows": 1000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "http_1",
|
||||
"type": "http",
|
||||
"name": "调用只读业务接口",
|
||||
"config": {
|
||||
"method": "GET",
|
||||
"url": "https://api.example.internal/tasks/{{ inputs.taskId }}/summary",
|
||||
"headers": {
|
||||
"Accept": "application/json"
|
||||
},
|
||||
"credentialRef": "cred_task_api",
|
||||
"timeoutSeconds": 30,
|
||||
"maxResponseBytes": 524288,
|
||||
"allowRedirects": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "agent_collect",
|
||||
"type": "agent",
|
||||
"name": "外部材料采集",
|
||||
"config": {
|
||||
"agentId": "report-collector",
|
||||
"promptTemplate": "围绕主题「{{ inputs.topic }}」搜集外部公开材料,任务 ID:{{ inputs.taskId }}。输出 JSON:{sources:[], notes:string}",
|
||||
"responseMode": "json",
|
||||
"responseSchema": {
|
||||
"type": "object",
|
||||
"required": ["sources", "notes"],
|
||||
"properties": {
|
||||
"sources": { "type": "array" },
|
||||
"notes": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"threadMode": "isolated_per_node",
|
||||
"timeoutSeconds": 600,
|
||||
"skillNames": ["web-research"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "merge_1",
|
||||
"type": "merge",
|
||||
"name": "合并材料并保留来源",
|
||||
"config": {
|
||||
"strategy": "all",
|
||||
"sources": ["sql_1", "http_1", "agent_collect"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "agent_clean",
|
||||
"type": "agent",
|
||||
"name": "数据整理与事实去重",
|
||||
"config": {
|
||||
"agentId": "report-cleaner",
|
||||
"promptTemplate": "整理并去重以下材料,输出严格 JSON:{{ nodes.merge_1.data }}",
|
||||
"responseMode": "json",
|
||||
"responseSchema": {
|
||||
"type": "object",
|
||||
"required": ["facts", "sources"],
|
||||
"properties": {
|
||||
"facts": { "type": "array" },
|
||||
"sources": { "type": "array" }
|
||||
}
|
||||
},
|
||||
"threadMode": "isolated_per_node",
|
||||
"timeoutSeconds": 600
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "agent_write",
|
||||
"type": "agent",
|
||||
"name": "撰写 Markdown 报告",
|
||||
"config": {
|
||||
"agentId": "report-writer",
|
||||
"promptTemplate": "根据整理结果撰写专题报告(Markdown)。主题:{{ inputs.topic }}\n材料:{{ nodes.agent_clean.data }}\n若存在审核意见:{{ nodes.agent_review.data.revisedAdvice }}",
|
||||
"responseMode": "text",
|
||||
"threadMode": "isolated_per_node",
|
||||
"timeoutSeconds": 900
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "agent_review",
|
||||
"type": "agent",
|
||||
"name": "审核报告",
|
||||
"config": {
|
||||
"agentId": "report-reviewer",
|
||||
"promptTemplate": "审核以下报告,输出 JSON {passed:boolean, issues:array, revisedAdvice:string}:\n{{ nodes.agent_write.data.answer }}",
|
||||
"responseMode": "json",
|
||||
"responseSchema": {
|
||||
"type": "object",
|
||||
"required": ["passed", "issues", "revisedAdvice"],
|
||||
"properties": {
|
||||
"passed": { "type": "boolean" },
|
||||
"issues": { "type": "array" },
|
||||
"revisedAdvice": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"threadMode": "isolated_per_node",
|
||||
"timeoutSeconds": 600
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "cond_review",
|
||||
"type": "condition",
|
||||
"name": "审核结果分支",
|
||||
"config": {
|
||||
"branches": [
|
||||
{
|
||||
"name": "passed",
|
||||
"expression": "{{ nodes.agent_review.data.passed }} == true"
|
||||
},
|
||||
{
|
||||
"name": "rejected",
|
||||
"expression": "{{ nodes.agent_review.data.passed }} == false"
|
||||
}
|
||||
],
|
||||
"defaultBranch": "rejected"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "loop_revise",
|
||||
"type": "loop",
|
||||
"name": "退回修改(最多 2 次)",
|
||||
"config": {
|
||||
"maxIterations": 2,
|
||||
"continueWhen": "{{ nodes.agent_review.data.passed }} == false",
|
||||
"bodyEntry": "agent_write"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "transform_pack",
|
||||
"type": "transform",
|
||||
"name": "组装最终字段",
|
||||
"config": {
|
||||
"operations": [
|
||||
{
|
||||
"op": "map",
|
||||
"from": "{{ nodes.agent_write.data.answer }}",
|
||||
"to": "reportMarkdown"
|
||||
},
|
||||
{
|
||||
"op": "map",
|
||||
"from": "{{ nodes.agent_review.data.passed }}",
|
||||
"to": "passed"
|
||||
},
|
||||
{
|
||||
"op": "map",
|
||||
"from": "{{ nodes.agent_review.data.issues }}",
|
||||
"to": "issues"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "output_1",
|
||||
"type": "output",
|
||||
"name": "输出",
|
||||
"config": {
|
||||
"mapping": {
|
||||
"reportMarkdown": "{{ nodes.transform_pack.data.reportMarkdown }}",
|
||||
"passed": "{{ nodes.transform_pack.data.passed }}",
|
||||
"issues": "{{ nodes.transform_pack.data.issues }}"
|
||||
},
|
||||
"llmRepair": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{ "id": "e_start_sql", "source": "start_1", "target": "sql_1" },
|
||||
{ "id": "e_start_http", "source": "start_1", "target": "http_1" },
|
||||
{ "id": "e_start_collect", "source": "start_1", "target": "agent_collect" },
|
||||
{ "id": "e_sql_merge", "source": "sql_1", "target": "merge_1" },
|
||||
{ "id": "e_http_merge", "source": "http_1", "target": "merge_1" },
|
||||
{ "id": "e_collect_merge", "source": "agent_collect", "target": "merge_1" },
|
||||
{ "id": "e_merge_clean", "source": "merge_1", "target": "agent_clean" },
|
||||
{ "id": "e_clean_write", "source": "agent_clean", "target": "agent_write" },
|
||||
{ "id": "e_write_review", "source": "agent_write", "target": "agent_review" },
|
||||
{ "id": "e_review_cond", "source": "agent_review", "target": "cond_review" },
|
||||
{
|
||||
"id": "e_cond_pass",
|
||||
"source": "cond_review",
|
||||
"target": "transform_pack",
|
||||
"sourcePort": "passed"
|
||||
},
|
||||
{
|
||||
"id": "e_cond_reject",
|
||||
"source": "cond_review",
|
||||
"target": "loop_revise",
|
||||
"sourcePort": "rejected"
|
||||
},
|
||||
{
|
||||
"id": "e_loop_write",
|
||||
"source": "loop_revise",
|
||||
"target": "agent_write",
|
||||
"sourcePort": "continue"
|
||||
},
|
||||
{
|
||||
"id": "e_loop_fail",
|
||||
"source": "loop_revise",
|
||||
"target": "transform_pack",
|
||||
"sourcePort": "exhausted"
|
||||
},
|
||||
{ "id": "e_pack_out", "source": "transform_pack", "target": "output_1" }
|
||||
],
|
||||
"settings": {
|
||||
"runTimeoutSeconds": 1800,
|
||||
"nodeTimeoutSeconds": 300,
|
||||
"maxSteps": 200,
|
||||
"maxLoopIterations": 5,
|
||||
"maxParallelism": 4
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"id": "wf_start_output",
|
||||
"name": "空图:开始→输出",
|
||||
"description": "阶段2运行骨架用的最小可执行图",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"outputSchema": {
|
||||
"type": "object",
|
||||
"required": ["message"],
|
||||
"properties": {
|
||||
"message": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"nodes": [
|
||||
{ "id": "start_1", "type": "start", "name": "开始", "config": {} },
|
||||
{
|
||||
"id": "output_1",
|
||||
"type": "output",
|
||||
"name": "输出",
|
||||
"config": {
|
||||
"mapping": { "message": "{{ inputs.message }}" },
|
||||
"llmRepair": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{ "id": "e1", "source": "start_1", "target": "output_1" }
|
||||
],
|
||||
"settings": {
|
||||
"runTimeoutSeconds": 60,
|
||||
"nodeTimeoutSeconds": 30,
|
||||
"maxSteps": 10,
|
||||
"maxLoopIterations": 1,
|
||||
"maxParallelism": 1
|
||||
}
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
"""Full JSON Schema validation for workflow input/output values.
|
||||
|
||||
The engine used to check only ``required`` keys; everything else (wrong type,
|
||||
unknown enum value, nested ``properties``/``items`` violations) slipped through
|
||||
until downstream nodes exploded with confusing errors. This module runs the
|
||||
graph's ``inputSchema`` / ``outputSchema`` through ``Draft202012Validator`` and
|
||||
returns structured, frontend-friendly errors (path + message) instead of
|
||||
leaking Python tracebacks.
|
||||
|
||||
Kept as pure functions so both the node executors (``WorkflowError``) and the
|
||||
HTTP router (400 responses) can share the exact same rules.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
# Cap the reported errors so a wildly invalid payload cannot produce a
|
||||
# multi-megabyte error body.
|
||||
_MAX_ERRORS = 20
|
||||
|
||||
|
||||
def schema_issues(value: Any, schema: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
"""Validate ``value`` against ``schema``.
|
||||
|
||||
Returns ``[]`` when the schema is empty/absent (nothing declared = nothing
|
||||
enforced). Each issue is ``{"path": [...], "message": str}`` with ``path``
|
||||
pointing at the offending field, e.g. ``["options", 0, "name"]``.
|
||||
"""
|
||||
if not isinstance(schema, dict) or not schema:
|
||||
return []
|
||||
try:
|
||||
validator = Draft202012Validator(schema)
|
||||
# Construction alone does not check the schema — malformed keywords
|
||||
# (e.g. a non-string ``pattern``) only explode during validation.
|
||||
errors = sorted(validator.iter_errors(value), key=lambda err: list(err.absolute_path))
|
||||
except Exception: # noqa: BLE001 - a malformed user schema must not 500
|
||||
return [{"path": [], "message": "schema 定义本身无效,无法用于校验"}]
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[tuple[Any, ...]] = set()
|
||||
for err in errors:
|
||||
path = [str(p) if not isinstance(p, int) else p for p in err.absolute_path]
|
||||
key = tuple(path)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append({"path": path, "message": _humanize(err)})
|
||||
if len(out) >= _MAX_ERRORS:
|
||||
out.append({"path": [], "message": f"……其余问题已省略(共 {len(errors)} 处)"})
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _humanize(err: Any) -> str:
|
||||
"""Map the common jsonschema phrasings to short Chinese messages."""
|
||||
message = str(err.message)
|
||||
validator = getattr(err, "validator", "")
|
||||
path = ".".join(str(p) for p in err.absolute_path) or "输入"
|
||||
if validator == "required":
|
||||
missing = message.split("'")[1] if "'" in message else "?"
|
||||
return f"{path} 缺少必填字段:{missing}"
|
||||
if validator == "enum":
|
||||
return f"{path} 的值不在允许范围内:{message}"
|
||||
if validator == "type":
|
||||
return f"{path} 类型不正确:{message}"
|
||||
if validator == "additionalProperties":
|
||||
return f"{path} 含有未声明的额外字段:{message}"
|
||||
return f"{path} 不符合 schema:{message}"
|
||||
|
||||
|
||||
__all__ = ["schema_issues"]
|
||||
@@ -0,0 +1,374 @@
|
||||
"""Frozen workflow graph schema v1.0 (internal DTO).
|
||||
|
||||
Coze canvas DTO is converted into this structure at the Gateway boundary.
|
||||
The runtime must never depend on Coze-private fields.
|
||||
|
||||
Wire JSON uses camelCase aliases; Python code uses snake_case.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
SCHEMA_VERSION = "1.0"
|
||||
|
||||
NodeType = Literal[
|
||||
"start",
|
||||
"output",
|
||||
"agent",
|
||||
"skill",
|
||||
"http",
|
||||
"sql_read",
|
||||
"code",
|
||||
"transform",
|
||||
"condition",
|
||||
"merge",
|
||||
"human_input",
|
||||
"subworkflow",
|
||||
"loop",
|
||||
]
|
||||
|
||||
ALL_NODE_TYPES: tuple[NodeType, ...] = (
|
||||
"start",
|
||||
"output",
|
||||
"agent",
|
||||
"skill",
|
||||
"http",
|
||||
"sql_read",
|
||||
"code",
|
||||
"transform",
|
||||
"condition",
|
||||
"merge",
|
||||
"human_input",
|
||||
"subworkflow",
|
||||
"loop",
|
||||
)
|
||||
|
||||
RunStatus = Literal[
|
||||
"queued",
|
||||
"running",
|
||||
"awaiting_input",
|
||||
"completed",
|
||||
"failed",
|
||||
"cancel_requested",
|
||||
"cancelled",
|
||||
]
|
||||
|
||||
ALL_RUN_STATUSES: tuple[RunStatus, ...] = (
|
||||
"queued",
|
||||
"running",
|
||||
"awaiting_input",
|
||||
"completed",
|
||||
"failed",
|
||||
"cancel_requested",
|
||||
"cancelled",
|
||||
)
|
||||
|
||||
TERMINAL_RUN_STATUSES: frozenset[RunStatus] = frozenset({"completed", "failed", "cancelled"})
|
||||
|
||||
NodeRunStatus = Literal[
|
||||
"queued",
|
||||
"running",
|
||||
"completed",
|
||||
"failed",
|
||||
"skipped",
|
||||
"cancelled",
|
||||
]
|
||||
|
||||
SkillMode = Literal["agent_skill", "callable_skill"]
|
||||
MergeStrategy = Literal["all", "first_success", "any_success"]
|
||||
HttpMethod = Literal["GET", "POST", "PUT", "PATCH", "DELETE"]
|
||||
AgentResponseMode = Literal["text", "json"]
|
||||
AgentThreadMode = Literal["isolated_per_node", "shared_run_thread"]
|
||||
ExecutionMode = Literal["normal"]
|
||||
|
||||
|
||||
class WorkflowGraphSettings(BaseModel):
|
||||
"""Per-graph resource limits (still capped by system limits at publish)."""
|
||||
|
||||
run_timeout_seconds: int = Field(default=1800, ge=1, alias="runTimeoutSeconds")
|
||||
node_timeout_seconds: int = Field(default=300, ge=1, alias="nodeTimeoutSeconds")
|
||||
max_steps: int = Field(default=200, ge=1, alias="maxSteps")
|
||||
max_loop_iterations: int = Field(default=5, ge=1, alias="maxLoopIterations")
|
||||
max_parallelism: int = Field(default=4, ge=1, alias="maxParallelism")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class WorkflowNode(BaseModel):
|
||||
"""A single node on the internal standard graph."""
|
||||
|
||||
id: str = Field(min_length=1)
|
||||
type: NodeType
|
||||
name: str = ""
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
@field_validator("id")
|
||||
@classmethod
|
||||
def _id_not_blank(cls, value: str) -> str:
|
||||
stripped = value.strip()
|
||||
if not stripped:
|
||||
raise ValueError("node id must be non-empty")
|
||||
return stripped
|
||||
|
||||
|
||||
class WorkflowEdge(BaseModel):
|
||||
"""Directed edge. ``source_port`` names a condition / loop outlet when set."""
|
||||
|
||||
id: str = Field(min_length=1)
|
||||
source: str
|
||||
target: str
|
||||
source_port: str | None = Field(default=None, alias="sourcePort")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class WorkflowGraph(BaseModel):
|
||||
"""Internal standard workflow graph (publish / run snapshot)."""
|
||||
|
||||
schema_version: Literal["1.0"] = Field(default=SCHEMA_VERSION, alias="schemaVersion")
|
||||
id: str = Field(min_length=1)
|
||||
name: str = ""
|
||||
description: str = ""
|
||||
input_schema: dict[str, Any] = Field(default_factory=dict, alias="inputSchema")
|
||||
output_schema: dict[str, Any] = Field(default_factory=dict, alias="outputSchema")
|
||||
nodes: list[WorkflowNode] = Field(default_factory=list)
|
||||
edges: list[WorkflowEdge] = Field(default_factory=list)
|
||||
settings: WorkflowGraphSettings = Field(default_factory=WorkflowGraphSettings)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _unique_ids(self) -> WorkflowGraph:
|
||||
node_ids = [n.id for n in self.nodes]
|
||||
if len(node_ids) != len(set(node_ids)):
|
||||
raise ValueError("duplicate node id")
|
||||
edge_ids = [e.id for e in self.edges]
|
||||
if len(edge_ids) != len(set(edge_ids)):
|
||||
raise ValueError("duplicate edge id")
|
||||
known = set(node_ids)
|
||||
for edge in self.edges:
|
||||
if edge.source not in known:
|
||||
raise ValueError(f"edge {edge.id} source not found: {edge.source}")
|
||||
if edge.target not in known:
|
||||
raise ValueError(f"edge {edge.id} target not found: {edge.target}")
|
||||
return self
|
||||
|
||||
def node_map(self) -> dict[str, WorkflowNode]:
|
||||
return {node.id: node for node in self.nodes}
|
||||
|
||||
def canonical_json(self) -> str:
|
||||
"""Deterministic JSON used for ``graph_hash`` (sorted keys, no whitespace)."""
|
||||
return self.model_dump_json(by_alias=True, exclude_none=True)
|
||||
|
||||
def graph_hash(self) -> str:
|
||||
return sha256(self.canonical_json().encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class ArtifactRef(BaseModel):
|
||||
"""Reference to a large/binary result; full bytes live outside events."""
|
||||
|
||||
artifact_id: str = Field(alias="artifactId")
|
||||
name: str
|
||||
mime_type: str = Field(default="application/octet-stream", alias="mimeType")
|
||||
path: str = ""
|
||||
size_bytes: int | None = Field(default=None, alias="sizeBytes")
|
||||
preview: str | None = None
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class NodeResult(BaseModel):
|
||||
"""Uniform result returned by every node executor."""
|
||||
|
||||
data: dict[str, Any] = Field(default_factory=dict)
|
||||
messages: list[dict[str, Any]] = Field(default_factory=list)
|
||||
artifacts: list[ArtifactRef] = Field(default_factory=list)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class StartRunRequest(BaseModel):
|
||||
"""Body for ``POST /api/workflows/{workflow_id}/runs``."""
|
||||
|
||||
version_id: str = Field(alias="versionId")
|
||||
inputs: dict[str, Any] = Field(default_factory=dict)
|
||||
idempotency_key: str | None = Field(default=None, alias="idempotencyKey")
|
||||
execution_mode: ExecutionMode = Field(default="normal", alias="executionMode")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class ResumeRunRequest(BaseModel):
|
||||
"""Body for ``POST /api/workflows/runs/{run_id}/resume``."""
|
||||
|
||||
resume_token: str = Field(alias="resumeToken")
|
||||
action: str
|
||||
payload: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
# ── Typed node config helpers (optional; validators may use later) ───────────
|
||||
|
||||
|
||||
class AgentNodeConfig(BaseModel):
|
||||
agent_id: str = Field(alias="agentId")
|
||||
prompt_template: str = Field(default="", alias="promptTemplate")
|
||||
response_mode: AgentResponseMode = Field(default="text", alias="responseMode")
|
||||
response_schema: dict[str, Any] = Field(default_factory=dict, alias="responseSchema")
|
||||
thread_mode: AgentThreadMode = Field(default="isolated_per_node", alias="threadMode")
|
||||
timeout_seconds: int = Field(default=600, ge=1, alias="timeoutSeconds")
|
||||
skill_names: list[str] = Field(default_factory=list, alias="skillNames")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class SkillNodeConfig(BaseModel):
|
||||
mode: SkillMode = "agent_skill"
|
||||
agent_id: str | None = Field(default=None, alias="agentId")
|
||||
skill_names: list[str] = Field(default_factory=list, alias="skillNames")
|
||||
prompt_template: str = Field(default="", alias="promptTemplate")
|
||||
# callable_skill fields (phase 4+; accepted in schema, not executed in phase 0)
|
||||
entry: str | None = None
|
||||
input_schema: dict[str, Any] = Field(default_factory=dict, alias="inputSchema")
|
||||
output_schema: dict[str, Any] = Field(default_factory=dict, alias="outputSchema")
|
||||
timeout_seconds: int = Field(default=300, ge=1, alias="timeoutSeconds")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class HttpNodeConfig(BaseModel):
|
||||
method: HttpMethod = "GET"
|
||||
url: str
|
||||
headers: dict[str, str] = Field(default_factory=dict)
|
||||
query: dict[str, Any] = Field(default_factory=dict)
|
||||
body: Any = None
|
||||
credential_ref: str | None = Field(default=None, alias="credentialRef")
|
||||
timeout_seconds: int = Field(default=60, ge=1, alias="timeoutSeconds")
|
||||
max_response_bytes: int = Field(default=1_048_576, ge=1, alias="maxResponseBytes")
|
||||
allow_redirects: bool = Field(default=False, alias="allowRedirects")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class SqlReadNodeConfig(BaseModel):
|
||||
data_source_id: str = Field(alias="dataSourceId")
|
||||
statement: str
|
||||
parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
max_rows: int = Field(default=1000, ge=1, alias="maxRows")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class CodeNodeConfig(BaseModel):
|
||||
language: Literal["python"] = "python"
|
||||
source: str
|
||||
timeout_seconds: int = Field(default=60, ge=1, alias="timeoutSeconds")
|
||||
allow_network: bool = Field(default=False, alias="allowNetwork")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class TransformNodeConfig(BaseModel):
|
||||
operations: list[dict[str, Any]] = Field(default_factory=list)
|
||||
template: str | None = None
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class ConditionBranch(BaseModel):
|
||||
name: str
|
||||
expression: str
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class ConditionNodeConfig(BaseModel):
|
||||
branches: list[ConditionBranch] = Field(default_factory=list)
|
||||
default_branch: str | None = Field(default=None, alias="defaultBranch")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class MergeNodeConfig(BaseModel):
|
||||
strategy: MergeStrategy = "all"
|
||||
sources: list[str] = Field(default_factory=list)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class HumanInputNodeConfig(BaseModel):
|
||||
form_schema: dict[str, Any] = Field(default_factory=dict, alias="formSchema")
|
||||
prompt: str = ""
|
||||
actions: list[str] = Field(default_factory=lambda: ["submit"])
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class SubworkflowNodeConfig(BaseModel):
|
||||
workflow_id: str = Field(alias="workflowId")
|
||||
version_id: str = Field(alias="versionId")
|
||||
input_mapping: dict[str, str] = Field(default_factory=dict, alias="inputMapping")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class LoopNodeConfig(BaseModel):
|
||||
max_iterations: int = Field(default=5, ge=1, alias="maxIterations")
|
||||
continue_when: str = Field(default="", alias="continueWhen")
|
||||
body_entry: str = Field(alias="bodyEntry")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class OutputNodeConfig(BaseModel):
|
||||
mapping: dict[str, str] = Field(default_factory=dict)
|
||||
llm_repair: bool = Field(default=False, alias="llmRepair")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SCHEMA_VERSION",
|
||||
"ALL_NODE_TYPES",
|
||||
"ALL_RUN_STATUSES",
|
||||
"TERMINAL_RUN_STATUSES",
|
||||
"AgentNodeConfig",
|
||||
"AgentResponseMode",
|
||||
"AgentThreadMode",
|
||||
"ArtifactRef",
|
||||
"CodeNodeConfig",
|
||||
"ConditionBranch",
|
||||
"ConditionNodeConfig",
|
||||
"ExecutionMode",
|
||||
"HttpMethod",
|
||||
"HttpNodeConfig",
|
||||
"HumanInputNodeConfig",
|
||||
"LoopNodeConfig",
|
||||
"MergeNodeConfig",
|
||||
"MergeStrategy",
|
||||
"NodeResult",
|
||||
"NodeRunStatus",
|
||||
"NodeType",
|
||||
"OutputNodeConfig",
|
||||
"ResumeRunRequest",
|
||||
"RunStatus",
|
||||
"SkillMode",
|
||||
"SkillNodeConfig",
|
||||
"SqlReadNodeConfig",
|
||||
"StartRunRequest",
|
||||
"SubworkflowNodeConfig",
|
||||
"TransformNodeConfig",
|
||||
"WorkflowEdge",
|
||||
"WorkflowGraph",
|
||||
"WorkflowGraphSettings",
|
||||
"WorkflowNode",
|
||||
]
|
||||
Loaded 100 of 110 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user