diff --git a/frontend-web/src/components/workspace/chats/chat-box.tsx b/frontend-web/src/components/workspace/chats/chat-box.tsx index 342abb5e..f8021559 100644 --- a/frontend-web/src/components/workspace/chats/chat-box.tsx +++ b/frontend-web/src/components/workspace/chats/chat-box.tsx @@ -16,6 +16,7 @@ import { apiFetch } from "@/core/api"; import { getBackendBaseURL } from "@/core/config"; import { isThreadRagModeLocked, useLocalSettings } from "@/core/settings"; import { isSingleRoundRagMessage } from "@/core/threads/use-single-round-rag"; +import { isMultiRoundRagMessage } from "@/core/threads/use-multi-round-rag"; import { useSkills } from "@/core/skills/hooks"; import type { SkillDisplayConfig } from "@/core/skills/type"; import { @@ -93,13 +94,20 @@ const ChatBox: React.FC<{ const [settings] = useLocalSettings(); const ragModeEnabled = settings.display.ragModeEnabled || isThreadRagModeLocked(threadId); - // Frontend single-round RAG writes its own source tool message, so it should - // use the same reference panel without changing the user's global RAG mode. - const hasSingleRoundRag = useMemo( - () => (thread.values.messages ?? []).some(isSingleRoundRagMessage), + // Frontend RAG modes write their own hidden source tool messages. They use + // the existing reference panel, but must not request or write reference + // batches through DeerFlow's backend APIs. + const hasFrontendRag = useMemo( + () => + (thread.values.messages ?? []).some( + (message) => + isSingleRoundRagMessage(message) || isMultiRoundRagMessage(message), + ), [thread.values.messages], ); - const referencesEnabled = ragModeEnabled || hasSingleRoundRag; + const referencesEnabled = ragModeEnabled || hasFrontendRag; + const shouldFetchBackendReferenceBatches = + referencesEnabled && !hasFrontendRag; const { skills } = useSkills(); const { @@ -128,7 +136,7 @@ const ChatBox: React.FC<{ ); }, [referencesEnabled, skillDisplayMap, thread.isLoading, thread.values.messages]); useEffect(() => { - if (!referencesEnabled) { + if (!shouldFetchBackendReferenceBatches) { setBackendReferenceBatches([]); return; } @@ -164,7 +172,12 @@ const ChatBox: React.FC<{ } }); return () => controller.abort(); - }, [referencesEnabled, thread.isLoading, threadId, thread.values.messages]); + }, [ + shouldFetchBackendReferenceBatches, + thread.isLoading, + threadId, + thread.values.messages, + ]); const adaptedBackendReferenceBatches = useMemo( () => adaptBackendReferenceBatches( @@ -175,15 +188,22 @@ const ChatBox: React.FC<{ ); const referenceBatches = useMemo( () => { - if (adaptedBackendReferenceBatches.length > 0) { + if ( + shouldFetchBackendReferenceBatches && + adaptedBackendReferenceBatches.length > 0 + ) { return adaptedBackendReferenceBatches; } return fallbackReferenceBatches; }, - [adaptedBackendReferenceBatches, fallbackReferenceBatches], + [ + adaptedBackendReferenceBatches, + fallbackReferenceBatches, + shouldFetchBackendReferenceBatches, + ], ); useEffect(() => { - if (!referencesEnabled) { + if (!shouldFetchBackendReferenceBatches) { return; } const signature = JSON.stringify({ @@ -210,7 +230,14 @@ const ChatBox: React.FC<{ fallbackBatches: debugReferenceBatches(fallbackReferenceBatches), finalBatches: debugReferenceBatches(referenceBatches), }); - }, [adaptedBackendReferenceBatches, backendReferenceBatches, fallbackReferenceBatches, referencesEnabled, referenceBatches, threadId]); + }, [ + adaptedBackendReferenceBatches, + backendReferenceBatches, + fallbackReferenceBatches, + referenceBatches, + shouldFetchBackendReferenceBatches, + threadId, + ]); const latestReferenceBatch = referenceBatches.length > 0 ? referenceBatches[referenceBatches.length - 1]! @@ -221,7 +248,7 @@ const ChatBox: React.FC<{ const referenceSources = activeReferenceBatch?.sources ?? []; const hasCitationReferences = referenceSources.length > 0; useEffect(() => { - if (!referencesEnabled || !activeReferenceBatch) { + if (!shouldFetchBackendReferenceBatches || !activeReferenceBatch) { return; } logReferenceDebug("frontend-active-reference-batch-payload", threadId, { @@ -230,7 +257,7 @@ const ChatBox: React.FC<{ sourceCount: activeReferenceBatch.sources.length, sources: activeReferenceBatch.sources.map(debugReferenceSource), }); - }, [activeReferenceBatch, referencesEnabled, threadId]); + }, [activeReferenceBatch, shouldFetchBackendReferenceBatches, threadId]); useEffect(() => { if (hasCitationReferences) { setReferencesOpen(true); diff --git a/frontend-web/src/components/workspace/citations/citation-link.tsx b/frontend-web/src/components/workspace/citations/citation-link.tsx index 438cbc99..e69918c4 100644 --- a/frontend-web/src/components/workspace/citations/citation-link.tsx +++ b/frontend-web/src/components/workspace/citations/citation-link.tsx @@ -8,8 +8,13 @@ export function CitationLink({ href, children, className, + onReferenceClick, + onClick, ...props -}: ComponentProps<"a">) { +}: ComponentProps<"a"> & { + /** Lets a message renderer show an in-place source preview instead of scrolling. */ + onReferenceClick?: (referenceId: string) => void; +}) { const childrenText = normalizeCitationText(children); const displayText = childrenText ?? ""; const referenceId = href?.startsWith("#reference-") @@ -28,6 +33,11 @@ export function CitationLink({ )} onClick={(event) => { event.preventDefault(); + if (onReferenceClick) { + onReferenceClick(referenceId); + onClick?.(event); + return; + } const batchId = event.currentTarget.closest("[data-reference-batch-id]") ?.dataset.referenceBatchId; @@ -41,6 +51,7 @@ export function CitationLink({ target?.scrollIntoView({ behavior: "smooth", block: "center" }); }; window.setTimeout(scroll, 240); + onClick?.(event); }} {...props} > @@ -58,6 +69,7 @@ export function CitationLink({ "text-primary decoration-primary/30 hover:decoration-primary/60 underline underline-offset-2 transition-colors", className, )} + onClick={onClick} {...props} > {displayText || children} diff --git a/frontend-web/src/components/workspace/input-box.tsx b/frontend-web/src/components/workspace/input-box.tsx index da77009e..6b9f815d 100644 --- a/frontend-web/src/components/workspace/input-box.tsx +++ b/frontend-web/src/components/workspace/input-box.tsx @@ -210,6 +210,9 @@ export function InputBox({ const { models } = useModels(); const singleRoundRagActive = context.execution_profile === "single_round_rag"; + const multiRoundRagActive = + context.execution_profile === "multi_round_rag"; + const frontendRagActive = singleRoundRagActive || multiRoundRagActive; const modelOptions = useMemo(() => { return singleRoundRagActive ? extraModels : models; }, [extraModels, models, singleRoundRagActive]); @@ -235,7 +238,12 @@ export function InputBox({ !userPreferences.hide_admin_prompt_prefix && recommendedQuestions.length > 0; - useEffect(() => { + useEffect(() => { + // Multi-round RAG is a frontend execution profile, not a model option. + // Do not let the normal model fallback clear it on every render. + if (multiRoundRagActive) { + return; + } if (modelOptions.length === 0) { return; } @@ -259,7 +267,7 @@ export function InputBox({ mode: nextMode, execution_profile: fallbackModel.execution_profile, }); - }, [context, modelOptions, onContextChange]); + }, [context, modelOptions, multiRoundRagActive, onContextChange]); const selectedModel = useMemo(() => { if (modelOptions.length === 0) { @@ -342,6 +350,13 @@ export function InputBox({ execution_profile: "single_round_rag", }); }, [context, extraModels, onContextChange]); + + const handleMultiRoundRagSelect = useCallback(() => { + onContextChange?.({ + ...context, + execution_profile: "multi_round_rag", + }); + }, [context, onContextChange]); const handleReasoningEffortSelect = useCallback( (effort: "minimal" | "low" | "medium" | "high") => { @@ -712,14 +727,16 @@ export function InputBox({ /> */} - - + )} + {!frontendRagActive && 写作 - - } + {!frontendRagActive && 记忆 - - } + {!frontendRagActive && 参考文献 - - } + {!frontendRagActive && 上下文压缩 - + }
- {selectedSingleRoundRag ? ( + {frontendRagActive ? ( ) : context.mode === "flash" ? ( @@ -881,8 +900,10 @@ export function InputBox({ context.mode === "ultra" ? "golden-text" : "", )} > - {selectedSingleRoundRag - ? "单轮 RAG" + {frontendRagActive + ? multiRoundRagActive + ? "多轮 RAG" + : "单轮 RAG" : (context.mode === "flash" && t.inputBox.flashMode) || (context.mode === "thinking" && t.inputBox.reasoningMode) || @@ -899,7 +920,7 @@ export function InputBox({ @@ -920,7 +941,7 @@ export function InputBox({ {t.inputBox.flashModeDescription}
- {!selectedSingleRoundRag && context.mode === "flash" ? ( + {!frontendRagActive && context.mode === "flash" ? ( ) : (
@@ -929,7 +950,7 @@ export function InputBox({ {supportThinking && ( @@ -950,7 +971,7 @@ export function InputBox({ {t.inputBox.reasoningModeDescription}
- {!selectedSingleRoundRag && context.mode === "thinking" ? ( + {!frontendRagActive && context.mode === "thinking" ? ( ) : (
@@ -959,7 +980,7 @@ export function InputBox({ )} @@ -980,7 +1001,7 @@ export function InputBox({ {t.inputBox.proModeDescription}
- {!selectedSingleRoundRag && context.mode === "pro" ? ( + {!frontendRagActive && context.mode === "pro" ? ( ) : (
@@ -988,7 +1009,7 @@ export function InputBox({
- {!selectedSingleRoundRag && context.mode === "ultra" ? ( + {!frontendRagActive && context.mode === "ultra" ? ( ) : (
@@ -1057,11 +1078,39 @@ export function InputBox({ )} + +
+
+ + 多轮 RAG 问答 +
+
+ 多次检索、展示过程与参考资料，适合复杂问题 +
+
+ {multiRoundRagActive ? ( + + ) : ( +
+ )} + - {supportReasoningEffort && context.mode !== "flash" && ( + {!frontendRagActive && supportReasoningEffort && context.mode !== "flash" && (
@@ -1179,8 +1228,8 @@ export function InputBox({ )} - {knowledgeSelector} - {agentSelector && ( + {!frontendRagActive && agentSelector && ( + {!frontendRagActive && knowledgeSelector} )} - {promptPrefix && ( + {!frontendRagActive && promptPrefix && ( )} - @@ -1266,7 +1315,7 @@ export function InputBox({ ))} - + } (null); + const resolveArtifact = useArtifactResolver(threadId); + const handleCitationClick = useCallback( + (referenceId: string) => { + const referenceIndex = Number(referenceId); + if (!Number.isInteger(referenceIndex) || referenceIndex < 1) return; + const source = + displaySources.find((item) => item.index === referenceIndex) ?? + displaySources[referenceIndex - 1]; + if (source) setSelectedCitation(source); + }, + [displaySources], + ); + const components = useMemo( () => ({ img: (props: ImgHTMLAttributes) => ( ), - a: ({ href, ...props }: AnchorHTMLAttributes) => { - if (href?.startsWith('#reference-')) { - return ; - } + a: ({ href, ...props }: AnchorHTMLAttributes) => { + if (href?.startsWith('#reference-')) { + return ( + + ); + } if ( typeof props.children === 'string' && /^citation:/i.test(props.children) @@ -593,8 +643,14 @@ function MessageContent_({ return ; }, }), - [threadId, resolveArtifact], - ); + [ + handleCitationClick, + isMultiRoundRag, + isSingleRoundRag, + resolveArtifact, + threadId, + ], + ); // Question carries its own `client_ts` (round-trips through the checkpoint). // The AI answer has no timestamp of its own, so it inherits the turn's ask @@ -661,7 +717,7 @@ function MessageContent_({ } // Reasoning-only AI message (no main response content yet) - if (!isHuman && reasoningContent && !rawContent) { + if (!isHuman && reasoningContent && !rawContent && !multiRoundTrace) { return ( @@ -707,32 +763,284 @@ function MessageContent_({ ); } - return ( - - {filesList} -
- -
- - {!isLoading && contentToDisplay ? ( - - ) : null} - - ); -} - -function EmbeddedPageContextCard({ + return ( + <> + + {filesList} +
+ {multiRoundTrace ? ( + + ) : ( + + )} +
+ + {!isLoading && contentToDisplay ? ( + + ) : null} + + {(isSingleRoundRag || isMultiRoundRag) && ( + { + if (!open) setSelectedCitation(null); + }} + /> + )} + + ); +} + +function SingleRoundRagCitationDialog({ + source, + onOpenChange, +}: { + source: RuntimeDisplaySource | null; + onOpenChange: (open: boolean) => void; +}) { + const sourceUrl = resolveSingleRoundRagSourceUrl(source); + return ( + + + + + {source?.title || "引用资料"} + + + 来源：{source?.source || source?.type || "未知来源"} + {source?.time && 时间：{source.time}} + + + {sourceUrl && ( + + + 查看原文链接 + + )} +
+ {source?.content || source?.snippet || "暂无引用内容"} +
+ + + ); +} + +function MultiRoundRagContent({ + trace, + content, + displaySources, + isLoading, + rehypePlugins, + components, +}: { + trace: NonNullable>; + content: string; + displaySources: RuntimeDisplaySource[]; + isLoading: boolean; + rehypePlugins: MarkdownContentProps['rehypePlugins']; + components: MarkdownContentProps['components']; +}) { + const [open, setOpen] = useState(!trace.isComplete); + useEffect(() => { + if (!trace.isComplete) setOpen(true); + }, [trace.isComplete]); + + return ( +
+
+ + {open && ( +
+ {trace.rounds.map((round, index) => ( + + ))} +
+ )} +
+
+
最终回答
+ {content ? ( + + ) : ( +
+ + 正在生成最终回答… +
+ )} +
+
+ ); +} + +function MultiRoundRagRoundCard({ + index, + round, + isLoading, + rehypePlugins, + components, +}: { + index: number; + round: MultiRoundRagRound; + isLoading: boolean; + rehypePlugins: MarkdownContentProps['rehypePlugins']; + components: MarkdownContentProps['components']; +}) { + return ( +
+
+ + {index} + + {round.title} + {round.isSearching && } +
+ {round.thought && ( +
+
思考内容
+ +
+ )} + {round.body && ( +
+
本轮正文
+ +
+ )} + {round.query && ( +
+ + {round.query} +
+ )} + {round.retrievals.length > 0 && ( +
+
+ 检索内容 · {round.retrievals.length} 条 +
+
+ {round.retrievals.map((item, resultIndex) => ( +
+ + {resultIndex + 1}. + {item.title} + +
+ {item.content || '暂无资料摘要'} +
+ {(item.source || item.time) && ( +
+ {[item.source, item.time].filter(Boolean).join(' · ')} +
+ )} +
+ ))} +
+
+ )} +
+ ); +} + +function resolveSingleRoundRagSourceUrl( + source: RuntimeDisplaySource | null, +): string | undefined { + if (!source) return undefined; + if (source.url && /^https?:\/\//i.test(source.url)) return source.url; + + const raw = source.raw ?? {}; + const recordId = + source.id ?? + readString(raw.recUuid) ?? + readString(raw.rec_uuid) ?? + readString(raw.uuid); + const knowledgeUrl = readRuntimeEnv("VITE_AIPI_KNOWLEDGE_URL")?.replace(/\/$/, ""); + if (!knowledgeUrl || !recordId) return undefined; + return `${knowledgeUrl}/#/knowledge/mid-detail?recUuid=${encodeURIComponent(recordId)}`; +} + +function readString(value: unknown): string | undefined { + if (typeof value === "string" && value.trim()) return value; + if (typeof value === "number" && Number.isFinite(value)) return String(value); + return undefined; +} + +function EmbeddedPageContextCard({ pageContext, }: { pageContext: EmbeddedPageContext; diff --git a/frontend-web/src/components/workspace/messages/message-list.tsx b/frontend-web/src/components/workspace/messages/message-list.tsx index d96ce5fa..ee5a2147 100644 --- a/frontend-web/src/components/workspace/messages/message-list.tsx +++ b/frontend-web/src/components/workspace/messages/message-list.tsx @@ -35,6 +35,7 @@ import type { Subtask } from "@/core/tasks"; import { useUpdateSubtask } from "@/core/tasks/context"; import type { AgentThreadState } from "@/core/threads"; import { isSingleRoundRagMessage } from "@/core/threads/use-single-round-rag"; +import { isMultiRoundRagMessage } from "@/core/threads/use-multi-round-rag"; import { cn } from "@/lib/utils"; import { resolveAllowMultiple } from "@/roundtable-planning/lib/clarification"; @@ -239,7 +240,11 @@ export function MessageList({ const ragModeEnabled = settings.display.ragModeEnabled || isThreadRagModeLocked(threadId); const hasSingleRoundRag = useMemo( - () => messages.some(isSingleRoundRagMessage), + () => + messages.some( + (message) => + isSingleRoundRagMessage(message) || isMultiRoundRagMessage(message), + ), [messages], ); const referencesEnabled = ragModeEnabled || hasSingleRoundRag; diff --git a/frontend-web/src/core/models/single-round-rag.ts b/frontend-web/src/core/models/single-round-rag.ts index 7591a2da..99696517 100644 --- a/frontend-web/src/core/models/single-round-rag.ts +++ b/frontend-web/src/core/models/single-round-rag.ts @@ -19,3 +19,18 @@ export function isSingleRoundRagProfile( "single_round_rag" ); } + +export function isMultiRoundRagProfile( + context: unknown, +): boolean { + return ( + typeof context === "object" && + context !== null && + (context as { execution_profile?: unknown }).execution_profile === + "multi_round_rag" + ); +} + +export function isFrontendRagProfile(context: unknown): boolean { + return isSingleRoundRagProfile(context) || isMultiRoundRagProfile(context); +} diff --git a/frontend-web/src/core/models/types.ts b/frontend-web/src/core/models/types.ts index a470b2e8..47410d7d 100644 --- a/frontend-web/src/core/models/types.ts +++ b/frontend-web/src/core/models/types.ts @@ -7,7 +7,7 @@ export interface Model { supports_thinking?: boolean; supports_reasoning_effort?: boolean; /** Execution profiles handled by the frontend instead of DeerFlow. */ - execution_profile?: "single_round_rag"; + execution_profile?: "single_round_rag" | "multi_round_rag"; } export interface TokenUsageSettings { diff --git a/frontend-web/src/core/skill-display/inline-citations.test.ts b/frontend-web/src/core/skill-display/inline-citations.test.ts index db6bf1d2..0b4f3cc1 100644 --- a/frontend-web/src/core/skill-display/inline-citations.test.ts +++ b/frontend-web/src/core/skill-display/inline-citations.test.ts @@ -71,6 +71,12 @@ test("keeps half-width syntax working", () => { assert.equal(out.includes(citation(3)), true); }); +test("converts double-bracket RAG references", () => { + const out = applyInlineCitations("RAG answer [[1]] and [[2]].", makeSources(2)); + assert.equal(out.includes(citation(1)), true); + assert.equal(out.includes(citation(2)), true); +}); + test("accepts spaced and full-width square brackets", () => { const out = applyInlineCitations("参考[ 1 ]和［2］。", makeSources(2)); assert.equal(out.includes(citation(1)), true); diff --git a/frontend-web/src/core/threads/hooks.ts b/frontend-web/src/core/threads/hooks.ts index 5da690c8..9e616af9 100644 --- a/frontend-web/src/core/threads/hooks.ts +++ b/frontend-web/src/core/threads/hooks.ts @@ -244,6 +244,11 @@ const TERMINAL_RUN_STATUSES = new Set([ // for a while, then reconcile it from the authoritative run/state endpoints. const STREAM_RECONCILIATION_IDLE_MS = 12_000; const STREAM_RECONCILIATION_POLL_MS = 5_000; +// A run can legitimately be quiet while a long tool call is in progress, so +// reconnect more conservatively than the terminal-status check above. The +// reconnection only replaces the local SSE reader; the backend run keeps going. +const STREAM_RECONNECT_IDLE_MS = 30_000; +const STREAM_RECONNECT_MAX_ATTEMPTS = 3; async function fetchThreadTitleRefreshState( threadId: string, @@ -466,9 +471,15 @@ export function useThreadStream({ const startedRef = useRef(false); const lastStreamActivityAtRef = useRef(0); const streamReconciliationInFlightRef = useRef(false); + const streamReconnectAttemptsRef = useRef(0); + const streamReconnectToastShownRef = useRef(false); + const streamReconnectFailureToastShownRef = useRef(false); const markStreamActivity = useCallback(() => { lastStreamActivityAtRef.current = Date.now(); + streamReconnectAttemptsRef.current = 0; + streamReconnectToastShownRef.current = false; + streamReconnectFailureToastShownRef.current = false; }, []); const listeners = useRef({ @@ -761,13 +772,6 @@ export function useThreadStream({ threadStreamRef.current = thread; useEffect(() => { - if (!thread.isLoading) { - // A normal terminal event reached the browser. A later run may use the - // same thread, so it must be eligible for recovery again. - streamReconciliationInFlightRef.current = false; - return; - } - const activeThreadId = threadIdRef.current ?? onStreamThreadId ?? null; if (!activeThreadId || typeof window === "undefined") { return; @@ -776,7 +780,7 @@ export function useThreadStream({ let disposed = false; let refreshTimer: number | null = null; - const reconcileCompletedRun = async () => { + const reconcileRun = async () => { if ( disposed || streamReconciliationInFlightRef.current || @@ -793,7 +797,51 @@ export function useThreadStream({ try { const client = getAPIClient(isMock); const run = await client.runs.get(activeThreadId, runId); - if (disposed || !TERMINAL_RUN_STATUSES.has(run.status)) { + if (disposed) { + return; + } + + if (!TERMINAL_RUN_STATUSES.has(run.status)) { + // ``useStream`` retries one failed connection on its own. If that + // reader quietly stalls, however, its internal queue never gets a + // terminal error and the page remains frozen despite the backend + // continuing the run. Abort only the local reader, then join the + // same run so no work or tokens are lost server-side. + if (Date.now() - lastStreamActivityAtRef.current < STREAM_RECONNECT_IDLE_MS) { + return; + } + + if (streamReconnectAttemptsRef.current >= STREAM_RECONNECT_MAX_ATTEMPTS) { + if (!streamReconnectFailureToastShownRef.current) { + streamReconnectFailureToastShownRef.current = true; + toast.error("回答仍在后台处理中，但页面连接暂未恢复；请稍后返回此会话查看结果。"); + } + return; + } + + streamReconnectAttemptsRef.current += 1; + // Avoid enqueueing another reconnect before this new reader has had + // enough time to receive its first event. A genuine event resets + // the attempt counter through ``markStreamActivity``. + lastStreamActivityAtRef.current = Date.now(); + if (!streamReconnectToastShownRef.current) { + streamReconnectToastShownRef.current = true; + toast.info("回答连接出现波动，正在自动恢复…"); + } + + const stream = threadStreamRef.current; + void Promise.resolve(stream.stop()) + .catch(() => { + // Stopping a stale local reader is best-effort; joining is still + // useful if the SDK already considers the stream stopped. + }) + .finally(() => { + if (disposed || threadIdRef.current !== activeThreadId) return; + void stream.joinStream(runId).catch(() => { + // The status poll will either reconcile a terminal run or + // schedule the next bounded reconnect attempt. + }); + }); return; } @@ -801,13 +849,17 @@ export function useThreadStream({ // receive any more useful events. Load the checkpoint directly before // resetting the SDK stream; this preserves the normal onFinish side // effects (e.g. marking a position delivery complete) even when the - // browser missed the terminal event. + // browser missed the terminal event or already reported a transport + // error and dropped ``isLoading``. const finalState = await client.threads.getState(activeThreadId); - if (disposed || !threadStreamRef.current.isLoading) { + if (disposed) { return; } storage?.removeItem(`lg:stream:${activeThreadId}`); + streamReconnectAttemptsRef.current = 0; + streamReconnectToastShownRef.current = false; + streamReconnectFailureToastShownRef.current = false; if (run.status === "success") { listeners.current.onFinish?.(finalState.values as AgentThreadState); void queryClient.invalidateQueries({ queryKey: ["threads", "search"] }); @@ -836,7 +888,7 @@ export function useThreadStream({ }; const interval = window.setInterval( - () => void reconcileCompletedRun(), + () => void reconcileRun(), STREAM_RECONCILIATION_POLL_MS, ); diff --git a/frontend-web/src/core/threads/types.ts b/frontend-web/src/core/threads/types.ts index 49dc31ac..9639a63e 100644 --- a/frontend-web/src/core/threads/types.ts +++ b/frontend-web/src/core/threads/types.ts @@ -27,8 +27,8 @@ export interface AgentThreadContext extends Record { knowledge_rag_enabled?: boolean; /** DeerFlow LLMWiki mapping ids selected for this conversation. */ llmwiki_knowledge_base_ids?: string[]; - /** Direct knowledge-base RAG mode; this request never enters Lead Agent. */ - execution_profile?: "single_round_rag"; + /** Direct RAG modes handled in the frontend; these requests never enter Lead Agent. */ + execution_profile?: "single_round_rag" | "multi_round_rag"; agent_id?: string; /** @deprecated kept for reading legacy thread metadata; prefer agent_id */ agent_name?: string; diff --git a/frontend-web/src/core/threads/use-multi-round-rag.ts b/frontend-web/src/core/threads/use-multi-round-rag.ts new file mode 100644 index 00000000..ee8ec224 --- /dev/null +++ b/frontend-web/src/core/threads/use-multi-round-rag.ts @@ -0,0 +1,483 @@ +import type { Message } from "@langchain/langgraph-sdk"; +import { useCallback, useEffect, useRef, useState } from "react"; + +import { readRuntimeEnv } from "@/env"; + +const STORAGE_PREFIX = "workspace.multi-round-rag."; + +export type MultiRoundRagRetrieval = { + id?: string; + title: string; + content: string; + source?: string; + time?: string; + recUuid?: string; + url?: string; +}; + +export type MultiRoundRagRound = { + id: string; + title: string; + thought: string; + /** Model text emitted for this round, before its retrieval results. */ + body: string; + query: string; + isSearching: boolean; + retrievals: MultiRoundRagRetrieval[]; +}; + +export type MultiRoundRagTrace = { + rounds: MultiRoundRagRound[]; + answer: string; + isComplete: boolean; +}; + +type MultiRoundRagSource = { + id?: string; + title: string; + type: string; + snippet: string; + content: string; + source?: string; + time?: string; + url?: string; + raw: Record; +}; + +type MultiRoundEvent = { + event?: unknown; + data?: unknown; +}; + +function storageKey(threadId: string) { + return `${STORAGE_PREFIX}${threadId}`; +} + +function readStoredMessages(threadId: string): Message[] { + if (typeof window === "undefined" || !threadId) return []; + try { + const raw = window.sessionStorage.getItem(storageKey(threadId)); + const parsed: unknown = raw ? JSON.parse(raw) : []; + return Array.isArray(parsed) ? (parsed as Message[]) : []; + } catch { + return []; + } +} + +function writeStoredMessages(threadId: string, messages: Message[]) { + if (typeof window === "undefined" || !threadId) return; + try { + window.sessionStorage.setItem(storageKey(threadId), JSON.stringify(messages)); + } catch { + // Storage quota failures must not interrupt a running answer. + } +} + +function createId(prefix: string) { + return `${prefix}-${crypto.randomUUID()}`; +} + +function isAbortError(error: unknown) { + return error instanceof DOMException && error.name === "AbortError"; +} + +function asRecord(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : null; +} + +function asString(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function parseJson(value: unknown): unknown { + if (typeof value !== "string") return value; + try { + return JSON.parse(value) as unknown; + } catch { + return value; + } +} + +function parseEventLine(line: string): MultiRoundEvent | null { + const trimmed = line.trim(); + if (!trimmed.startsWith("data:")) return null; + try { + return JSON.parse(trimmed.slice("data:".length).trim()) as MultiRoundEvent; + } catch { + return null; + } +} + +function initialTrace(): MultiRoundRagTrace { + return { + rounds: [ + { + id: createId("multi-rag-round"), + title: "正在准备检索", + thought: "", + body: "", + query: "", + isSearching: true, + retrievals: [], + }, + ], + answer: "", + isComplete: false, + }; +} + +function latestRound(trace: MultiRoundRagTrace): MultiRoundRagRound { + const current = trace.rounds.at(-1); + if (current) return current; + const next = initialTrace().rounds[0]!; + trace.rounds.push(next); + return next; +} + +function addThinkingRound(trace: MultiRoundRagTrace, thought: string) { + const current = latestRound(trace); + if (current.title === "正在准备检索" && !current.thought && !current.query) { + current.title = "思考过程"; + current.thought = thought; + return; + } + current.isSearching = false; + trace.rounds.push({ + id: createId("multi-rag-round"), + title: "思考过程", + thought, + body: "", + query: "", + isSearching: true, + retrievals: [], + }); +} + +function appendRoundBody(trace: MultiRoundRagTrace, body: string) { + if (!body) return; + const current = latestRound(trace); + // Stored traces created before `body` was introduced remain readable. + current.body = `${current.body ?? ""}${body}`; +} + +function mapRetrievals(value: unknown): MultiRoundRagRetrieval[] { + const root = asRecord(parseJson(value)); + const list = + (Array.isArray(root?.results) && root.results) || + (Array.isArray(root?.organic) && root.organic) || + []; + return list.flatMap((item, index) => { + const record = asRecord(item); + if (!record) return []; + const recUuid = asString(record.recUuid ?? record.rec_uuid); + const url = asString(record.url ?? record.link ?? record.href); + return [ + { + id: asString(record.id) || recUuid || undefined, + title: asString(record.title) || `检索资料 ${index + 1}`, + content: asString(record.content ?? record.snippet ?? record.text), + source: asString(record.source ?? record.source1 ?? url) || undefined, + time: asString(record.publish_time ?? record.m_publish ?? record.date) || undefined, + recUuid: recUuid || undefined, + url: /^https?:\/\//i.test(url) ? url : undefined, + }, + ]; + }); +} + +function mapReferences(value: unknown): MultiRoundRagSource[] { + const record = asRecord(value); + const list = Array.isArray(record?.references) ? record.references : []; + return list.flatMap((item, index) => { + const source = asRecord(item); + if (!source) return []; + const content = asString(source.content ?? source.snippet); + const rawUrl = asString(source.url ?? source.link ?? source.href); + const recUuid = asString(source.recUuid ?? source.rec_uuid ?? source.uuid); + return [ + { + id: asString(source.id) || recUuid || undefined, + title: asString(source.title) || `参考资料 ${index + 1}`, + type: asString(source.type) || "参考资料", + snippet: asString(source.snippet) || content, + content, + source: asString(source.source ?? source.source1) || undefined, + time: asString(source.time ?? source.m_publish ?? source.publish_time) || undefined, + url: /^https?:\/\//i.test(rawUrl) ? rawUrl : undefined, + raw: source, + }, + ]; + }); +} + +function cloneTrace(trace: MultiRoundRagTrace): MultiRoundRagTrace { + return { + ...trace, + rounds: trace.rounds.map((round) => ({ + ...round, + retrievals: [...round.retrievals], + })), + }; +} + +function isSingleRoundRagMessage(message: Message) { + return message.additional_kwargs?.single_round_rag === true; +} + +export function isMultiRoundRagMessage(message: Message): boolean { + return message.additional_kwargs?.multi_round_rag === true; +} + +export function getMultiRoundRagTrace( + message: Message, +): MultiRoundRagTrace | null { + const trace = message.additional_kwargs?.multi_round_rag_trace; + return asRecord(trace) + ? (trace as unknown as MultiRoundRagTrace) + : null; +} + +/** + * Frontend adapter for GeneralQA's multi-round endpoint. It keeps the external + * SSE protocol outside DeerFlow and stores only its UI messages in sessionStorage. + */ +export function useMultiRoundRagMessages(threadId: string) { + const [messages, setMessages] = useState(() => readStoredMessages(threadId)); + const messagesRef = useRef(messages); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(null); + const controllerRef = useRef(null); + + useEffect(() => { + const stored = readStoredMessages(threadId); + messagesRef.current = stored; + setMessages(stored); + setError(null); + }, [threadId]); + + useEffect(() => { + messagesRef.current = messages; + }, [messages]); + + useEffect(() => () => controllerRef.current?.abort(), []); + + const updateMessages = useCallback( + (targetThreadId: string, updater: (current: Message[]) => Message[]) => { + setMessages((current) => { + const next = updater(current); + messagesRef.current = next; + writeStoredMessages(targetThreadId, next); + return next; + }); + }, + [], + ); + + const stop = useCallback(() => controllerRef.current?.abort(), []); + + const submit = useCallback( + async ({ targetThreadId, query }: { targetThreadId: string; query: string }) => { + if (isLoading) return; + const question = query.trim(); + if (!question) return; + + const humanId = createId("multi-rag-human"); + const assistantId = createId("multi-rag-ai"); + const sourceId = createId("multi-rag-sources"); + const trace = initialTrace(); + let sources: MultiRoundRagSource[] = []; + const clientTs = new Date().toISOString(); + const userMessage: Message = { + id: humanId, + type: "human", + content: [{ type: "text", text: question }], + additional_kwargs: { client_ts: clientTs, multi_round_rag: true }, + }; + const assistantMessage: Message = { + id: assistantId, + type: "ai", + content: "", + additional_kwargs: { + multi_round_rag: true, + multi_round_rag_trace: trace, + }, + }; + updateMessages(targetThreadId, (current) => [ + ...current, + userMessage, + assistantMessage, + ]); + setError(null); + setIsLoading(true); + + const controller = new AbortController(); + controllerRef.current = controller; + const refreshAssistant = () => { + const traceSnapshot = cloneTrace(trace); + updateMessages(targetThreadId, (current) => { + const next = current.filter((message) => message.id !== sourceId); + const assistantIndex = next.findIndex((message) => message.id === assistantId); + if (assistantIndex < 0) return current; + next[assistantIndex] = { + ...next[assistantIndex], + content: traceSnapshot.answer, + additional_kwargs: { + ...next[assistantIndex]!.additional_kwargs, + multi_round_rag: true, + multi_round_rag_trace: traceSnapshot, + reasoning_content: traceSnapshot.rounds + .map((round) => round.thought) + .filter(Boolean) + .join("\n\n") || null, + }, + } as Message; + if (sources.length > 0) { + next.splice(assistantIndex, 0, { + id: sourceId, + type: "tool", + name: "multi_round_rag_references", + content: { results: sources }, + additional_kwargs: { + hide_from_ui: true, + multi_round_rag: true, + }, + } as unknown as Message); + } + return next; + }); + }; + + try { + const baseUrl = readRuntimeEnv("VITE_AIPI_GENERATE_CHAT_URL") || ""; + if (!baseUrl) { + throw new Error("未配置 VITE_AIPI_GENERATE_CHAT_URL，无法发起多轮 RAG 问答。"); + } + const response = await fetch(`${baseUrl.replace(/\/$/, "")}/generate`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + signal: controller.signal, + body: JSON.stringify({ prompt: question }), + }); + if (!response.ok) { + throw new Error(`多轮 RAG 服务请求失败（${response.status}）。`); + } + if (!response.body) { + throw new Error("多轮 RAG 服务未返回流式响应。"); + } + + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() ?? ""; + for (const line of lines) { + const event = parseEventLine(line); + if (!event) continue; + const eventType = asString(event.event); + const data = event.data; + if (eventType === "tool_call") { + const tool = asRecord(data); + const toolName = asString(tool?.tool_name); + const input = asRecord(tool?.tool_input) ?? {}; + const current = latestRound(trace); + if (toolName === "show_text") { + const text = asString(input.text); + if (text.includes("")) { + const thinkEnd = text.indexOf(""); + addThinkingRound( + trace, + (thinkEnd >= 0 + ? text.slice(0, thinkEnd + "".length) + : text + ) + .replace(/<\/?think>/g, "") + .trim(), + ); + if (thinkEnd >= 0) { + const roundBody = text + .slice(thinkEnd + "".length) + .trim(); + if (roundBody) { + appendRoundBody(trace, roundBody); + } + } + } else { + // A non-thinking `show_text` immediately after a thought is + // this round's narrative; the terminal `message` event is + // still rendered only as the final answer below all rounds. + current.isSearching = false; + trace.answer += text; + trace.isComplete = true; + } + } else if ( + toolName === "local_search" || + toolName === "tavily_search" || + toolName === "google_search" + ) { + const queryData = asRecord(parseJson(input.result)); + current.title = + toolName === "local_search" ? "本地知识检索" : "联网检索"; + current.query = + asString(input.q ?? input.query) || + asString(queryData?.query) || + current.query; + const retrievals = mapRetrievals(input.result); + if (retrievals.length > 0) current.retrievals = retrievals; + current.isSearching = false; + } else if (toolName === "get_textid_content") { + const article = asRecord(parseJson(input.result)); + const body = + asString(article?.content ?? article?.text) || + asString(input.content); + if (body) appendRoundBody(trace, body); + current.isSearching = false; + } + refreshAssistant(); + } else if (eventType === "message" && typeof data === "string") { + latestRound(trace).isSearching = false; + trace.answer += data; + trace.isComplete = true; + refreshAssistant(); + } else if (eventType === "references") { + sources = mapReferences(data); + refreshAssistant(); + } else if (eventType === "done") { + trace.isComplete = true; + for (const round of trace.rounds) round.isSearching = false; + refreshAssistant(); + } + } + } + trace.isComplete = true; + for (const round of trace.rounds) round.isSearching = false; + refreshAssistant(); + } catch (caught) { + trace.isComplete = true; + for (const round of trace.rounds) round.isSearching = false; + if (!isAbortError(caught)) { + const message = + caught instanceof Error ? caught.message : "多轮 RAG 问答失败。"; + setError(message); + if (!trace.answer) trace.answer = message; + } + refreshAssistant(); + } finally { + if (controllerRef.current === controller) controllerRef.current = null; + setIsLoading(false); + } + }, + [isLoading, updateMessages], + ); + + return { messages, isLoading, error, submit, stop }; +} + +export function isFrontendRagMessage(message: Message): boolean { + return isSingleRoundRagMessage(message) || isMultiRoundRagMessage(message); +} diff --git a/frontend-web/src/core/threads/use-single-round-rag.ts b/frontend-web/src/core/threads/use-single-round-rag.ts index 7287a315..e90d5fc8 100644 --- a/frontend-web/src/core/threads/use-single-round-rag.ts +++ b/frontend-web/src/core/threads/use-single-round-rag.ts @@ -22,6 +22,13 @@ type RagSource = { raw: Record; }; +type RagHistoryEntry = { + role: "user" | "assistant"; + content: string; +}; + +const MAX_HISTORY_MESSAGES = 12; + /** Same JSON-object framing used by the established GeneralQA single-RAG flow. */ function extractJsonObjects(raw: string): { objects: string[]; rest: string } { const objects: string[] = []; @@ -199,6 +206,44 @@ function isAbortError(error: unknown) { return error instanceof DOMException && error.name === "AbortError"; } +function textFromMessage(message: Message): string { + if (typeof message.content === "string") { + return message.content.trim(); + } + if (!Array.isArray(message.content)) { + return ""; + } + return message.content + .map((part) => { + if (typeof part === "string") return part; + if (!part || typeof part !== "object") return ""; + const text = (part as { text?: unknown }).text; + return typeof text === "string" ? text : ""; + }) + .join("\n") + .trim(); +} + +/** + * The general-QA endpoint accepts the usual `{ role, content }` history + * format. Keep it scoped to completed single-round-RAG turns so a normal + * DeerFlow conversation never leaks into this external endpoint. + */ +function buildRagHistory(messages: Message[]): RagHistoryEntry[] { + return messages + .filter( + (message) => + message.additional_kwargs?.single_round_rag === true && + (message.type === "human" || message.type === "ai"), + ) + .map((message): RagHistoryEntry => ({ + role: message.type === "human" ? "user" : "assistant", + content: textFromMessage(message), + })) + .filter((entry) => entry.content.length > 0) + .slice(-MAX_HISTORY_MESSAGES); +} + export function isSingleRoundRagMessage(message: Message): boolean { return message.additional_kwargs?.single_round_rag === true; } @@ -209,21 +254,29 @@ export function isSingleRoundRagMessage(message: Message): boolean { */ export function useSingleRoundRagMessages(threadId: string) { const [messages, setMessages] = useState(() => readStoredMessages(threadId)); + const messagesRef = useRef(messages); const [isLoading, setIsLoading] = useState(false); const [error, setError] = useState(null); const controllerRef = useRef(null); useEffect(() => { - setMessages(readStoredMessages(threadId)); + const stored = readStoredMessages(threadId); + messagesRef.current = stored; + setMessages(stored); setError(null); }, [threadId]); + useEffect(() => { + messagesRef.current = messages; + }, [messages]); + useEffect(() => () => controllerRef.current?.abort(), []); const updateMessages = useCallback( (targetThreadId: string, updater: (current: Message[]) => Message[]) => { setMessages((current) => { const next = updater(current); + messagesRef.current = next; writeStoredMessages(targetThreadId, next); return next; }); @@ -251,6 +304,7 @@ export function useSingleRoundRagMessages(threadId: string) { const assistantId = createId("single-rag-ai"); const sourceId = createId("single-rag-sources"); const clientTs = new Date().toISOString(); + const history = buildRagHistory(messagesRef.current); const userMessage: Message = { id: humanId, type: "human", @@ -319,7 +373,7 @@ export function useSingleRoundRagMessages(threadId: string) { knowledge_base_name: [], top_k: 5, score_threshold: 0.5, - history: [], + history, stream: true, model_name: modelName, temperature: 0.3, diff --git a/frontend-web/src/pages/ChatPage.tsx b/frontend-web/src/pages/ChatPage.tsx index 9d136ad0..ee708752 100644 --- a/frontend-web/src/pages/ChatPage.tsx +++ b/frontend-web/src/pages/ChatPage.tsx @@ -46,6 +46,8 @@ import { useAgentSelector, useAgentsPaginated } from "@/core/agents"; import { useI18n } from "@/core/i18n/hooks"; import { useModels } from "@/core/models/hooks"; import { + isFrontendRagProfile, + isMultiRoundRagProfile, isSingleRoundRagProfile, SINGLE_ROUND_RAG_MODELS, } from "@/core/models/single-round-rag"; @@ -59,6 +61,7 @@ import { import { useThreadStream } from "@/core/threads/hooks"; import { queueLlmWikiConversationDeposit } from "@/core/llmwiki/conversation-deposit"; import { useSingleRoundRagMessages } from "@/core/threads/use-single-round-rag"; +import { useMultiRoundRagMessages } from "@/core/threads/use-multi-round-rag"; import { useLayoutMode } from "@/core/system-settings/hooks"; import { useRouteBase } from "@/core/route-base"; import { textOfMessage } from "@/core/threads/utils"; @@ -310,19 +313,38 @@ export default function ChatPage() { }, }); const singleRoundRag = useSingleRoundRagMessages(threadId); + const multiRoundRag = useMultiRoundRagMessages(threadId); const singleRoundRagSelected = isSingleRoundRagProfile(settings.context); + const multiRoundRagSelected = isMultiRoundRagProfile(settings.context); + const frontendRagSelected = isFrontendRagProfile(settings.context); const displayThread = useMemo(() => { - if (singleRoundRag.messages.length === 0 && !singleRoundRag.isLoading) { + if ( + singleRoundRag.messages.length === 0 && + multiRoundRag.messages.length === 0 && + !singleRoundRag.isLoading && + !multiRoundRag.isLoading + ) { return thread; } - const messages = [...thread.messages, ...singleRoundRag.messages]; + const messages = [ + ...thread.messages, + ...singleRoundRag.messages, + ...multiRoundRag.messages, + ]; return { ...thread, values: { ...thread.values, messages }, messages, - isLoading: thread.isLoading || singleRoundRag.isLoading, + isLoading: + thread.isLoading || singleRoundRag.isLoading || multiRoundRag.isLoading, } as typeof thread; - }, [singleRoundRag.isLoading, singleRoundRag.messages, thread]); + }, [ + multiRoundRag.isLoading, + multiRoundRag.messages, + singleRoundRag.isLoading, + singleRoundRag.messages, + thread, + ]); const writingArtifactPath = useMemo(() => { const isMarkdown = (path: string | null | undefined) => @@ -337,9 +359,9 @@ export default function ChatPage() { const handleSubmit = useCallback( (message: PromptInputMessage) => { - if (singleRoundRagSelected) { + if (frontendRagSelected) { if (message.files && message.files.length > 0) { - toast.error("单轮 RAG 问答暂不支持附件，请移除附件后重试。"); + toast.error("RAG 问答暂不支持附件，请移除附件后重试。"); return; } setHasStartedConversation(true); @@ -366,13 +388,23 @@ export default function ChatPage() { .catch(() => undefined); } } - await singleRoundRag.submit({ - targetThreadId, - query: message.text, - modelName: settings.context.model_name ?? "OpenAI3", - }); + if (multiRoundRagSelected) { + await multiRoundRag.submit({ targetThreadId, query: message.text }); + } else { + await singleRoundRag.submit({ + targetThreadId, + query: message.text, + modelName: settings.context.model_name ?? "OpenAI3", + }); + } } catch (error) { - toast.error(error instanceof Error ? error.message : "单轮 RAG 问答失败。"); + toast.error( + error instanceof Error + ? error.message + : multiRoundRagSelected + ? "多轮 RAG 问答失败。" + : "单轮 RAG 问答失败。", + ); } })(); return; @@ -483,6 +515,9 @@ export default function ChatPage() { writingArtifactPath, singleRoundRag, singleRoundRagSelected, + multiRoundRag, + multiRoundRagSelected, + frontendRagSelected, routeBase, setIsNewThread, setThreadId, @@ -580,8 +615,12 @@ export default function ChatPage() { singleRoundRag.stop(); return; } + if (multiRoundRag.isLoading) { + multiRoundRag.stop(); + return; + } await thread.stop(); - }, [singleRoundRag, thread]); + }, [multiRoundRag, singleRoundRag, thread]); const promptPrefixProp = useMemo(() => { if (!promptPrefixSettings?.enabled) return undefined; @@ -776,13 +815,18 @@ export default function ChatPage() { : "ready" } context={settings.context} - extraModels={SINGLE_ROUND_RAG_MODELS} + extraModels={ + singleRoundRagSelected + ? SINGLE_ROUND_RAG_MODELS + : undefined + } disabled={ env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" || isUploading || agentHandoffPending || knowledgeSelectionLoading || - singleRoundRag.isLoading + singleRoundRag.isLoading || + multiRoundRag.isLoading } onContextChange={(context) => setSettings("context", context) diff --git a/frontend-web/src/position-roundtable/api/sessions.ts b/frontend-web/src/position-roundtable/api/sessions.ts index e9e97497..04af1c2f 100644 --- a/frontend-web/src/position-roundtable/api/sessions.ts +++ b/frontend-web/src/position-roundtable/api/sessions.ts @@ -138,6 +138,52 @@ const PRIMARY_API_PATH = "/api/position-roundtable"; const COMPAT_API_PATH = "/api/multi-agent/position-roundtable"; const BASE = () => `${getBackendBaseURL()}${PRIMARY_API_PATH}`; +/** + * 生成一次写操作的幂等键（command_id）。 + * + * 后端命令式状态机用 ``UNIQUE(session_id, node_key, command_id)`` 保证同一 command_id + * 只执行一次。把它在**一次调用入口**生成并固定进请求体，使得 `positionApiFetch` 的 + * 主/兼容路由回退重试复用同一 command_id —— 网络层重发绝不会把同一操作执行两遍 + * （不会重复 +revision / 重复解锁下游 / 重复驳回）。每次新的用户操作则取新键。 + */ +function newCommandId(): string { + if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") { + return crypto.randomUUID().replace(/-/g, ""); + } + return `${Date.now().toString(36)}${Math.random().toString(36).slice(2, 12)}`; +} + +/** + * 用**同一个 command_id** 重试一个写操作至多 ``attempts`` 次（指数退避）。 + * + * 后端命令幂等表保证同 command_id + 同参数只执行一次：即便第一次其实已提交成功、 + * 只是响应在网络层丢失，重试也只会重放上次结果，而不会重复推进状态机（重复 + * +revision / 重复解锁下游）。因此「提交产物给下一阶段」这类关键写操作可以放心 + * 重试，把偶发网络错误/超时变成自愈，而不是让下一阶段永远拿不到上游产物。 + * + * 返回 ``{ ok, result?, error? }``；``ok=false`` 时 ``error`` 是最后一次失败。 + */ +export async function retryWithSameCommandId( + operation: (commandId: string) => Promise, + options?: { attempts?: number; baseDelayMs?: number }, +): Promise<{ ok: true; result: T } | { ok: false; error: unknown }> { + const attempts = Math.max(1, options?.attempts ?? 3); + const baseDelay = options?.baseDelayMs ?? 400; + const commandId = newCommandId(); + let lastError: unknown = null; + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { + return { ok: true, result: await operation(commandId) }; + } catch (error) { + lastError = error; + if (attempt < attempts - 1) { + await new Promise((resolve) => setTimeout(resolve, baseDelay * 2 ** attempt)); + } + } + } + return { ok: false, error: lastError }; +} + /** * Some intranet gateways only forward the long-established `/api/multi-agent` * namespace. The position workspace was added later under its own top-level @@ -383,7 +429,7 @@ export async function activatePositionSession( await positionApiFetch(`${BASE()}/sessions/${encodeURIComponent(sessionId)}/activate`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ chain_id: chainId }), + body: JSON.stringify({ chain_id: chainId, command_id: newCommandId() }), }), "activate position session", ); @@ -417,6 +463,7 @@ export async function bindPositionNodeThread( sessionId: string, nodeKey: string, threadId: string, + commandId?: string, ): Promise { const result = await readJson( await positionApiFetch( @@ -424,7 +471,7 @@ export async function bindPositionNodeThread( { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ thread_id: threadId }), + body: JSON.stringify({ thread_id: threadId, command_id: commandId ?? newCommandId() }), }, ), "bind position node thread", @@ -438,6 +485,7 @@ export async function savePositionNodeConversation( input: { threadId?: string | null; conversationSnapshot: PositionConversationSnapshot; + commandId?: string; }, ): Promise { const result = await readJson( @@ -449,6 +497,7 @@ export async function savePositionNodeConversation( body: JSON.stringify({ thread_id: input.threadId || null, conversation_snapshot: input.conversationSnapshot, + command_id: input.commandId ?? newCommandId(), }), }, ), @@ -533,6 +582,8 @@ export async function completePositionNodeTurn( latestAnswer: string; artifactManifest?: Array>; upstreamRevision?: Record | null; + /** Retry-stable idempotency key; omit to auto-generate per call. */ + commandId?: string; }, ): Promise { // 后端依据 artifactManifest 判断是否有 Markdown 交付物，并在需要时解锁下游。 @@ -546,6 +597,7 @@ export async function completePositionNodeTurn( latest_answer: input.latestAnswer, artifact_manifest: input.artifactManifest ?? [], upstream_revision: input.upstreamRevision ?? null, + command_id: input.commandId ?? newCommandId(), }), }, ), @@ -557,7 +609,7 @@ export async function completePositionNodeTurn( export async function rejectPositionNode( sessionId: string, nodeKey: string, - input: { reason?: string; title?: string }, + input: { reason?: string; title?: string; commandId?: string }, ): Promise { const result = await readJson( await positionApiFetch( @@ -570,6 +622,7 @@ export async function rejectPositionNode( // workspace deliberately sends only an optional reason. ...(input.title ? { title: input.title } : {}), reason: input.reason ?? "", + command_id: input.commandId ?? newCommandId(), }), }, ), diff --git a/frontend-web/src/position-roundtable/components/PositionNodeChat.tsx b/frontend-web/src/position-roundtable/components/PositionNodeChat.tsx index f566fbb1..b2d63d94 100644 --- a/frontend-web/src/position-roundtable/components/PositionNodeChat.tsx +++ b/frontend-web/src/position-roundtable/components/PositionNodeChat.tsx @@ -27,6 +27,7 @@ import { bindPositionNodeThread, completePositionNodeTurn, getPositionNodeContext, + retryWithSameCommandId, savePositionNodeConversation, type PositionRoundtableNode, } from "@/position-roundtable/api/sessions"; @@ -530,17 +531,26 @@ export function PositionNodeChat({ // Artifact listing is best-effort: a valid text answer must still be // persisted and unlock downstream work even if files cannot be read. } - try { - const updated = await completePositionNodeTurn(sessionId, node.nodeKey, { + // 这是「岗位完成问答 → 把产物提交给下一阶段」的关键路径。偶发网络错误/超时 + // 不能让它悄悄失败（否则下一阶段永远拿不到上游产物）：用同一个 command_id + // 重试，后端命令幂等表保证即使第一次其实已提交成功，重试也只重放结果， + // 绝不重复推进状态机（不会重复 +revision / 重复解锁下游）。 + const delivery = await retryWithSameCommandId((commandId) => + completePositionNodeTurn(sessionId, node.nodeKey, { latestAnswer, artifactManifest, upstreamRevision: upstreamRevisionRef.current, - }); + commandId, + }), + ); + if (delivery.ok) { + const updated = delivery.result; onNodeUpdate(updated); if (updated.status !== "done" && !hasMarkdownArtifact(artifactManifest)) { toast.info("已保存本轮回答；该席位需生成并交付一份 Markdown 报告文档后才会标记完成。"); } - } catch (error) { + } else { + const error = delivery.error; toast.error(`节点结果保存失败：${error instanceof Error ? error.message : String(error)}`); } })(); @@ -665,14 +675,19 @@ export function PositionNodeChat({ [messageId]: textOfMessage(updatedMessage) ?? content, })); if (updatedMessage.type === "ai" && messageId === latestAiMessageId) { - try { - const updatedNode = await completePositionNodeTurn(sessionId, node.nodeKey, { + // 与自动交付路径同样：同一 command_id 重试，幂等保证不重复推进状态机。 + const sync = await retryWithSameCommandId((commandId) => + completePositionNodeTurn(sessionId, node.nodeKey, { latestAnswer: content, artifactManifest: node.artifactManifest, upstreamRevision: node.upstreamRevision, - }); - onNodeUpdate(updatedNode); - } catch (syncError) { + commandId, + }), + ); + if (sync.ok) { + onNodeUpdate(sync.result); + } else { + const syncError = sync.error; toast.warning( `消息已保存，但节点交接摘要同步失败：${ syncError instanceof Error ? syncError.message : String(syncError) diff --git a/frontend-web/src/roundtable-planning/api/drafts.ts b/frontend-web/src/roundtable-planning/api/drafts.ts index 920e6ff6..38074d8d 100644 --- a/frontend-web/src/roundtable-planning/api/drafts.ts +++ b/frontend-web/src/roundtable-planning/api/drafts.ts @@ -189,6 +189,11 @@ export interface DraftMeta { /** Auto-derived task summary (objective → first user message → timestamp). */ title: string; furthestStep: 1 | 2 | 3; + /** + * 乐观锁版本号：写回时经 `expectedVersion` 回传；并发冲突（409）时用它识别 + * 本地副本已过期，重拉最新草稿合并后重试。 + */ + version: number; /** * 本次会商所用业务链名（历史卡片展示）。无业务链（自由/recommend 模式）→ null/undefined， * 由前端展示为「自由模式」。后端从 step2 快照里抽取（见 roundtable_drafts/sql.py）。 @@ -234,6 +239,33 @@ export interface DraftUpdatePayload { step1?: DraftStep1Snapshot; step2?: DraftStep2Snapshot; step3?: DraftStep3Snapshot | null; + /** + * 本地副本基于的草稿版本号（乐观锁）。提供时服务端做原子条件 UPDATE； + * 版本不匹配 → 409（`DraftVersionConflictError`），调用方应重拉合并后重试。 + * 省略 = 盲写（向后兼容旧调用点，不推荐）。 + */ + expectedVersion?: number; +} + +/** + * 乐观锁冲突（HTTP 409）：服务端草稿版本比本地副本新。 + * `currentVersion` 是服务端最新版本，调用方应重拉最新草稿、合并本地改动后用 + * 新版本号重试——一致性来自 DB，不靠前端运气。 + */ +export class DraftVersionConflictError extends Error { + readonly draftId: string; + readonly currentVersion: number | null; + + constructor(draftId: string, currentVersion: number | null) { + super(`Draft ${draftId} was modified concurrently (server version ${currentVersion})`); + this.name = "DraftVersionConflictError"; + this.draftId = draftId; + this.currentVersion = currentVersion; + } +} + +export function isDraftVersionConflictError(error: unknown): error is DraftVersionConflictError { + return error instanceof DraftVersionConflictError; } const BASE = () => `${getBackendBaseURL()}/api/roundtable-drafts`; @@ -256,6 +288,7 @@ export interface ServerDraftMeta { task_id?: string | null; title: string; furthest_step: number; + version?: number; chain_title?: string | null; created_at: string | null; updated_at: string | null; @@ -276,6 +309,7 @@ export function toMeta(s: ServerDraftMeta): DraftMeta { taskId: s.task_id ?? null, title: s.title ?? "", furthestStep: step, + version: s.version ?? 0, chainTitle: s.chain_title ?? null, createdAt: s.created_at ?? "", updatedAt: s.updated_at ?? "", @@ -313,6 +347,7 @@ export function fromUpdate(p: DraftUpdatePayload) { if (p.step1 !== undefined) out.step1 = p.step1; if (p.step2 !== undefined) out.step2 = p.step2; if (p.step3 !== undefined) out.step3 = p.step3; + if (p.expectedVersion !== undefined) out.expected_version = p.expectedVersion; return out; } @@ -366,6 +401,23 @@ export async function createDraft(payload: DraftCreatePayload): Promise { + if (res.status !== 409) return; + const raw = await res.text().catch(() => ""); + let currentVersion: number | null = null; + try { + const parsed = JSON.parse(raw) as { detail?: { current_version?: number } }; + currentVersion = parsed?.detail?.current_version ?? null; + } catch { + currentVersion = null; + } + throw new DraftVersionConflictError(id, currentVersion); +} + export async function updateDraft(id: string, patch: DraftUpdatePayload): Promise { const res = await apiFetch(`${BASE()}/${encodeURIComponent(id)}`, { method: "PUT", @@ -373,6 +425,7 @@ export async function updateDraft(id: string, patch: DraftUpdatePayload): Promis body: JSON.stringify(fromUpdate(patch)), }); if (res.status === 404) return null; + await throwIfDraftConflict(res, id); const data = await readJson(res, "update draft"); const record = toRecord(data); writeYLog({ diff --git a/frontend-web/src/roundtable-planning/api/intent.ts b/frontend-web/src/roundtable-planning/api/intent.ts index f563a21d..6752fb13 100644 --- a/frontend-web/src/roundtable-planning/api/intent.ts +++ b/frontend-web/src/roundtable-planning/api/intent.ts @@ -190,6 +190,12 @@ export interface StreamIntentRequest { model?: string; /** Explicit built-in Step-1 agent. Omit to retain the multi-agent default. */ intentAgentId?: string; + /** + * 本回合幂等键（Phase 4 单飞）：同一 (threadId, clientTurnId) 在后端至多一个 run。 + * 缺省由调用方（useStep1Intent）每条消息生成；同一回合的重试 / 双击复用它，后端据此 + * 去重（在跑→丢弃重复请求 / 已完成→幂等回放），杜绝并发下双发意图 / 重复澄清卡。 + */ + clientTurnId?: string; /** Optional hidden business/task context injected into the model run. */ taskContext?: Record | null; /** Step 1 用户上传的参考文件（可选）。 */ @@ -403,6 +409,8 @@ export async function streamIntent(req: StreamIntentRequest): Promise 0 ? req.files : undefined, }), diff --git a/frontend-web/src/roundtable-planning/api/roundtable-jobs.ts b/frontend-web/src/roundtable-planning/api/roundtable-jobs.ts index f683b57f..9d2fe0f3 100644 --- a/frontend-web/src/roundtable-planning/api/roundtable-jobs.ts +++ b/frontend-web/src/roundtable-planning/api/roundtable-jobs.ts @@ -11,6 +11,8 @@ * 增量事件归并。 */ +import { nanoid } from "nanoid"; + import { apiFetch } from "@/core/api/fetch-client"; import { getBackendBaseURL } from "@/core/config"; import { OptType, writeYLog } from "@/core/log/y-log"; @@ -87,6 +89,9 @@ function toStartBody(p: StartJobPayload) { // 缺省(未选模式)时不带这三字段 → 后端按 policy 默认(≈pro)，向后兼容。 const seat = p.seatMode ? seatModeToRunParams(p.seatMode) : null; return { + // 幂等请求 id：缺省现生成；「同一草稿至多一个活跃作业」由后端作用域唯一键保证， + // requestId 仅作审计/排查线索（重试时复用可让后端识别「同 id 但入参变了」的异常重试）。 + requestId: p.requestId ?? nanoid(), draftId: p.draftId ?? null, // taskId 非空 → task 作业（按 task 共享、不分权，报告写回独立 task-draft 存储）。 ...(p.taskId ? { taskId: p.taskId } : {}), diff --git a/frontend-web/src/roundtable-planning/api/task-drafts.ts b/frontend-web/src/roundtable-planning/api/task-drafts.ts index 2d73a0f1..d80f10f3 100644 --- a/frontend-web/src/roundtable-planning/api/task-drafts.ts +++ b/frontend-web/src/roundtable-planning/api/task-drafts.ts @@ -18,6 +18,7 @@ import { fromCreate, fromUpdate, readJson, + throwIfDraftConflict, toMeta, toRecord, type DraftCreatePayload, @@ -103,6 +104,7 @@ export async function updateTaskDraft(id: string, patch: DraftUpdatePayload): Pr body: JSON.stringify(fromUpdate(patch)), }); if (res.status === 404) return null; + await throwIfDraftConflict(res, id); const data = await readJson(res, "update task draft"); const record = toRecord(data); writeYLog({ diff --git a/frontend-web/src/roundtable-planning/hooks/useDraftPersistence.ts b/frontend-web/src/roundtable-planning/hooks/useDraftPersistence.ts index fd3b0b85..c652b7da 100644 --- a/frontend-web/src/roundtable-planning/hooks/useDraftPersistence.ts +++ b/frontend-web/src/roundtable-planning/hooks/useDraftPersistence.ts @@ -24,10 +24,12 @@ import { deleteDraft, deriveDraftName, getDraft, + isDraftVersionConflictError, listDrafts, updateDraft, type DraftCreatePayload, type DraftRecord, + type DraftStep2RunSnapshot, type DraftStep2Snapshot, type DraftStep3Snapshot, type DraftUpdatePayload, @@ -42,6 +44,34 @@ import { } from "@/roundtable-planning/api/task-drafts"; import type { JobStatus } from "@/roundtable-planning/lib/job-types"; +/** + * 409 冲突后的 step2 合并：以**服务端**为基准，把本地 runs 按 id upsert 回去。 + * + * 这样既保住并发写手（后台作业 / 另一个标签页）新增的 runs，又保留本标签页刚产出的 + * run 内容。activeRunId 若本地显式有值则以本地为准（本标签页正在看的 run），否则保留 + * 服务端值。其余 step2 顶层字段以本地为准（本地是最新交互态）。一致性来自 DB 版本号， + * 合并只是把双方都不丢地拼回来。 + */ +function mergeStep2OnConflict( + local: DraftStep2Snapshot, + server: DraftStep2Snapshot | null, +): DraftStep2Snapshot { + if (!server) return local; + const byId = new Map(); + for (const run of server.runs ?? []) { + if (run?.id) byId.set(run.id, run); + } + for (const run of local.runs ?? []) { + if (run?.id) byId.set(run.id, run); // 本地覆盖同 id 的服务端 run + } + return { + ...server, + ...local, + runs: Array.from(byId.values()), + activeRunId: local.activeRunId ?? server.activeRunId, + }; +} + /** 历史下拉渲染用的轻量列表项（`name` 映射自后端 `title`）。 */ export interface DraftListItem { id: string; @@ -198,6 +228,10 @@ export function useDraftPersistence( // 同步镜像，供保存队列在 render 之前读到最新 id（修复重复建草稿竞态）。 const currentDraftIdRef = useRef(null); + // 当前草稿的乐观锁版本号（本地副本基于的 server version）。update 时经 expectedVersion + // 回传；冲突（409）后重拉最新草稿、合并本地改动、用新版本号重试。load/create/update 成功 + // 后同步刷新，保证下一次写基于最新版本。 + const draftVersionRef = useRef(null); // 串行化所有写操作，避免并发 create。 const saveQueueRef = useRef>(Promise.resolve()); // 当前步骤镜像 —— 自动保存(不带 furthestStep)落到 create 分支时,用它兜底建档步骤。 @@ -207,10 +241,12 @@ export function useDraftPersistence( }, [currentStep]); // 包一层 setter：同时更新 state 与同步 ref。主页面通过它 setCurrentDraftId(null)。 + // 切换到不同草稿（或清空）时把乐观锁版本一并复位，防止拿旧草稿的版本号去写新草稿。 const setCurrentDraftId = useCallback>>( (action) => { setCurrentDraftIdState((prev) => { const next = typeof action === "function" ? (action as (p: string | null) => string | null)(prev) : action; + if (next !== currentDraftIdRef.current) draftVersionRef.current = null; currentDraftIdRef.current = next; return next; }); @@ -273,14 +309,40 @@ export function useDraftPersistence( const id = currentDraftIdRef.current; if (id) { - const updated = await apiUpdate(id, { - ...(furthestStep !== undefined ? { furthestStep } : {}), - step1, - step2, - ...(step3 ? { step3 } : {}), - }); - if (updated) return updated.id; - // 服务端已不存在（被删/换账号）→ 落回新建。 + try { + const updated = await apiUpdate(id, { + ...(furthestStep !== undefined ? { furthestStep } : {}), + step1, + step2, + ...(step3 ? { step3 } : {}), + ...(draftVersionRef.current !== null ? { expectedVersion: draftVersionRef.current } : {}), + }); + if (updated) { + draftVersionRef.current = updated.version; + return updated.id; + } + // 服务端已不存在（被删/换账号）→ 落回新建。 + } catch (err) { + if (!isDraftVersionConflictError(err)) throw err; + // 乐观锁冲突：另一写手（后台作业 / 另一标签页 / 重命名）已推进版本。 + // 重拉最新草稿，把本地 step2.runs 合并回去，用新版本号重试一次——双方都不丢。 + const latest = await apiGet(id); + if (latest) { + const mergedStep2 = mergeStep2OnConflict(step2, latest.step2); + const retried = await apiUpdate(id, { + ...(furthestStep !== undefined ? { furthestStep } : {}), + step1, + step2: mergedStep2, + ...(step3 ? { step3 } : {}), + expectedVersion: latest.version, + }); + if (retried) { + draftVersionRef.current = retried.version; + return retried.id; + } + } + // latest 为 null（被删）或重试仍 404 → 落回新建。 + } } const created = await apiCreate({ ...(taskIdRef.current ? { taskId: taskIdRef.current } : {}), @@ -294,6 +356,7 @@ export function useDraftPersistence( ...(step3 ? { step3 } : {}), }); currentDraftIdRef.current = created.id; + draftVersionRef.current = created.version; setCurrentDraftId(created.id); return created.id; }; @@ -301,7 +364,7 @@ export function useDraftPersistence( saveQueueRef.current = next.catch(() => undefined); return next as Promise; }, - [apiCreate, apiUpdate, getStep1Snapshot, getStep2Snapshot, getStep3Snapshot, hasSavableContent, setCurrentDraftId], + [apiCreate, apiGet, apiUpdate, getStep1Snapshot, getStep2Snapshot, getStep3Snapshot, hasSavableContent, setCurrentDraftId], ); const ensureDraft = useCallback( @@ -340,12 +403,35 @@ export function useDraftPersistence( const run = async (): Promise => { const id = currentDraftIdRef.current; if (id) { - const updated = await apiUpdate(id, { furthestStep, step2 }); - if (updated) { - currentDraftIdRef.current = updated.id; - return updated.id; + try { + const updated = await apiUpdate(id, { + furthestStep, + step2, + ...(draftVersionRef.current !== null ? { expectedVersion: draftVersionRef.current } : {}), + }); + if (updated) { + currentDraftIdRef.current = updated.id; + draftVersionRef.current = updated.version; + return updated.id; + } + // 服务端已不存在 → 落回新建。 + } catch (err) { + if (!isDraftVersionConflictError(err)) throw err; + // 冲突 → 重拉合并 runs 后用新版本号重试一次（与 enqueueSave 同策略）。 + const latest = await apiGet(id); + if (latest) { + const retried = await apiUpdate(id, { + furthestStep, + step2: mergeStep2OnConflict(step2, latest.step2), + expectedVersion: latest.version, + }); + if (retried) { + currentDraftIdRef.current = retried.id; + draftVersionRef.current = retried.version; + return retried.id; + } + } } - // 服务端已不存在 → 落回新建。 } const step1 = getStep1Snapshot(); if (!hasSavableContent(step1, step2)) return null; @@ -360,6 +446,7 @@ export function useDraftPersistence( step2, }); currentDraftIdRef.current = created.id; + draftVersionRef.current = created.version; setCurrentDraftId(created.id); return created.id; }; @@ -367,7 +454,7 @@ export function useDraftPersistence( saveQueueRef.current = next.catch(() => undefined); return next as Promise; }, - [apiCreate, apiUpdate, getStep1Snapshot, hasSavableContent, setCurrentDraftId], + [apiCreate, apiGet, apiUpdate, getStep1Snapshot, hasSavableContent, setCurrentDraftId], ); const handleManualSave = useCallback(() => { @@ -463,6 +550,8 @@ export function useDraftPersistence( setStep2InitGate(draft.furthestStep >= 2); setCurrentDraftId(draft.id); + // 必须在 setCurrentDraftId 之后：切换草稿会先把版本复位，这里再打上加载到的版本。 + draftVersionRef.current = draft.version; setIsHistoryOpen(false); prevStepRef.current = draft.furthestStep; setCurrentStep(draft.furthestStep); @@ -563,8 +652,12 @@ export function useDraftPersistence( void (async () => { try { // 只发 title —— 与自动保存的「不带 title」互补,重命名是唯一改标题的入口。 + // 不带 expectedVersion：部分更新不触碰内容字段，与自动保存并发时双方各写各的 + // 字段；自动保存侧撞 409 会重拉合并重试（标题被保留），一致性由 DB 版本保证。 const updated = await apiUpdate(id, { title: trimmed }); if (updated) { + // 重命名的正是当前草稿 → 同步版本号，省掉下一次自动保存的一次 409 往返。 + if (id === currentDraftIdRef.current) draftVersionRef.current = updated.version; await refreshList(); triggerToast("已重命名"); } diff --git a/frontend-web/src/roundtable-planning/hooks/useStep1Intent.ts b/frontend-web/src/roundtable-planning/hooks/useStep1Intent.ts index 780b4459..1ca4edbe 100644 --- a/frontend-web/src/roundtable-planning/hooks/useStep1Intent.ts +++ b/frontend-web/src/roundtable-planning/hooks/useStep1Intent.ts @@ -447,12 +447,19 @@ export function useStep1Intent(options: UseStep1IntentOptions): UseStep1IntentRe } } + // Phase 4 单飞幂等键：每个回合一个稳定 id。后端据此对 (threadId, clientTurnId) + // 跨 worker 单飞——重复 / 双击命中既有回合即幂等返回，杜绝并发下双发意图。 + const clientTurnId = + (typeof crypto !== "undefined" && crypto.randomUUID && crypto.randomUUID()) || + `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + const res = await streamIntent({ threadId: intentThreadId, message: trimmed, model: selectedModelRef.current || undefined, intentAgentId: intentAgentIdRef.current, taskContext: taskContextRef.current, + clientTurnId, files: fileRefs, signal: aborter.signal, onTextDelta: (delta) => { diff --git a/frontend-web/src/roundtable-planning/lib/job-types.ts b/frontend-web/src/roundtable-planning/lib/job-types.ts index 30daffa3..1f617f69 100644 --- a/frontend-web/src/roundtable-planning/lib/job-types.ts +++ b/frontend-web/src/roundtable-planning/lib/job-types.ts @@ -153,6 +153,13 @@ export interface StartJobPayload { * 普通会商省略。`toStartBody` 里下发为后端 `business_code`。 */ businessCode?: string | null; + /** + * 幂等请求 id（Phase 3 并发根治）。缺省由 `toStartBody` 用 nanoid 现生成；调用方若想在 + * 逻辑重试间复用同一 id（便于后端审计「同 requestId 但入参变了」的异常重试），可显式传入。 + * 「同一草稿至多一个活跃作业」的真正互斥由后端按 `(draftId/userId/taskId)` 作用域的唯一活跃键 + * 保证，requestId 仅作审计/排查线索，不承担正确性。 + */ + requestId?: string; } /** 终态集合——SSE/轮询判定作业是否结束。 */ diff --git a/offline-backend-20260512/.env b/offline-backend-20260512/.env index 5af82acb..c8867034 100644 --- a/offline-backend-20260512/.env +++ b/offline-backend-20260512/.env @@ -64,3 +64,6 @@ ROUNDTABLE_JOB_GATEWAY=inprocess DEEPSEEK_CHAT_API_KEY=sk-686b7bc000be4b7cbed6dccb3897c411 GLM5_FP8_API_KEY=modalresearch_gWu46FDpVLYZWPVEA3b_2fq8Jw2UJiKMgEm_Rxmn1SA SILICONFLOW_API_KEY=sk-ighylafootabtuthwueirmoycccyqtdgbifyhozduvolkdyt + +# WeKnora admin credentials (DeerFlow server-side iframe SSO proxy) +WEKNORA_ADMIN_PASSWORD=1qaz@WSX diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index af724f6c..8bf9644d 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -124,6 +124,28 @@ schema-hidden compatibility prefix `/api/multi-agent/position-roundtable`. Clients may retry the compatibility route only for a framework/proxy-level `404 Not Found`; domain-level missing-session responses must remain real 404s. +The position-roundtable state machine is multi-worker safe: node commands +(`complete`, `bind`, conversation save, reject) and session commands +(`activate`, archive, delete) lock the parent session and affected nodes in one +transaction, then record a unique command id for retry replay. The task and +intent snapshot writes use the same parent-row lock; they are permitted only +while `status=intent_pending`, so a late “confirm intent” write cannot change a +chain that another user has already activated. Activation itself rechecks the +stored confirmed intent after taking the lock. `tests/test_position_roundtable_ +commands.py` and `tests/test_position_roundtable_concurrency.py` cover the +state transitions; `tests/test_roundtable_postgres_concurrency.py` is the real +PostgreSQL acceptance suite (set `DEERFLOW_POSTGRES_CONCURRENCY_TEST_URL`). + +Roundtable backend jobs use a database unique active-dedupe key plus a leased +dispatcher, never process-local start deduplication. Cancellation is two +phase (`cancel_requested` then `cancelled`): the lease owner’s watcher normally +finalizes it, while the dispatcher reaps a cancellation whose lease has +expired after a worker crash. A worker that loses lease stops its local model +run immediately. Do not reintroduce direct `executor.cancel()` from the HTTP +cancel route, because it can cancel the watcher before the database state is +finalized. The boolean defaults on these new rows must use SQLAlchemy +`false()` rather than numeric `0`, so fresh PostgreSQL schemas can be created. + Position roundtable selects the separately seeded built-in agent `position-roundtable-intent` by passing `intent_agent_id` to the shared `/api/intent/init` and `/api/intent/stream` endpoints. Its request lifecycle, diff --git a/offline-backend-20260512/backend/README.md b/offline-backend-20260512/backend/README.md index a5bdbc46..159cdbbe 100644 --- a/offline-backend-20260512/backend/README.md +++ b/offline-backend-20260512/backend/README.md @@ -163,6 +163,25 @@ The separately deployed TaskCOP Vue app uses `POST /api/v1/auth/login/username` Position-roundtable history supports two modes. A non-empty `external_task_id` from the frontend route's `taskId` creates a row with `task_scoped = true`, so all sessions, nodes and persisted conversation snapshots for that task are shared across users; `user_id` is then creator audit metadata only. When the field is absent or empty, the original per-user personal-session behavior remains unchanged. Alembic revision `20260808_01` adds `task_scoped` with the safe default `false` for historical rows plus the task/update-time index. +### Roundtable concurrency guarantees + +Position-roundtable node completion, rejection, thread binding and activation +are database transactions with row locking and retry-stable command ids. This +keeps simultaneous seat completion from leaving a downstream stage locked. +Task and intent snapshots are frozen when the chain becomes active, so a late +request from another user cannot alter the confirmed input. Background +roundtable jobs use a database unique active-job key and renewable worker +leases; cancelled jobs are finalized by the lease holder or recovered after an +expired lease, allowing users to start a new job after a worker failure. + +The ordinary unit tests use SQLite. To verify deployed database semantics with +real PostgreSQL row locks and independent connections, set +`DEERFLOW_POSTGRES_CONCURRENCY_TEST_URL` and run: + +```bash +PYTHONPATH=. uv run --no-sync pytest tests/test_roundtable_postgres_concurrency.py -q +``` + AI writing supports a sample-imitation mode: the frontend can upload or paste a sample article, the Gateway extracts text for document files, and the `ai_writing` graph runs a `sample_analyzer` node before normal intent/research/outline/draft stages. The resulting style profile is injected into outline and draft prompts with explicit guardrails against copying source sentences, facts, names, or data. ### Custom Agents diff --git a/offline-backend-20260512/backend/app/gateway/deps.py b/offline-backend-20260512/backend/app/gateway/deps.py index f8156dd9..c8df1675 100644 --- a/offline-backend-20260512/backend/app/gateway/deps.py +++ b/offline-backend-20260512/backend/app/gateway/deps.py @@ -255,23 +255,23 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]: app.state.roundtable_draft_share_store = make_roundtable_draft_share_store(sf) - from deerflow.persistence.roundtable_chains import make_roundtable_chain_store - - app.state.roundtable_chain_store = make_roundtable_chain_store(sf) - - # 独立的岗位协同 Session / Node 存储。它只索引业务流程状态，真实消息和 - # 文件仍复用 LangGraph thread/checkpoint 与现有 artifact API。 - from deerflow.persistence.position_roundtable import make_position_roundtable_store - - app.state.position_roundtable_store = make_position_roundtable_store(sf) - - # 岗位会商工作视角（名称 / 顺序 / 特殊意图岗）是全局配置，和组织/RBAC - # 岗位分开保存。启动只补齐缺失内置项，不覆盖管理员的既有修改。 - from app.gateway.position_role_defaults import POSITION_ROLE_SEED - from deerflow.persistence.position_roles import make_position_role_store - - app.state.position_role_store = make_position_role_store(sf) - await app.state.position_role_store.ensure_seed(POSITION_ROLE_SEED) + from deerflow.persistence.roundtable_chains import make_roundtable_chain_store + + app.state.roundtable_chain_store = make_roundtable_chain_store(sf) + + # 独立的岗位协同 Session / Node 存储。它只索引业务流程状态，真实消息和 + # 文件仍复用 LangGraph thread/checkpoint 与现有 artifact API。 + from deerflow.persistence.position_roundtable import make_position_roundtable_store + + app.state.position_roundtable_store = make_position_roundtable_store(sf) + + # 岗位会商工作视角（名称 / 顺序 / 特殊意图岗）是全局配置，和组织/RBAC + # 岗位分开保存。启动只补齐缺失内置项，不覆盖管理员的既有修改。 + from app.gateway.position_role_defaults import POSITION_ROLE_SEED + from deerflow.persistence.position_roles import make_position_role_store + + app.state.position_role_store = make_position_role_store(sf) + await app.state.position_role_store.ensure_seed(POSITION_ROLE_SEED) from deerflow.persistence.business_mapping import make_business_mapping_store @@ -293,30 +293,30 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]: app.state.dashboard_session_store = make_dashboard_session_store(sf) - from deerflow.persistence.task_buttons import make_task_button_store - - app.state.task_button_store = make_task_button_store(sf) - - from deerflow.integrations.weknora.runtime import register_llmwiki_store - from deerflow.persistence.llmwiki import make_llmwiki_store - - app.state.llmwiki_store = make_llmwiki_store(sf) - register_llmwiki_store(app.state.llmwiki_store) - - from deerflow.persistence.taskcop_tasks import make_taskcop_task_store - - app.state.taskcop_task_store = make_taskcop_task_store(sf) - - from deerflow.persistence.task_reports import make_task_report_store - - app.state.task_report_store = make_task_report_store(sf) - - from deerflow.persistence.roundtable_artifact_submissions import make_roundtable_artifact_submission_store - - # Task-scoped artifact submission status: shared by task; actor is audit-only. - app.state.roundtable_artifact_submission_store = make_roundtable_artifact_submission_store(sf) - - from deerflow.persistence.parallel_agent_tasks import make_parallel_agent_task_store + from deerflow.persistence.task_buttons import make_task_button_store + + app.state.task_button_store = make_task_button_store(sf) + + from deerflow.integrations.weknora.runtime import register_llmwiki_store + from deerflow.persistence.llmwiki import make_llmwiki_store + + app.state.llmwiki_store = make_llmwiki_store(sf) + register_llmwiki_store(app.state.llmwiki_store) + + from deerflow.persistence.taskcop_tasks import make_taskcop_task_store + + app.state.taskcop_task_store = make_taskcop_task_store(sf) + + from deerflow.persistence.task_reports import make_task_report_store + + app.state.task_report_store = make_task_report_store(sf) + + from deerflow.persistence.roundtable_artifact_submissions import make_roundtable_artifact_submission_store + + # Task-scoped artifact submission status: shared by task; actor is audit-only. + app.state.roundtable_artifact_submission_store = make_roundtable_artifact_submission_store(sf) + + from deerflow.persistence.parallel_agent_tasks import make_parallel_agent_task_store app.state.parallel_agent_task_store = make_parallel_agent_task_store(sf) @@ -332,6 +332,13 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]: # （Phase 2 后台编排图写进度用）。 _register_roundtable_job_default_store(app.state.roundtable_job_store) + # 意图回合单飞 store（Phase 4）：``/api/intent/stream`` 据此对 (thread_id, + # client_turn_id) 跨 worker 单飞 + 幂等回放，杜绝并发双发意图。sf 为 None（无 DB） + # 时为 None，路由侧判空跳过单飞、走原流程。 + from deerflow.persistence.intent_turns import make_intent_turn_store + + app.state.intent_turn_store = make_intent_turn_store(sf) + # 圆桌会商「诊断日志」store：会商全流程各处报错 / 关键事件落库，供管理员诊断页筛选。 # 同样登记进程级默认仓储——harness 侧编排引擎 / 进程内网关跨边界写诊断用。 from deerflow.persistence.roundtable_diagnostics import ( @@ -393,18 +400,32 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]: # task 作业把最终报告写回**独立的** task-draft 存储（不分权）。 task_draft_store=getattr(app.state, "roundtable_task_draft_store", None), ) + + # Phase 3 dispatcher：跨 worker 领取 queued/租约过期作业 + outbox 补偿器。 + # 每个 worker 进程都跑一个；谁先 claim_job 成功谁执行（DB 条件 UPDATE 互斥）， + # 崩溃的 worker 租约过期后由其它 worker 接管。路由 start/resume 后 nudge 立即唤醒。 + from app.gateway.roundtable_job_dispatcher import RoundtableJobDispatcher + + app.state.roundtable_job_dispatcher = RoundtableJobDispatcher( + app.state.roundtable_job_store, + app.state.roundtable_job_executor, + draft_store=getattr(app.state, "roundtable_draft_store", None), + task_draft_store=getattr(app.state, "roundtable_task_draft_store", None), + ) + app.state.roundtable_job_dispatcher.start() else: app.state.roundtable_job_executor = None + app.state.roundtable_job_dispatcher = None # Run event store (has its own factory with config-driven backend selection) run_events_config = getattr(config, "run_events", None) app.state.run_event_store = make_run_event_store(run_events_config) - # RunManager with store backing for persistence - app.state.run_manager = RunManager(store=app.state.run_store) - from deerflow.runtime import make_redis_concurrency_gate - - app.state.concurrency_gate = make_redis_concurrency_gate(config) + # RunManager with store backing for persistence + app.state.run_manager = RunManager(store=app.state.run_store) + from deerflow.runtime import make_redis_concurrency_gate + + app.state.concurrency_gate = make_redis_concurrency_gate(config) try: yield @@ -412,16 +433,19 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]: from deerflow.persistence.notifications import set_notification_store set_notification_store(None) + dispatcher = getattr(app.state, "roundtable_job_dispatcher", None) + if dispatcher is not None: + await dispatcher.stop() queue = getattr(app.state, "knowledge_auto_ingest_queue", None) if queue is not None: from deerflow.knowledge.auto_ingest import set_auto_ingest_queue await queue.stop() set_auto_ingest_queue(None) - await close_engine() - gate = getattr(app.state, "concurrency_gate", None) - if gate is not None: - await gate.aclose() + await close_engine() + gate = getattr(app.state, "concurrency_gate", None) + if gate is not None: + await gate.aclose() # --------------------------------------------------------------------------- @@ -447,12 +471,12 @@ get_run_manager: Callable[[Request], RunManager] = _require("run_manager", "Run get_checkpointer: Callable[[Request], Checkpointer] = _require("checkpointer", "Checkpointer") get_run_event_store: Callable[[Request], RunEventStore] = _require("run_event_store", "Run event store") get_feedback_repo: Callable[[Request], FeedbackRepository] = _require("feedback_repo", "Feedback") -get_run_store: Callable[[Request], RunStore] = _require("run_store", "Run store") - - -def get_concurrency_gate(request: Request): - """Return the optional cross-worker concurrency gate.""" - return getattr(request.app.state, "concurrency_gate", None) +get_run_store: Callable[[Request], RunStore] = _require("run_store", "Run store") + + +def get_concurrency_gate(request: Request): + """Return the optional cross-worker concurrency gate.""" + return getattr(request.app.state, "concurrency_gate", None) def get_store(request: Request): @@ -624,34 +648,34 @@ _cached_local_provider: LocalAuthProvider | None = None _cached_repo: SQLiteUserRepository | None = None -def get_request_access_token(request: Request) -> str | None: - """Extract access token from cookie or Authorization header. - - Priority: - 1. ``access_token`` cookie (browser session mode) - 2. ``Authorization: Bearer `` header (API client mode) - 3. ``access_token`` query parameter for the WeKnora iframe bootstrap only - """ - cookie_token = request.cookies.get("access_token") - if cookie_token: - return cookie_token - - authorization = request.headers.get("authorization", "").strip() - if authorization: - scheme, _, token = authorization.partition(" ") - if scheme.lower() == "bearer": - token = token.strip() - if token: - return token - - from app.gateway.proxy_path import app_relative_path - - path = app_relative_path(request).rstrip("/") - if path.startswith("/api/llmwiki/knowledge-bases/") and path.endswith("/weknora/frame"): - query_token = (request.query_params.get("access_token") or "").strip() - if query_token: - return query_token - return None +def get_request_access_token(request: Request) -> str | None: + """Extract access token from cookie or Authorization header. + + Priority: + 1. ``access_token`` cookie (browser session mode) + 2. ``Authorization: Bearer `` header (API client mode) + 3. ``access_token`` query parameter for the WeKnora iframe bootstrap only + """ + cookie_token = request.cookies.get("access_token") + if cookie_token: + return cookie_token + + authorization = request.headers.get("authorization", "").strip() + if authorization: + scheme, _, token = authorization.partition(" ") + if scheme.lower() == "bearer": + token = token.strip() + if token: + return token + + from app.gateway.proxy_path import app_relative_path + + path = app_relative_path(request).rstrip("/") + if path.startswith("/api/llmwiki/knowledge-bases/") and path.endswith("/weknora/frame"): + query_token = (request.query_params.get("access_token") or "").strip() + if query_token: + return query_token + return None def get_local_provider() -> LocalAuthProvider: diff --git a/offline-backend-20260512/backend/app/gateway/roundtable_job_dispatcher.py b/offline-backend-20260512/backend/app/gateway/roundtable_job_dispatcher.py new file mode 100644 index 00000000..cf095ec3 --- /dev/null +++ b/offline-backend-20260512/backend/app/gateway/roundtable_job_dispatcher.py @@ -0,0 +1,355 @@ +"""圆桌后台作业 dispatcher（Phase 3）：跨 worker 领取 + 崩溃接管 + outbox 补偿。 + +把「谁来跑这个作业」从「发起请求的那个 worker 的进程内存」收敛到 **DB 租约**： + +- **领取（claim）**：后台循环定期 ``list_claimable``（``queued`` 新作业/续跑，或 ``running`` + 但租约过期的僵尸），对每个候选用 ``claim_job`` 的**原子条件 UPDATE** 抢占 —— 多 worker + 并发扫到同一候选时只有一个 ``rowcount==1``，其余拿到 None 自动跳过。一致性来自 DB， + 不是进程内锁。 +- **心跳**：领取成功后交给 ``RoundtableJobExecutor`` 跑，执行器内部定期 ``renew_lease``； + worker 崩溃则心跳停止，租约最长一个 TTL 后过期，下一轮循环在其它 worker 重新领取 + （``attempt+1``）—— 崩溃可接管。 +- **启动扫描**：本 worker 启动时立即跑一轮 ``dispatch_once``，尽快接走历史遗留的 + queued / 租约过期作业（不用等第一个轮询周期）。 +- **outbox 补偿器**：``draft_persist_pending=1`` 的作业（后台研讨结果因乐观锁冲突没能写回 + 草稿）在这里被重新投影到草稿 step2.runs，成功后清标记 —— 「研讨结果不丢」由 DB 标记驱动。 + +所有 worker 都跑同一个 dispatcher；谁先 ``claim_job`` 成功谁执行，天然幂等、互斥。 +""" + +from __future__ import annotations + +import asyncio +import logging +from contextlib import suppress +from datetime import UTC, datetime, timedelta +from typing import Any + +from app.gateway.roundtable_job_executor import ( + _LEASE_TTL_SECONDS, + WORKER_ID, + params_from_snapshot, +) + +logger = logging.getLogger(__name__) + +# dispatcher 轮询间隔（秒）。路由 start/resume 成功后会 ``nudge()`` 立即唤醒，故正常路径 +# 几乎无感延迟；该间隔只是「没有 nudge 时的兜底」，兼作崩溃接管的发现周期。 +_DISPATCH_POLL_INTERVAL = 3.0 +# outbox 补偿器轮询间隔（秒）。pending 标记是低频异常路径，慢一点无妨。 +_COMPENSATE_INTERVAL = 10.0 +# 单轮最多领取的作业数（防止一次扫太多把事件循环占死）。 +_CLAIM_BATCH = 20 + + +class RoundtableJobDispatcher: + """跨 worker 的圆桌后台作业调度器（每个 worker 进程一个实例，lifespan 起停）。""" + + def __init__( + self, + store, + executor, + *, + draft_store=None, + task_draft_store=None, + worker_id: str | None = None, + ) -> None: + self._store = store + self._executor = executor + # outbox 补偿器投影草稿用（个人草稿 / task 草稿两个存储）。 + self._draft_store = draft_store + self._task_draft_store = task_draft_store + self._worker_id = worker_id or WORKER_ID + self._wake = asyncio.Event() + self._task: asyncio.Task | None = None + self._stopping = False + self._last_compensate = datetime.now(UTC) + + # ── 生命周期 ───────────────────────────────────────────────────────────── + + def start(self) -> None: + """启动后台循环（lifespan startup 调用）。幂等。""" + if self._task is not None and not self._task.done(): + return + self._stopping = False + self._task = asyncio.create_task(self._loop(), name="roundtable-job-dispatcher") + logger.info("[roundtable-jobs] dispatcher started (worker=%s)", self._worker_id) + + async def stop(self) -> None: + """停止后台循环（lifespan shutdown 调用）。""" + self._stopping = True + self._wake.set() + if self._task is not None: + self._task.cancel() + with suppress(asyncio.CancelledError, Exception): + await self._task + self._task = None + + def nudge(self) -> None: + """立即唤醒一轮领取（路由 start/resume 成功后调用，降低启动延迟）。""" + self._wake.set() + + # ── 主循环 ─────────────────────────────────────────────────────────────── + + async def _loop(self) -> None: + # 启动即扫一轮：尽快接管历史遗留 queued / 租约过期（僵尸）作业。 + await self._safe_dispatch() + while not self._stopping: + try: + await asyncio.wait_for(self._wake.wait(), timeout=_DISPATCH_POLL_INTERVAL) + except TimeoutError: + pass + self._wake.clear() + if self._stopping: + break + await self._safe_dispatch() + # outbox 补偿器按自己的节奏跑（低频）。 + if (datetime.now(UTC) - self._last_compensate).total_seconds() >= _COMPENSATE_INTERVAL: + self._last_compensate = datetime.now(UTC) + await self._safe_compensate() + + async def _safe_dispatch(self) -> None: + try: + await self.dispatch_once() + except Exception: # noqa: BLE001 — 单轮失败不能杀死循环，下轮重试 + logger.exception("[roundtable-jobs] dispatch round failed") + + async def _safe_compensate(self) -> None: + try: + await self.compensate_once() + except Exception: # noqa: BLE001 + logger.exception("[roundtable-jobs] compensate round failed") + + # ── 领取 ───────────────────────────────────────────────────────────────── + + async def dispatch_once(self) -> int: + """扫一轮可领取作业并逐个条件抢占。返回本轮成功启动的作业数。""" + now = datetime.now(UTC) + await self._reap_cancel_requested(now) + try: + candidate_ids = await self._store.list_claimable(now=now, limit=_CLAIM_BATCH) + except Exception: # noqa: BLE001 + logger.warning("[roundtable-jobs] list_claimable failed", exc_info=True) + return 0 + if not candidate_ids: + return 0 + + started = 0 + lease_until = now + timedelta(seconds=_LEASE_TTL_SECONDS) + for job_id in candidate_ids: + try: + row = await self._store.claim_job( + job_id, lease_owner=self._worker_id, lease_until=lease_until + ) + except Exception: # noqa: BLE001 + logger.warning("[roundtable-jobs] claim_job failed for %s", job_id, exc_info=True) + continue + if row is None: + # 别的 worker 先领了 / 状态已变 —— 条件 UPDATE rowcount=0，跳过即可。 + continue + params = params_from_snapshot( + row.get("input_snapshot"), job_id=job_id, lease_owner=self._worker_id + ) + if params is None: + # 快照缺失/损坏：无法重建执行参数 —— 标记 error 并释放活跃键，避免僵死占位。 + logger.error( + "[roundtable-jobs] job %s claimed but snapshot missing/corrupt; marking error", + job_id, + ) + with suppress(Exception): + await self._store.update_progress( + job_id, + lease_owner=self._worker_id, + status="error", + error="input_snapshot 缺失或损坏，无法在后台重建研讨", + ) + continue + self._executor.start_job(params) + started += 1 + logger.info( + "[roundtable-jobs] worker=%s claimed job %s (attempt=%s, resume=%s)", + self._worker_id, job_id, row.get("attempt"), params.is_resume, + ) + return started + + async def _reap_cancel_requested(self, now: datetime) -> int: + """收口持租 worker 崩溃后遗留的 ``cancel_requested`` 作业。 + + 有效租约仍在时，必须让持租 worker 的取消看门狗收口；无租约或租约过期时，已经 + 不存在可信的持租者，此处通过 ``finalize_cancel`` 的条件 UPDATE 安全地回收。 + """ + try: + candidate_ids = await self._store.list_cancel_reapable( + now=now, limit=_CLAIM_BATCH + ) + except Exception: # noqa: BLE001 + logger.warning("[roundtable-jobs] list_cancel_reapable failed", exc_info=True) + return 0 + + finalized = 0 + for job_id in candidate_ids: + try: + row = await self._store.finalize_cancel(job_id) + except Exception: # noqa: BLE001 + logger.warning( + "[roundtable-jobs] finalize orphaned cancellation failed for %s", + job_id, + exc_info=True, + ) + continue + if row is not None: + finalized += 1 + logger.info( + "[roundtable-jobs] reaped orphaned cancellation for job %s", job_id + ) + return finalized + + # ── outbox 补偿器 ──────────────────────────────────────────────────────── + + def _draft_store_for(self, task_id: str | None): + """task 作业 → task-draft 存储；普通作业 → 个人草稿存储（与执行器同规则）。""" + return self._task_draft_store if task_id else self._draft_store + + async def compensate_once(self) -> int: + """把 ``draft_persist_pending=1`` 的作业结果投影回草稿，成功后清标记。 + + 投影逻辑与执行器 ``_write_run_to_draft`` 的终态写一致：读最新草稿 → 按 + ``run_id=job-`` upsert 一个 run（用作业行里的 dialogues/step3/status）→ + 带 ``expected_version`` 乐观锁写回。写成功（或草稿已含该 run 的终态）即清标记。 + 返回本轮成功投影的作业数。 + """ + try: + pending = await self._store.list_draft_persist_pending(limit=_CLAIM_BATCH) + except Exception: # noqa: BLE001 + logger.warning("[roundtable-jobs] list_draft_persist_pending failed", exc_info=True) + return 0 + if not pending: + return 0 + + projected = 0 + for job in pending: + job_id = job.get("id") + draft_id = job.get("draft_id") + draft_store = self._draft_store_for(job.get("task_id")) + if not draft_id or draft_store is None: + # 无草稿可投影（如纯 task 作业但 task_store 未配）——直接清标记，避免死循环。 + with suppress(Exception): + await self._store.clear_draft_persist_pending(job_id) + continue + try: + ok = await self._project_job_to_draft(draft_store, job) + except Exception: # noqa: BLE001 + logger.warning( + "[roundtable-jobs] compensate project failed for job %s", job_id, exc_info=True + ) + continue + if ok: + with suppress(Exception): + await self._store.clear_draft_persist_pending(job_id) + projected += 1 + logger.info("[roundtable-jobs] compensated draft write for job %s", job_id) + return projected + + async def _project_job_to_draft(self, draft_store: Any, job: dict[str, Any]) -> bool: + """把单个作业的最新状态投影成草稿 step2.runs 里的一个 run。成功返回 True。 + + 与执行器不同：这里没有 ``params``（作业可能跑在别的 worker / 已结束），只能用作业行 + 里已落库的 ``dialogues``/``step3``/``status`` 重建 run —— 这些正是「研讨结果」本体， + 足够让前端加载草稿时看到完整的后台研讨记录与报告。 + """ + from deerflow.persistence.roundtable_drafts.sql import DraftConcurrentWriteError + + job_id = job.get("id") + draft_id = job.get("draft_id") + user_id = job.get("user_id") + run_id = f"job-{job_id}" + status = job.get("status") or "running" + dialogues = job.get("dialogues") or [] + step3 = job.get("step3") + + # task 草稿不分权：get/update 不传 user_id；个人草稿按 user。 + is_task = bool(job.get("task_id")) + + for _attempt in range(3): + try: + draft = ( + await draft_store.get_draft(draft_id) + if is_task + else await draft_store.get_draft(draft_id, user_id) + ) + except Exception: # noqa: BLE001 + draft = None + if draft is None: + # 草稿已被删除 —— 无处投影，视作完成（清标记）。 + return True + expected_version = draft.get("version") + + step2 = dict(draft.get("step2") or {}) + runs = list(step2.get("runs") or []) + idx = next((i for i, r in enumerate(runs) if r.get("id") == run_id), -1) + created = runs[idx].get("createdAt") if idx >= 0 else (job.get("created_at") or datetime.now(UTC).isoformat()) + prev_step3 = runs[idx].get("step3") if idx >= 0 else None + effective_step3 = step3 if step3 is not None else prev_step3 + + # 从 dialogues 里提取席位信息（尽量还原 selectedAgents）。 + agents_seen: list[dict[str, str]] = [] + seen_ids: set[str] = set() + for d in dialogues: + aid = str(d.get("agentId") or "") + if aid and aid not in seen_ids and d.get("role") == "seat": + seen_ids.add(aid) + agents_seen.append({"agent_id": aid, "name": str(d.get("name") or aid)}) + + run = { + "id": run_id, + "title": "后台研讨", + "createdAt": created, + "source": job.get("orchestration_mode") or "recommend", + "selectedAgents": agents_seen, + "threadIds": job.get("thread_ids"), + "coordinatorName": job.get("coordinator_name"), + "step2RoundtableDialogues": dialogues, + "lastLeaderContent": next( + (d.get("content") for d in reversed(dialogues) if d.get("role") == "leader"), "" + ) or "", + "hasConsensus": status == "done", + "consensusPercentage": int(job.get("consensus_percentage") or (100 if status == "done" else 0)), + "budgetLimit": 0, + "orchestrationMode": job.get("orchestration_mode") or "recommend", + "chain": job.get("chain"), + "orchestrationPlan": job.get("orchestration_plan"), + "step3": effective_step3, + "jobId": job_id, + "jobStatus": status, + } + runs = [run if i == idx else r for i, r in enumerate(runs)] if idx >= 0 else [*runs, run] + step2["runs"] = runs + step2["activeRunId"] = run_id + has_report = effective_step3 is not None + update_kwargs: dict[str, Any] = { + "step2": step2, + "furthest_step": 3 if (status == "done" and has_report) else 2, + } + if has_report: + update_kwargs["step3"] = effective_step3 + + try: + if is_task: + await draft_store.update_draft( + draft_id, expected_version=expected_version, **update_kwargs + ) + else: + await draft_store.update_draft( + draft_id, user_id, expected_version=expected_version, **update_kwargs + ) + return True + except DraftConcurrentWriteError: + # 并发写冲突 —— 重读最新草稿再合并重试（与执行器同款乐观锁重试）。 + continue + except Exception: # noqa: BLE001 + logger.warning( + "[roundtable-jobs] compensate update_draft failed for %s", draft_id, exc_info=True + ) + return False + # 乐观锁重试耗尽：保留 pending 标记，等下一轮补偿器再试。 + return False diff --git a/offline-backend-20260512/backend/app/gateway/roundtable_job_executor.py b/offline-backend-20260512/backend/app/gateway/roundtable_job_executor.py index 7f3b11cd..9166b0ce 100644 --- a/offline-backend-20260512/backend/app/gateway/roundtable_job_executor.py +++ b/offline-backend-20260512/backend/app/gateway/roundtable_job_executor.py @@ -16,9 +16,12 @@ from __future__ import annotations import asyncio import logging +import os +import uuid as _uuid from collections.abc import Callable +from contextlib import suppress from dataclasses import dataclass, field -from datetime import UTC, datetime +from datetime import UTC, datetime, timedelta from typing import Any from deerflow.agents.roundtable_orchestrator import ( @@ -27,10 +30,34 @@ from deerflow.agents.roundtable_orchestrator import ( run_orchestration, ) from deerflow.persistence.roundtable_diagnostics import DiagnosticsRecorder, get_default_store +from deerflow.persistence.roundtable_drafts.sql import DraftConcurrentWriteError from deerflow.runtime.user_context import reset_current_user, set_current_user logger = logging.getLogger(__name__) +# 本 worker 进程的唯一标识（Phase 3 租约持有者）。pid 保证同机多 worker 进程互不相同， +# 追加随机后缀避免 pid 复用导致的极端撞车。跨进程「谁在跑这个作业」由它写进 lease_owner。 +WORKER_ID = f"w-{os.getpid()}-{_uuid.uuid4().hex[:8]}" + +# 租约 TTL 与心跳间隔（Phase 3）。心跳每 _HEARTBEAT_INTERVAL 秒续租一次，把 lease_until +# 推到 now + _LEASE_TTL_SECONDS。只要 worker 活着，租约永远不过期、别的 worker 抢不走； +# worker 崩溃后心跳停止，最长 _LEASE_TTL_SECONDS 后租约过期，dispatcher 在其它 worker 上 +# 重新领取（attempt+1）—— 崩溃可接管的一致性来自 DB 租约而非进程内存。 +# 约束：_LEASE_TTL_SECONDS 必须明显大于 _HEARTBEAT_INTERVAL（留足续租余量）。 +_LEASE_TTL_SECONDS = 60 +_HEARTBEAT_INTERVAL = 20.0 + +# 跨进程取消轮询间隔（秒）。cancel 端点把 ``status='cancel_requested'`` 写进 DB（权威意图）， +# 实际跑作业的 worker 上的看门狗协程定期读 DB 检测到后，收口 ``cancelled`` 终态并对本进程的 +# 编排 task 调 ``cancel()``。多 worker 下 cancel 请求可能打到非执行 worker，靠此机制最长延迟 +# 一个轮询周期即可让幽灵作业停下。 +_CANCEL_POLL_INTERVAL = 5.0 + +# 草稿乐观锁重试次数。每次尝试都带 expected_version（冲突则重读重试）；重试全部耗尽后 +# **绝不退化为无版本检查的 last-write-wins 盲写**，而是给作业置 draft_persist_pending +# 标记，由 outbox 补偿器（Phase 3）投影结果到草稿——一致性来自 DB 标记而非运气。 +_DRAFT_WRITE_RETRIES = 3 + # 「后台挂起」的 Step3 自动产出（方案总结报告 md + 结果绘制 HTML）总开关。 # 历史：2026-06-05 因后台 start_run 路径报告不稳定临时禁用；2026-06-12 定位到根因是 # ``_enforce_agent_access`` 对进程内虚拟 request 的 ``request.state`` 裸访问（AttributeError @@ -203,6 +230,115 @@ class StartParams: # 深链接业务码（rwfx→6BF 等）。透传给 run_orchestration → build_summary_prompt：写实业务额外注入 # 「业务链完整性核对」，让后台总结报告核对产出相对业务链完不完整。普通会商为空 → 不注入。 business_code: str | None = None + # Phase 3 租约持有者（= dispatcher 领取时写入的 worker_id）。执行器据此：① 所有进度落库 + # 带 lease_owner（update_progress 租约门，非持租者的过期写被拒）；② 心跳续租；③ 看门狗 + # 检测到 cancel_requested 时以持租者身份收口 cancelled。None = 旧路径（不走租约门）。 + lease_owner: str | None = None + + +# ── Phase 3：input_snapshot 序列化 / 反序列化 ───────────────────────────────── +# +# 路由 start/resume 时把整份启动入参序列化成 JSON 存进 ``input_snapshot``；dispatcher +# 在**任意 worker** 领取 queued/租约过期作业后，据此无损重建 StartParams —— 这是「跨进程 +# 可接管 / 崩溃可恢复」的前提（不再依赖发起请求的那个 worker 的进程内存）。 +# +# 快照 schema 版本 ``_SNAPSHOT_VERSION``：将来字段变更时升版并兼容读旧版。 +_SNAPSHOT_VERSION = 1 + + +def build_start_snapshot( + *, + user_id: str, + agents: list[SeatRef], + seed_message: str, + model: str | None = None, + mode: str = "recommend", + intent_text: str = "", + thread_ids: dict[str, str] | None = None, + coordinator_name: str = "roundtable-coordinator", + draft_id: str | None = None, + task_id: str | None = None, + is_resume: bool = False, + chain: dict[str, Any] | None = None, + orchestration_plan: dict[str, Any] | None = None, + enable_report: bool = ROUNDTABLE_BACKGROUND_REPORT_ENABLED, + seat_thinking_enabled: bool | None = None, + seat_reasoning_effort: str | None = None, + seat_subagent_enabled: bool = False, + seat_skill_directives: dict[str, bool] | None = None, + seat_modes: dict[str, str] | None = None, + gather_first: bool = False, + business_code: str | None = None, +) -> dict[str, Any]: + """把一次启动入参序列化为可 JSON 落库的快照 dict（snake_case，对齐 StartParams）。""" + return { + "v": _SNAPSHOT_VERSION, + "user_id": user_id, + "agents": [{"agent_id": a.agent_id, "name": a.name} for a in agents], + "seed_message": seed_message, + "model": model, + "mode": mode, + "intent_text": intent_text, + "thread_ids": thread_ids, + "coordinator_name": coordinator_name, + "draft_id": draft_id, + "task_id": task_id, + "is_resume": is_resume, + "chain": chain, + "orchestration_plan": orchestration_plan, + "enable_report": enable_report, + "seat_thinking_enabled": seat_thinking_enabled, + "seat_reasoning_effort": seat_reasoning_effort, + "seat_subagent_enabled": seat_subagent_enabled, + "seat_skill_directives": dict(seat_skill_directives or {}), + "seat_modes": dict(seat_modes or {}), + "gather_first": bool(gather_first), + "business_code": business_code, + } + + +def params_from_snapshot( + snapshot: dict[str, Any] | None, *, job_id: str, lease_owner: str | None +) -> StartParams | None: + """从 ``input_snapshot`` 无损重建 StartParams。快照缺失 / 损坏返回 None（调用方跳过）。""" + if not snapshot or not isinstance(snapshot, dict): + return None + try: + agents = [ + SeatRef(str(a.get("agent_id") or ""), str(a.get("name") or a.get("agent_id") or "")) + for a in (snapshot.get("agents") or []) + if isinstance(a, dict) and a.get("agent_id") + ] + if not agents: + return None + return StartParams( + job_id=job_id, + user_id=str(snapshot.get("user_id") or ""), + agents=agents, + seed_message=str(snapshot.get("seed_message") or ""), + model=snapshot.get("model"), + mode=str(snapshot.get("mode") or "recommend"), + intent_text=str(snapshot.get("intent_text") or ""), + thread_ids=snapshot.get("thread_ids"), + coordinator_name=str(snapshot.get("coordinator_name") or "roundtable-coordinator"), + draft_id=snapshot.get("draft_id"), + task_id=snapshot.get("task_id"), + is_resume=bool(snapshot.get("is_resume")), + chain=snapshot.get("chain"), + orchestration_plan=snapshot.get("orchestration_plan"), + enable_report=bool(snapshot.get("enable_report", ROUNDTABLE_BACKGROUND_REPORT_ENABLED)), + seat_thinking_enabled=snapshot.get("seat_thinking_enabled"), + seat_reasoning_effort=snapshot.get("seat_reasoning_effort"), + seat_subagent_enabled=bool(snapshot.get("seat_subagent_enabled")), + seat_skill_directives=dict(snapshot.get("seat_skill_directives") or {}), + seat_modes=dict(snapshot.get("seat_modes") or {}), + gather_first=bool(snapshot.get("gather_first")), + business_code=snapshot.get("business_code"), + lease_owner=lease_owner, + ) + except Exception: # noqa: BLE001 — 快照损坏不应让 dispatcher 崩溃 + logger.exception("failed to rebuild StartParams from snapshot for job %s", job_id) + return None # 网关工厂：给定一次启动参数，返回一个 RoundtableGateway 实例。 @@ -259,7 +395,10 @@ class RoundtableJobExecutor: thread_ids, coordinator_name = await init() try: await self._store.update_progress( - params.job_id, thread_ids=thread_ids, coordinator_name=coordinator_name + params.job_id, + lease_owner=params.lease_owner, + thread_ids=thread_ids, + coordinator_name=coordinator_name, ) except Exception: # noqa: BLE001 logger.warning("persist thread_ids for job %s failed", params.job_id, exc_info=True) @@ -277,22 +416,52 @@ class RoundtableJobExecutor: agents=params.agents, on_dialogue=_on_dialogue if (self._draft_store_for(params) and params.draft_id) else None, recorder=diag, + # Phase 3：进度落库带租约持有者 —— update_progress 租约门只放行持租者的写， + # 非持租者（被接管/已取消）的过期进度写被 DB 条件 UPDATE 拒绝。 + lease_owner=params.lease_owner, ) - result = await run_orchestration( - gateway=gateway, - progress=progress, - agents=params.agents, - thread_ids=thread_ids, - coordinator_name=coordinator_name, - model=params.model, - seed_message=params.seed_message, - mode=params.mode, - intent_text=params.intent_text, - enable_report=params.enable_report, - plan=params.orchestration_plan, - gather_first=params.gather_first, - business_code=params.business_code, + # Phase 3 心跳续租：活着就持续把 lease_until 推后，保证本 worker 的租约不过期、 + # 别的 worker 抢不走；本协程随编排 task 生死（finally 里 cancel）。崩溃则心跳停， + # 租约最长 _LEASE_TTL_SECONDS 后过期 → dispatcher 在其它 worker 接管。 + main_task = asyncio.current_task() + heartbeat = ( + asyncio.create_task(self._heartbeat(params.job_id, params.lease_owner, main_task)) + if params.lease_owner + else None ) + # 跨进程取消看门狗：run_orchestration 是一个不透明长 await（多轮 LLM）， + # 无法在其内部插检查点。改为起一个并行的看门狗协程，定期读 DB 的 job status； + # cancel 端点把 status=cancel_requested 写进 DB（权威意图），看门狗检测到后以**持租者** + # 身份收口 cancelled 终态（清唯一活跃键 + 租约）并对本编排 task 调 cancel() → + # CancelledError 在 run_orchestration 的下一个 await 点抛出并上抛。 + # 这让 cancel 在多 worker 下跨进程生效——cancel 打到非执行 worker 也无妨。 + watcher = asyncio.create_task( + self._watch_cancellation(params.job_id, main_task, params.lease_owner) + ) + try: + result = await run_orchestration( + gateway=gateway, + progress=progress, + agents=params.agents, + thread_ids=thread_ids, + coordinator_name=coordinator_name, + model=params.model, + seed_message=params.seed_message, + mode=params.mode, + intent_text=params.intent_text, + enable_report=params.enable_report, + plan=params.orchestration_plan, + gather_first=params.gather_first, + business_code=params.business_code, + ) + finally: + watcher.cancel() + with suppress(asyncio.CancelledError, Exception): + await watcher + if heartbeat is not None: + heartbeat.cancel() + with suppress(asyncio.CancelledError, Exception): + await heartbeat # 终态：把最终对话 + step3 + jobStatus 落进同一个 run（含 furthest_step=3）。 await self._write_run_to_draft( params, @@ -302,16 +471,18 @@ class RoundtableJobExecutor: step3=result.step3, thread_ids=thread_ids, ) - # step3 也挂 job（API / 弹窗用）。 + # step3 也挂 job（API / 弹窗用）。带租约门：仅持租者可写。 if result.status == "done" and result.step3: try: - await self._store.update_progress(params.job_id, step3=result.step3) + await self._store.update_progress( + params.job_id, lease_owner=params.lease_owner, step3=result.step3 + ) except Exception: # noqa: BLE001 logger.warning("persist step3 to job %s failed", params.job_id, exc_info=True) except asyncio.CancelledError: # 干净停止即可，**不在被取消的任务里写 DB**：取消态下二次 await 会再抛 # CancelledError、并可能打断进行中的 sqlite 连接。``cancelled`` 终态由 - # 路由的 cancel 端点（executor.cancel + store.update_progress）权威写入。 + # 取消看门狗（或租约过期后的 dispatcher 回收）权威写入。 logger.info("roundtable job %s cancelled", params.job_id) raise except Exception as exc: # noqa: BLE001 — 顶层兜底：run_orchestration 内部已尽量标 error @@ -324,6 +495,97 @@ class RoundtableJobExecutor: finally: reset_current_user(token) + async def _heartbeat( + self, job_id: str, lease_owner: str, main_task: asyncio.Task + ) -> None: + """定期续租：把 ``lease_until`` 推到 now + TTL，声明「本 worker 仍在跑、仍持租」。 + + 只要主任务活着且续租成功，别的 worker 的 dispatcher 看到租约未过期就不会抢；主任务 + 结束（或本协程被 cancel）后不再续租，租约自然到期。续租失败（租约已被接管 / 状态已变） + 说明本 worker 已不是持租者，必须立即停止本地编排，避免过期 worker 继续消耗模型资源； + 其任何迟到进度写也会被租约门拒绝。 + """ + try: + while not main_task.done(): + await asyncio.sleep(_HEARTBEAT_INTERVAL) + if main_task.done(): + return + try: + ok = await self._store.renew_lease( + job_id, + lease_owner=lease_owner, + lease_until=datetime.now(UTC) + timedelta(seconds=_LEASE_TTL_SECONDS), + ) + except Exception: # noqa: BLE001 — DB 抖动不中断心跳，下轮重试 + logger.warning("roundtable job %s renew_lease failed", job_id, exc_info=True) + continue + if not ok: + logger.info( + "roundtable job %s: renew_lease rejected (lease lost or status changed)", + job_id, + ) + if not main_task.done(): + main_task.cancel() + return + except asyncio.CancelledError: + # 编排结束 → _run 的 finally cancel 了心跳 → 安静退出。 + return + + async def _watch_cancellation( + self, job_id: str, main_task: asyncio.Task, lease_owner: str | None + ) -> None: + """定期轮询 DB 检测跨进程取消标志（``cancel_requested``），并以持租者身份收口。 + + 多 worker 下 cancel 请求可能打到**非执行** worker，该 worker 的 ``cancel()`` 在 + ``_tasks`` 里找不到此 job（返回 False）。但 cancel 端点始终把 ``status='cancel_requested'`` + 写进 DB（权威意图）。本看门狗跑在**实际执行作业的 worker** 上，检测到后： + + 1. 以**持租者**身份 ``finalize_cancel``——条件 UPDATE 把 cancel_requested 收口为终态 + ``cancelled``，同时清唯一活跃键 + 租约（一致性来自 DB，绝不双收口）； + 2. 对本地编排 task 调 ``cancel()``，让 ``run_orchestration`` 在下一个 await 点抛 + ``CancelledError`` 干净退出。 + + DB 读失败时静默重试（不中断看门狗）。兼容旧数据：若 DB 里已是终态 ``cancelled`` + （旧版 cancel 端点直写），直接取消本地 task。 + """ + try: + while not main_task.done(): + await asyncio.sleep(_CANCEL_POLL_INTERVAL) + if main_task.done(): + return + try: + # user_id=None → 不按 user 过滤，纯读 status（看门狗不关心归属）。 + row = await self._store.get(job_id, user_id=None) + except Exception: # noqa: BLE001 — DB 读失败不中断看门狗，下轮重试 + continue + status = (row.get("status") or "") if row is not None else "" + if status == "cancel_requested": + logger.info( + "roundtable job %s: detected cancel_requested in DB, finalizing " + "cancelled and cancelling orchestration task", + job_id, + ) + # 以持租者身份收口；非持租者（理论上不该发生）则退化为无租约条件收口。 + try: + await self._store.finalize_cancel(job_id, lease_owner=lease_owner) + except Exception: # noqa: BLE001 — 收口失败不阻断本地取消 + logger.warning("roundtable job %s finalize_cancel failed", job_id, exc_info=True) + if not main_task.done(): + main_task.cancel() + return + if status == "cancelled": + # 兼容：旧路径已直接把 cancelled 写进 DB —— 本地 task 照样停下。 + logger.info( + "roundtable job %s: detected cancelled in DB, cancelling orchestration task", + job_id, + ) + if not main_task.done(): + main_task.cancel() + return + except asyncio.CancelledError: + # 编排正常结束 → _run 的 finally cancel 了看门狗 → 安静退出。 + return + async def _write_run_to_draft( self, params: StartParams, @@ -338,77 +600,106 @@ class RoundtableJobExecutor: 读现有 step2 → 按 run_id=``job-`` upsert（保留其它手动 run）→ 设为 activeRun → update_draft。前端加载草稿即可像手动跑一样 hydrate 出对话。 + + 带乐观锁重试：**每次**读-改-写都带上 ``expected_version``，若并发写导致版本冲突 + （``DraftConcurrentWriteError``）则重新读取最新草稿、重新合并 run 再试。重试全部 + 耗尽后**绝不**退化为无版本检查的 last-write-wins 盲写，而是给作业置 + ``draft_persist_pending`` 标记（由 Phase 3 outbox 补偿器投影结果到草稿）—— + 消除后台作业与前台 PUT 并发时的 step2.runs 丢失更新，且一致性来自 DB 标记而非运气。 """ draft_store = self._draft_store_for(params) if draft_store is None or not params.draft_id: return + # 这些只依赖 params/dialogues，重试间不变——提前算好。 idx_by_agent = {a.agent_id: i for i, a in enumerate(params.agents)} s2_dialogues = [_to_step2_dialogue(d, idx_by_agent) for d in dialogues] last_leader = next( (d.get("content") for d in reversed(dialogues) if d.get("role") == "leader"), "" ) run_id = f"job-{params.job_id}" - try: - draft = await draft_store.get_draft(params.draft_id, params.user_id) - except Exception: # noqa: BLE001 - draft = None - step2 = dict((draft or {}).get("step2") or {}) - runs = list(step2.get("runs") or []) - idx = next((i for i, r in enumerate(runs) if r.get("id") == run_id), -1) - created = runs[idx].get("createdAt") if idx >= 0 else datetime.now(UTC).isoformat() - prev_step3 = runs[idx].get("step3") if idx >= 0 else None - # 本次有效 step3：新报告优先，否则保留上次（running 中间态 step3=None 不抹掉已存的）。 - effective_step3 = step3 if step3 is not None else prev_step3 - run = { - "id": run_id, - "title": "后台研讨", - "createdAt": created, - "source": params.mode, - "selectedAgents": [{"agent_id": a.agent_id, "name": a.name} for a in params.agents], - # 后台真建的 leader/seat 线程是真实持久的——存进去，前端加载后能像手动跑一样 - # 按 thread 打开各席位产物（HTTP outputs 兜底）。hydrate 是纯展示、绝不 resume， - # 不会触发重跑（init 闸门 + isRoundTableRunning=false 双重拦截）。空则存 None。 - "threadIds": (thread_ids or None), - "coordinatorName": params.coordinator_name, - "step2RoundtableDialogues": s2_dialogues, - "lastLeaderContent": last_leader or "", - "hasConsensus": status == "done", - "consensusPercentage": int(consensus), - "budgetLimit": 0, - # 从各席位 write_file 调用重建 stub，使加载草稿后点开产物的虚拟预览可用。 - "seatStubMessages": _seat_stub_messages(dialogues), - "orchestrationMode": params.mode, - "chain": params.chain, - # dag 模式分层计划：让前端加载草稿后能像手动跑一样按 stage 画流程图。 - "orchestrationPlan": params.orchestration_plan, - "step3": effective_step3, - "jobId": params.job_id, - "jobStatus": status, - } - runs = [run if i == idx else r for i, r in enumerate(runs)] if idx >= 0 else [*runs, run] - step2["runs"] = runs - step2["activeRunId"] = run_id - # 顶层 draft.step3 也写一份（同 Step2 把对话写进 run 的同一次落库一并带上）。 - # 前端 hydrateStep3 用**顶层** draft.step3 重建报告气泡（结果绘制的聊天记录 + HTML 预览）； - # 只存 run 内 step3 时，加载会被 hydrateStep3(draft.step3=None) 覆盖成空 → 报告聊天记录丢失。 - # furthest_step 跟随**是否真有 step3**：done 且有报告→3（落到结果绘制）； - # 否则停在 2（Step2 研讨）。后台禁用报告时 step3=None → 停在 Step2，用户可在前端手动画报告。 - has_report = effective_step3 is not None - update_kwargs: dict[str, Any] = {"step2": step2, "furthest_step": 3 if (status == "done" and has_report) else 2} - if has_report: - update_kwargs["step3"] = effective_step3 - try: - await draft_store.update_draft(params.draft_id, params.user_id, **update_kwargs) - except Exception as exc: # noqa: BLE001 — 写草稿失败不影响作业本身 - logger.warning("write run to draft %s failed", params.draft_id, exc_info=True) - from deerflow.persistence.roundtable_diagnostics import record_diagnostic - await record_diagnostic( - scope="background", stage="job", level="error", event="draft_write_failed", - message=f"研讨结果写回草稿失败（status={status}）：{exc}", - detail={"type": type(exc).__name__, "error": str(exc)}, - job_id=params.job_id, draft_id=params.draft_id, task_id=params.task_id, user_id=params.user_id, - ) + for attempt in range(_DRAFT_WRITE_RETRIES): + # 读最新草稿（冲突重试时重新读，拿到别的小伙伴刚写的 runs）。 + try: + draft = await draft_store.get_draft(params.draft_id, params.user_id) + except Exception: # noqa: BLE001 + draft = None + expected_version = (draft or {}).get("version") + + step2 = dict((draft or {}).get("step2") or {}) + runs = list(step2.get("runs") or []) + idx = next((i for i, r in enumerate(runs) if r.get("id") == run_id), -1) + created = runs[idx].get("createdAt") if idx >= 0 else datetime.now(UTC).isoformat() + prev_step3 = runs[idx].get("step3") if idx >= 0 else None + # 本次有效 step3：新报告优先，否则保留上次（running 中间态 step3=None 不抹掉已存的）。 + effective_step3 = step3 if step3 is not None else prev_step3 + run = { + "id": run_id, + "title": "后台研讨", + "createdAt": created, + "source": params.mode, + "selectedAgents": [{"agent_id": a.agent_id, "name": a.name} for a in params.agents], + "threadIds": (thread_ids or None), + "coordinatorName": params.coordinator_name, + "step2RoundtableDialogues": s2_dialogues, + "lastLeaderContent": last_leader or "", + "hasConsensus": status == "done", + "consensusPercentage": int(consensus), + "budgetLimit": 0, + "seatStubMessages": _seat_stub_messages(dialogues), + "orchestrationMode": params.mode, + "chain": params.chain, + "orchestrationPlan": params.orchestration_plan, + "step3": effective_step3, + "jobId": params.job_id, + "jobStatus": status, + } + runs = [run if i == idx else r for i, r in enumerate(runs)] if idx >= 0 else [*runs, run] + step2["runs"] = runs + step2["activeRunId"] = run_id + has_report = effective_step3 is not None + update_kwargs: dict[str, Any] = { + "step2": step2, + "furthest_step": 3 if (status == "done" and has_report) else 2, + } + if has_report: + update_kwargs["step3"] = effective_step3 + + # 每次尝试都带 expected_version（乐观锁）；绝不降级为无版本检查的盲写。 + try: + await draft_store.update_draft( + params.draft_id, params.user_id, + expected_version=expected_version, + **update_kwargs, + ) + return # 写入成功 + except DraftConcurrentWriteError: + if attempt < _DRAFT_WRITE_RETRIES - 1: + logger.debug( + "roundtable draft %s version conflict (attempt %d/%d), retrying", + params.draft_id, attempt + 1, _DRAFT_WRITE_RETRIES, + ) + continue + # 重试全部耗尽：不盲写，置 pending 标记交由 outbox 补偿器投影。 + logger.warning( + "roundtable draft %s: version conflicts exhausted after %d attempts, " + "marking job %s draft_persist_pending", + params.draft_id, _DRAFT_WRITE_RETRIES, params.job_id, + ) + with suppress(Exception): + await self._store.mark_draft_persist_pending(params.job_id) + return + except Exception as exc: # noqa: BLE001 — 写草稿失败不影响作业本身 + logger.warning("write run to draft %s failed", params.draft_id, exc_info=True) + from deerflow.persistence.roundtable_diagnostics import record_diagnostic + + await record_diagnostic( + scope="background", stage="job", level="error", event="draft_write_failed", + message=f"研讨结果写回草稿失败（status={status}）：{exc}", + detail={"type": type(exc).__name__, "error": str(exc)}, + job_id=params.job_id, draft_id=params.draft_id, task_id=params.task_id, user_id=params.user_id, + ) + return def cancel(self, job_id: str) -> bool: """请求取消一个在跑的作业。返回是否命中在跑任务。""" diff --git a/offline-backend-20260512/backend/app/gateway/routers/intent.py b/offline-backend-20260512/backend/app/gateway/routers/intent.py index 98ef5bed..1f3303ca 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/intent.py +++ b/offline-backend-20260512/backend/app/gateway/routers/intent.py @@ -52,10 +52,14 @@ SSE 协议（与前端 ``streamIntent`` 对齐） from __future__ import annotations +import hashlib import json import logging +import os import re +import uuid from collections.abc import AsyncIterator +from datetime import UTC, datetime, timedelta from typing import Any import httpx @@ -89,6 +93,37 @@ from deerflow.tools.builtins.clarification_utils import resolve_allow_multiple logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/intent", tags=["intent"]) +# ── Phase 4：意图回合跨 worker 单飞 + 幂等 ───────────────────────────────────── +# 进程内 RunManager 是每 worker 内存态，多 worker 下同 thread 的两个请求落到不同 +# worker 会各启一个 run（无 409）。单飞只能靠 DB：intent_turns 表对 (thread_id, +# client_turn_id) 的唯一键 + 条件 UPDATE 抢占。前端每回合生成稳定 client_turn_id， +# 重试 / 双击复用，杜绝「同一回合被启两个 run → 双发意图 / 重复澄清卡」。 +_WORKER_ID = f"w-{os.getpid()}-{uuid.uuid4().hex[:8]}" +_INTENT_TURN_LEASE_TTL_SECONDS = 120 + + +def _intent_message_hash(message: str, files: list, task_context: dict | None) -> str: + """回合内容指纹：message + files + task_context 的 sha256。 + + 检测「同一 client_turn_id 但内容变了」的客户端 bug（→ conflict）。文件按 + (filename, path) 归一，避免临时字段（size/status）造成假性不一致。 + """ + file_repr = sorted( + (str(f.get("filename") or ""), str(f.get("path") or "")) for f in (files or []) + ) + blob = json.dumps( + {"message": message, "files": file_repr, "task_context": task_context or {}}, + ensure_ascii=False, + sort_keys=True, + ) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +def _get_intent_turn_store(request: Request): + """意图回合单飞 store（未配置 DB 时为 None → 跳过单飞，走原流程）。""" + return getattr(request.app.state, "intent_turn_store", None) + + # 内置 agent 目录名，须与 ``.deer-flow/agents/roundtable-intent/`` 一致。 INTENT_AGENT_ID = "roundtable-intent" _SUPPORTED_INTENT_AGENT_IDS = frozenset({INTENT_AGENT_ID, POSITION_INTENT_AGENT_ID}) @@ -485,6 +520,60 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin # 因此既无须 trust_env=False 防代理，也不存在 loopback 40s 超时这种坑。 async with _make_loopback_client(request) as client: try: + # —— Phase 4：意图回合跨 worker 单飞 + 幂等 —— + # 同一 (thread_id, client_turn_id) 至多一个 run：重复 / 双击 / 网络重发 + # 命中既有行即幂等返回，绝不重复启 run（杜绝双发意图 / 重复澄清卡）。 + turn_store = _get_intent_turn_store(request) + client_turn_id = str(payload.get("client_turn_id") or "").strip() or uuid.uuid4().hex + msg_hash = _intent_message_hash(message, files, task_context) + claim_action = "run" # 无 store 时走原流程 + if turn_store is not None: + claim = await turn_store.try_claim( + thread_id=thread_id, + client_turn_id=client_turn_id, + message_hash=msg_hash, + lease_owner=_WORKER_ID, + lease_until=datetime.now(UTC) + + timedelta(seconds=_INTENT_TURN_LEASE_TTL_SECONDS), + ) + claim_action = claim.get("action") or "run" + if claim_action == "replay": + cached = (claim.get("row") or {}).get("result_json") + if isinstance(cached, dict): + yield f"data: {json.dumps(cached, ensure_ascii=False)}\n\n" + return + # 缓存缺失/损坏 → 降级重跑（极少见）。 + claim_action = "run" + elif claim_action == "duplicate": + # 既有回合仍在跑且租约有效：本请求是重复 in-flight，丢弃 + # （原请求正在流式吐结果）。前端按 duplicate 标记忽略。 + dup = {"status": "asking", "content": "", "summary": None, "duplicate": True} + yield f"data: {json.dumps(dup, ensure_ascii=False)}\n\n" + return + elif claim_action == "conflict": + err_conflict = { + "status": "error", + "content": "意图回合幂等键冲突：同一 client_turn_id 但消息内容不同。", + "summary": None, + } + yield f"data: {json.dumps(err_conflict, ensure_ascii=False)}\n\n" + return + + async def _record_terminal(frame: dict[str, Any]) -> None: + """终态帧落库（仅本 worker 赢得领取时），供后续重复请求幂等回放。""" + if turn_store is None or claim_action != "run": + return + try: + await turn_store.record_result( + thread_id=thread_id, + client_turn_id=client_turn_id, + lease_owner=_WORKER_ID, + status="error" if frame.get("status") == "error" else "done", + result_json=frame, + ) + except Exception: + logger.warning("intent_turn record_result failed", exc_info=True) + # —— 澄清轮次硬上限兜底 —— # 通用会商保持原来的 3 次 ask 工具调用上限；岗位会商允许一个 # AI turn 同时产生多张独立协助卡，所以必须按 AI 消息（一次 @@ -657,6 +746,7 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin "content": ready_text, "summary": summary, } + await _record_terminal(final_frame) yield f"data: {json.dumps(final_frame, ensure_ascii=False)}\n\n" return @@ -710,6 +800,7 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin "content": fallback_msg, "summary": None, } + await _record_terminal(final_frame) yield f"data: {json.dumps(final_frame, ensure_ascii=False)}\n\n" return @@ -724,6 +815,7 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin "clarifications": clarification_payloads, "summary": None, } + await _record_terminal(final_frame) yield f"data: {json.dumps(final_frame, ensure_ascii=False)}\n\n" return @@ -733,6 +825,7 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin "content": content, "summary": None, } + await _record_terminal(final_frame) yield f"data: {json.dumps(final_frame, ensure_ascii=False)}\n\n" except HTTPException as exc: @@ -766,6 +859,7 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin "summary": None, "error_detail": error_detail, } + await _record_terminal(err) yield f"data: {json.dumps(err, ensure_ascii=False)}\n\n" except Exception as exc: diagnostic = _diagnose_exception(exc, context="intent_stream") @@ -781,6 +875,7 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin "summary": None, "error_detail": diagnostic, } + await _record_terminal(err) yield f"data: {json.dumps(err, ensure_ascii=False)}\n\n" return StreamingResponse( diff --git a/offline-backend-20260512/backend/app/gateway/routers/multi_agent.py b/offline-backend-20260512/backend/app/gateway/routers/multi_agent.py index 1ef19359..fcac9393 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/multi_agent.py +++ b/offline-backend-20260512/backend/app/gateway/routers/multi_agent.py @@ -1116,9 +1116,16 @@ async def init_structure_thread(request: Request) -> dict[str, str]: # 回环 /runs/stream 在**响应头阶段**返回这些状态码时,视为「单 worker + 单 sqlite 锁被 # 并发打满」造成的瞬时失败,退避重试(详见 _stream_upstream 内注释): -# 409 同 thread 抢占竞态;429 限流;500/503/504 高并发下 start_run 里 DB/checkpoint 写抖动。 +# 429 限流;500/503/504 高并发下 start_run 里 DB/checkpoint 写抖动。 # 不含 400/401/403/404/422 等确定性错误——那些重试也没用,立即上报。 -_UPSTREAM_RETRYABLE_STATUSES = frozenset({409, 429, 500, 502, 503, 504}) +# +# ⚠️ 409 **不在**重试集合里。409 = 同 thread 已有活跃 run(``multitask_strategy=reject`` +# 拒绝)。此时重发**不是**幂等的:LangGraph 会把同一条 human message 再追加一次 → 用户 +# 这一轮被处理两遍(重复澄清卡 / 双发意图),或两次退避后仍 502 —— 这正是用户报的 +# 「确定意图接口报错」。故 409 必须透传:由上层(intent_turns 去重 / 前端按 client_turn_id +# 等待)决定是等待还是放弃,而不是盲目重发。其余(rollback)调用方本就靠抢占取消在跑 run, +# 几乎不触 409,移除亦无碍。 +_UPSTREAM_RETRYABLE_STATUSES = frozenset({429, 500, 502, 503, 504}) # 重试次数(不含首次)。总尝试 = 1 + _UPSTREAM_MAX_RETRIES。 _UPSTREAM_MAX_RETRIES = 2 # 指数退避基数(秒):0.4 → 0.8 → …,足够让在跑的席位 run 让出 sqlite 锁,又不至于明显拖慢。 diff --git a/offline-backend-20260512/backend/app/gateway/routers/position_roundtable.py b/offline-backend-20260512/backend/app/gateway/routers/position_roundtable.py index eacc2a33..7f201f35 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/position_roundtable.py +++ b/offline-backend-20260512/backend/app/gateway/routers/position_roundtable.py @@ -14,11 +14,28 @@ import asyncio from datetime import UTC, datetime from pathlib import Path from typing import Any +from uuid import uuid4 from fastapi import APIRouter, HTTPException, Query, Request from pydantic import BaseModel, Field, field_validator from app.gateway.path_utils import aresolve_thread_virtual_path +from deerflow.persistence.position_roundtable.delivery import ( + artifact_signature as _artifact_signature, +) +from deerflow.persistence.position_roundtable.delivery import ( + is_intent_position_node as _is_intent_position_by_id, +) +from deerflow.persistence.position_roundtable.delivery import ( + is_json_artifact as _is_json_artifact, +) +from deerflow.persistence.position_roundtable.delivery import ( + is_markdown_artifact as _is_markdown_artifact, +) +from deerflow.persistence.position_roundtable.delivery import ( + session_intent_position_id as _chain_intent_position_id, +) +from deerflow.persistence.position_roundtable.sql import SessionWriteConflictError from deerflow.runtime.user_context import get_effective_user_id # The application mounts this router below both `/api` (canonical) and the @@ -69,6 +86,8 @@ class NodeResponse(BaseModel): artifact_manifest: list[dict[str, Any]] = Field(default_factory=list) conversation_snapshot: dict[str, Any] | None = None revision: int = 0 + # 节点乐观锁版本号（任意字段变更自增；revision 仅计真实交付）。 + version: int = 0 upstream_revision: dict[str, Any] | None = None rejection_count: int = 0 last_rejection: dict[str, Any] | None = None @@ -96,6 +115,8 @@ class SessionResponse(BaseModel): action_plan_snapshot: dict[str, Any] | None = None action_plan_conversation: dict[str, Any] | None = None status: str + # Session 级乐观锁版本号：每次命令式写操作自增，前端可回传做 CAS。 + version: int = 0 created_at: str | None = None updated_at: str | None = None @@ -132,20 +153,26 @@ class ConversationStateUpdateRequest(BaseModel): thread_id: str | None = Field(default=None, max_length=128) conversation_snapshot: dict[str, Any] = Field(default_factory=dict) + # Client-generated idempotency key; one click/retry reuses the same id so + # the command executes exactly once. Absent → server generates one (legacy). + command_id: str | None = Field(default=None, max_length=64) class ActivateRequest(BaseModel): chain_id: str = Field(min_length=1, max_length=64) + command_id: str | None = Field(default=None, max_length=64) class NodeThreadRequest(BaseModel): thread_id: str = Field(min_length=1, max_length=128) + command_id: str | None = Field(default=None, max_length=64) class CompleteTurnRequest(BaseModel): latest_answer: str = Field(default="", max_length=500_000) artifact_manifest: list[dict[str, Any]] = Field(default_factory=list) upstream_revision: dict[str, Any] | None = None + command_id: str | None = Field(default=None, max_length=64) class RejectNodeRequest(BaseModel): @@ -154,6 +181,7 @@ class RejectNodeRequest(BaseModel): # default so neither field turns a valid rejection into a 422/400. title: str = Field(default="产物驳回", max_length=120) reason: str = Field(default="", max_length=5_000) + command_id: str | None = Field(default=None, max_length=64) class ActionPlanDictionaryOption(BaseModel): @@ -261,6 +289,38 @@ async def _require_writable_session( return store, user_id, session +def _command_id_or_new(command_id: str | None) -> str: + """Use the client's idempotency key, or mint one for legacy clients.""" + value = (command_id or "").strip() + return value or uuid4().hex + + +def _command_result_or_raise(result: dict[str, Any]) -> dict[str, Any]: + """Return an ok command result, or translate a rejected command into HTTP.""" + if result.get("ok"): + return result + raise HTTPException( + status_code=int(result.get("http_status") or 409), + detail=result.get("detail") or "命令执行被拒绝", + ) + + +async def _update_session_or_raise( + store: Any, + session_id: str, + user_id: str, + **changes: Any, +) -> dict[str, Any]: + """Apply a locked session snapshot write and map stale-state races to 409.""" + try: + row = await store.update_session(session_id, user_id, **changes) + except SessionWriteConflictError as exc: + raise HTTPException(status_code=409, detail=exc.detail) from exc + if row is None: + raise HTTPException(status_code=404, detail="Position roundtable session not found") + return row + + def _project_session(row: dict[str, Any]) -> SessionResponse: return SessionResponse(**row) @@ -400,18 +460,15 @@ def _is_intent_position_node( node: dict[str, Any], intent_position_id: str | None = None, ) -> bool: - position_id = str(node.get("position_id") or "").strip() - if intent_position_id: - return position_id == intent_position_id - # Existing persisted sessions predate the configurable directory. Retain - # their historical interpretation until they are archived/recreated. - return position_id.lower() in {"intelligence", "intel", "intent"} + """Node-dict wrapper over the shared harness decision (see delivery.py).""" + return _is_intent_position_by_id( + str(node.get("position_id") or "").strip(), intent_position_id + ) def _session_intent_position_id(session: dict[str, Any]) -> str | None: - chain = session.get("chain_snapshot") if isinstance(session.get("chain_snapshot"), dict) else {} - value = chain.get("_intent_position_id") - return value.strip() if isinstance(value, str) and value.strip() else None + """Session-dict wrapper over the shared harness decision (see delivery.py).""" + return _chain_intent_position_id(session.get("chain_snapshot")) async def _configured_intent_position_id(request: Request) -> str: @@ -437,62 +494,11 @@ async def _configured_intent_position_id(request: Request) -> str: return "intelligence" -def _is_markdown_artifact(artifact: Any) -> bool: - if not isinstance(artifact, dict): - return False - path = str(artifact.get("path") or artifact.get("artifact_url") or "").strip().lower() - name = str(artifact.get("name") or artifact.get("filename") or "").strip().lower() - mime = str(artifact.get("mime_type") or artifact.get("mimeType") or "").strip().lower() - if path and not path.replace("\\", "/").lstrip("/").startswith("mnt/user-data/outputs/"): - return False - return ( - "markdown" in mime - or path.endswith(".md") - or path.endswith(".markdown") - or name.endswith(".md") - or name.endswith(".markdown") - ) - - def _node_has_markdown_delivery(node: dict[str, Any]) -> bool: artifacts = node.get("artifact_manifest") return isinstance(artifacts, list) and any(_is_markdown_artifact(item) for item in artifacts) -def _markdown_delivery_changed( - previous_artifacts: Any, - incoming_artifacts: list[dict[str, Any]], -) -> bool: - """Whether this turn rewrote a Markdown deliverable, not just chatted. - - A completed node can receive normal follow-up Q&A. That must not bump its - effective delivery revision or invalidate every downstream stage unless a - Markdown report really changed. The standard artifact listing includes - `modified_at` and `size_bytes`, which together make a stable lightweight - version marker without reading a potentially large file again. - """ - old = previous_artifacts if isinstance(previous_artifacts, list) else [] - old_signatures = { - ( - str(item.get("path") or item.get("artifact_url") or ""), - str(item.get("modified_at") or item.get("modifiedAt") or ""), - str(item.get("size_bytes") or item.get("sizeBytes") or ""), - ) - for item in old - if isinstance(item, dict) and _is_markdown_artifact(item) - } - new_signatures = { - ( - str(item.get("path") or item.get("artifact_url") or ""), - str(item.get("modified_at") or item.get("modifiedAt") or ""), - str(item.get("size_bytes") or item.get("sizeBytes") or ""), - ) - for item in incoming_artifacts - if isinstance(item, dict) and _is_markdown_artifact(item) - } - return bool(new_signatures - old_signatures) - - def _intent_string_list(intent: dict[str, Any], key: str) -> list[str]: """Return a clean, display-safe string list from a persisted intent field.""" value = intent.get(key) @@ -907,26 +913,6 @@ def _source_revisions(nodes: list[dict[str, Any]]) -> dict[str, int]: return {str(node["node_key"]): int(node.get("revision") or 0) for node in _delivered_nodes(nodes)} -def _is_json_artifact(artifact: Any) -> bool: - if not isinstance(artifact, dict): - return False - path = str(artifact.get("path") or artifact.get("artifact_url") or "").strip().lower() - name = str(artifact.get("name") or artifact.get("filename") or "").strip().lower() - mime = str(artifact.get("mime_type") or artifact.get("mimeType") or "").strip().lower() - if path and not path.replace("\\", "/").lstrip("/").startswith("mnt/user-data/outputs/"): - return False - return "json" in mime or path.endswith(".json") or name.endswith(".json") - - -def _artifact_signature(artifact: dict[str, Any]) -> tuple[str, str, str]: - """Use server-returned path/mtime/size to detect an actually rewritten file.""" - return ( - str(artifact.get("path") or artifact.get("artifact_url") or ""), - str(artifact.get("modified_at") or artifact.get("modifiedAt") or ""), - str(artifact.get("size_bytes") or artifact.get("sizeBytes") or ""), - ) - - def _artifact_manifest_refs(artifacts: list[dict[str, Any]]) -> list[dict[str, Any]]: """Persist display metadata plus any database recovery payload.""" refs: list[dict[str, Any]] = [] @@ -1276,7 +1262,8 @@ async def update_task_snapshot( request: Request, session_id: str, body: TaskSnapshotUpdateRequest ) -> SessionResponse: store, user_id, current = await _require_writable_session(request, session_id) - row = await store.update_session( + row = await _update_session_or_raise( + store, session_id, user_id, task_snapshot=_task_snapshot_for_scope( @@ -1284,9 +1271,8 @@ async def update_task_snapshot( body.task_snapshot, task_scoped=bool(current.get("task_scoped")), ), + allowed_statuses={"intent_pending"}, ) - if row is None: - raise HTTPException(status_code=404, detail="Position roundtable session not found") return _project_session(row) @@ -1295,15 +1281,15 @@ async def update_intent_snapshot( request: Request, session_id: str, body: IntentUpdateRequest ) -> SessionResponse: store, user_id, _ = await _require_writable_session(request, session_id) - row = await store.update_session( + row = await _update_session_or_raise( + store, session_id, user_id, intent_thread_id=body.intent_thread_id, intent_snapshot=body.intent_snapshot, intent_conversation=body.conversation_snapshot, + allowed_statuses={"intent_pending"}, ) - if row is None: - raise HTTPException(status_code=404, detail="Position roundtable session not found") return _project_session(row) @@ -1312,14 +1298,13 @@ async def update_summary_conversation( request: Request, session_id: str, body: ConversationStateUpdateRequest ) -> SessionResponse: store, user_id, _ = await _require_writable_session(request, session_id) - row = await store.update_session( + row = await _update_session_or_raise( + store, session_id, user_id, summary_thread_id=body.thread_id, summary_conversation=body.conversation_snapshot, ) - if row is None: - raise HTTPException(status_code=404, detail="Position roundtable session not found") return _project_session(row) @@ -1328,14 +1313,13 @@ async def update_action_plan_conversation( request: Request, session_id: str, body: ConversationStateUpdateRequest ) -> SessionResponse: store, user_id, _ = await _require_writable_session(request, session_id) - row = await store.update_session( + row = await _update_session_or_raise( + store, session_id, user_id, action_plan_thread_id=body.thread_id, action_plan_conversation=body.conversation_snapshot, ) - if row is None: - raise HTTPException(status_code=404, detail="Position roundtable session not found") return _project_session(row) @@ -1374,34 +1358,60 @@ async def activate_session( current.get("intent_snapshot"), intent_position_id=intent_position_id, ) - activated = await store.activate_session( - session_id, - user_id, - chain_id=body.chain_id, - chain_snapshot=chain_snapshot, - nodes=nodes, + # 命令式状态机：锁 session + 锁内判定「已激活→链相同幂等/链不同 409」， + # 消除原 activate_session 的 check-then-insert 竞态（并发双击 → 唯一键冲突 500）。 + result = _command_result_or_raise( + await store.apply_session_command( + session_id, + user_id, + command_id=_command_id_or_new(body.command_id), + command_type="activate", + payload={ + "chain_id": body.chain_id, + "chain_snapshot": chain_snapshot, + "nodes": nodes, + }, + ) + ) + return SessionDetailResponse( + **result["session"], + nodes=[_project_node(node) for node in result["nodes"]], ) - if activated is None: - raise HTTPException(status_code=404, detail="Position roundtable session not found") - row, node_rows = activated - return SessionDetailResponse(**row, nodes=[_project_node(node) for node in node_rows]) @router.post("/sessions/{session_id}/archive", response_model=SessionResponse) async def archive_session(request: Request, session_id: str) -> SessionResponse: - row = await _get_store(request).archive_session(session_id, _current_user_id(request)) - if row is None: - raise HTTPException(status_code=404, detail="Position roundtable session not found") - return _project_session(row) + store = _get_store(request) + user_id = _current_user_id(request) + # 命令式状态机：已归档幂等返回，锁内串行化并发归档。每次操作一个稳定的 + # command_id（URL 无 body），用 session 级固定派生键，重复请求天然重放。 + result = _command_result_or_raise( + await store.apply_session_command( + session_id, + user_id, + command_id=f"archive-{session_id}", + command_type="archive", + payload={}, + ) + ) + return _project_session(result["session"]) @router.delete("/sessions/{session_id}") async def delete_session(request: Request, session_id: str) -> dict[str, bool]: - deleted = await _get_store(request).delete_session( - session_id, _current_user_id(request) + store = _get_store(request) + user_id = _current_user_id(request) + # 命令式状态机：命令记录无外键（保留供重放），删除后同 command_id 重试 → 重放 + # ``{deleted: True}``，避免已删会话再次请求报 404 让前端误判失败。 + _command_result_or_raise( + await store.apply_session_command( + session_id, + user_id, + command_id=f"delete-{session_id}", + command_type="delete", + payload={}, + ) ) - if not deleted: - raise HTTPException(status_code=404, detail="Position roundtable session not found") return {"deleted": True} @@ -1418,23 +1428,19 @@ async def bind_node_thread( request: Request, session_id: str, node_key: str, body: NodeThreadRequest ) -> NodeResponse: store, user_id, _ = await _require_writable_session(request, session_id) - current = await store.get_node(session_id, node_key, user_id) - if current is None: - raise HTTPException(status_code=404, detail="Position roundtable node not found") - if current["status"] == "locked": - raise HTTPException(status_code=409, detail="上游节点尚未完成，当前节点未解锁") - if current.get("invalidated_by"): - raise HTTPException(status_code=409, detail="上游产物已被驳回或重做，当前节点需等待前序节点重新完成") - if current.get("thread_id") and current["thread_id"] != body.thread_id: - raise HTTPException(status_code=409, detail="该节点已绑定另一条对话线程") - row = await store.update_node( - session_id, - node_key, - user_id, - thread_id=body.thread_id, - status="running" if current.get("status") == "ready" else current.get("status", "locked"), + # 命令式状态机：锁内判定「是否解锁 / 是否已被其它线程占用」并绑定。 + # 原实现在事务外读取节点状态做判定（TOCTOU），并发下两个请求可同时通过校验。 + result = _command_result_or_raise( + await store.apply_node_command( + session_id, + node_key, + user_id, + command_id=_command_id_or_new(body.command_id), + command_type="bind", + payload={"thread_id": body.thread_id}, + ) ) - return _project_node(row) + return _project_node(result["node"]) @router.put( @@ -1448,25 +1454,21 @@ async def update_node_conversation( body: ConversationStateUpdateRequest, ) -> NodeResponse: store, user_id, session = await _require_writable_session(request, session_id) - current = await store.get_node(session_id, node_key, user_id) - if current is None: - raise HTTPException(status_code=404, detail="Position roundtable node not found") - if ( - body.thread_id - and current.get("thread_id") - and current["thread_id"] != body.thread_id - ): - raise HTTPException(status_code=409, detail="Position node is bound to another thread") - row = await store.update_node( - session_id, - node_key, - user_id, - thread_id=body.thread_id if body.thread_id else current.get("thread_id"), - conversation_snapshot=body.conversation_snapshot, + # 命令式状态机：线程归属校验移入 FOR UPDATE 锁内。 + result = _command_result_or_raise( + await store.apply_node_command( + session_id, + node_key, + user_id, + command_id=_command_id_or_new(body.command_id), + command_type="conversation", + payload={ + "thread_id": body.thread_id, + "conversation_snapshot": body.conversation_snapshot, + }, + ) ) - if row is None: - raise HTTPException(status_code=404, detail="Position roundtable node not found") - return _project_node(row) + return _project_node(result["node"]) @router.post("/sessions/{session_id}/nodes/{node_key}/context") @@ -1483,126 +1485,56 @@ async def complete_node_turn( node = await store.get_node(session_id, node_key, user_id) if node is None: raise HTTPException(status_code=404, detail="Position roundtable node not found") - if node["status"] == "locked": - raise HTTPException(status_code=409, detail="上游节点尚未完成，当前节点未解锁") - if node.get("invalidated_by"): - raise HTTPException(status_code=409, detail="上游产物已被驳回或重做，当前节点需等待前序节点重新完成") - # Artifact listing is best-effort on the client. Never erase a previously - # delivered report merely because one listing request transiently failed. - effective_artifacts = body.artifact_manifest or list(node.get("artifact_manifest") or []) + # 产物内容持久化是文件 IO，保持在数据库事务之外（避免长事务）。清单为空表示 + # 「本轮无新清单」，由锁内回退到节点当前清单——绝不在事务外用过期快照兜底。 + persisted: list[dict[str, Any]] | None = None if body.artifact_manifest: - effective_artifacts = await _persist_artifact_contents( + persisted = await _persist_artifact_contents( node.get("thread_id"), - effective_artifacts, + body.artifact_manifest, ) - has_markdown = any(_is_markdown_artifact(item) for item in effective_artifacts) - markdown_changed = _markdown_delivery_changed(node.get("artifact_manifest"), effective_artifacts) - requires_fresh_delivery = node.get("status") in {"rejected", "stale"} - next_status = ( - "done" - if _is_intent_position_node(node, _session_intent_position_id(session)) - else "done" - if has_markdown and (not requires_fresh_delivery or markdown_changed) - else "running" - ) - # Effective revisions are delivery revisions, not every conversational - # answer. This prevents a harmless follow-up on a completed seat from - # invalidating all later stages just because its old Markdown file still - # appears in the artifact list. - new_effective_delivery = ( - next_status == "done" - and ( - _is_intent_position_node(node, _session_intent_position_id(session)) - or node.get("status") != "done" - or markdown_changed + # 命令式状态机：能否完成 / 是否新交付 / revision+1 / 失效下游 / 解锁下一 stage， + # 全部在单一 FOR UPDATE 事务内基于锁定行判定并原子落库（apply_node_command）。 + # 原实现先读快照算好 next_status 再进事务，两个并发请求会基于同一过期快照各自 + # 判定，后写覆盖先写 → 下一 stage 卡 locked / invalidated_by 残留 → 断链。 + result = _command_result_or_raise( + await store.apply_node_command( + session_id, + node_key, + user_id, + command_id=_command_id_or_new(body.command_id), + command_type="complete", + payload={ + "latest_answer": body.latest_answer, + "artifact_manifest": persisted, + "upstream_revision": body.upstream_revision, + }, ) ) - row = await store.update_node( - session_id, - node_key, - user_id, - status=next_status, - latest_answer=body.latest_answer, - artifact_manifest=effective_artifacts, - upstream_revision=body.upstream_revision, - invalidated_by=None, - increment_revision=new_effective_delivery, - ) - if row is None: - raise HTTPException(status_code=404, detail="Position roundtable node not found") - - # A node has no coordinator. Its effective delivery alone drives the frozen - # chain: all parallel seats in the current stage must finish before the - # next stage is made writable. A plain conversational follow-up keeps the - # existing downstream deliveries valid; only a newly written Markdown - # report advances the revision and makes later stages stale. - nodes = await store.list_nodes(session_id, user_id) - if nodes is None: - raise HTTPException(status_code=404, detail="Position roundtable session not found") - if new_effective_delivery and node.get("revision", 0) > 0: - for downstream in nodes: - if ( - downstream["stage_index"] > row["stage_index"] - and downstream["status"] in {"ready", "running", "done", "stale", "rejected", "error"} - ): - has_material = ( - downstream.get("revision", 0) > 0 - or bool(downstream.get("latest_answer")) - or bool(downstream.get("artifact_manifest")) - ) - await store.update_node( - session_id, - downstream["node_key"], - user_id, - status="stale" if has_material else "locked", - invalidated_by=row["node_key"], - ) - nodes = await store.list_nodes(session_id, user_id) or [] - - current_stage = [item for item in nodes if item["stage_index"] == row["stage_index"]] - later_stages = sorted( - {item["stage_index"] for item in nodes if item["stage_index"] > row["stage_index"]} - ) - if row["status"] == "done" and current_stage and all(item["status"] == "done" for item in current_stage) and later_stages: - next_stage = later_stages[0] - for next_node in nodes: - if next_node["stage_index"] == next_stage and next_node["status"] in {"locked", "stale"}: - await store.update_node( - session_id, - next_node["node_key"], - user_id, - status="ready" if next_node["status"] == "locked" else "stale", - invalidated_by=None, - ) - return _project_node(row) + return _project_node(result["node"]) @router.post("/sessions/{session_id}/nodes/{node_key}/reject", response_model=SessionDetailResponse) async def reject_node( request: Request, session_id: str, node_key: str, body: RejectNodeRequest ) -> SessionDetailResponse: - title = body.title.strip() or "产物驳回" - reason = body.reason.strip() store, user_id, _ = await _require_writable_session(request, session_id) - current = await store.get_node(session_id, node_key, user_id) - if current is None: - raise HTTPException(status_code=404, detail="Position roundtable node not found") - if current["status"] == "locked": - raise HTTPException(status_code=409, detail="当前节点尚未解锁，暂无可驳回的交付产物") - if current.get("revision", 0) <= 0 and not current.get("latest_answer") and not current.get("artifact_manifest"): - raise HTTPException(status_code=409, detail="当前节点暂无可驳回的交付产物") - rejected = await store.reject_node( - session_id, - node_key, - user_id, - title=title, - reason=reason, - rejected_by=user_id, + # 命令式状态机：可驳回性判定 + 驳回 + 下游失效在同一 FOR UPDATE 事务内完成， + # 并 bump session/node version。同 command_id 的重复驳回原样重放、不重复计数。 + result = _command_result_or_raise( + await store.apply_node_command( + session_id, + node_key, + user_id, + command_id=_command_id_or_new(body.command_id), + command_type="reject", + payload={"title": body.title, "reason": body.reason}, + ) + ) + return SessionDetailResponse( + **result["session"], + nodes=[_project_node(node) for node in result["nodes"]], ) - if rejected is None: - raise HTTPException(status_code=404, detail="Position roundtable node not found") - row, node_rows = rejected - return SessionDetailResponse(**row, nodes=[_project_node(node) for node in node_rows]) async def _prepare_builtin_run( @@ -1706,7 +1638,8 @@ async def _complete_builtin_run( _artifact_signature(previous_primary) != _artifact_signature(new_primary) ) if kind == "summary": - row = await store.update_session( + row = await _update_session_or_raise( + store, session_id, user_id, summary_thread_id=body.thread_id, @@ -1717,15 +1650,14 @@ async def _complete_builtin_run( status="active", ) else: - row = await store.update_session( + row = await _update_session_or_raise( + store, session_id, user_id, action_plan_thread_id=body.thread_id, action_plan_snapshot=snapshot, status="completed", ) - if row is None: - raise HTTPException(status_code=404, detail="Position roundtable session not found") node_rows = await store.list_nodes(session_id, user_id) or [] return SessionDetailResponse(**row, nodes=[_project_node(node) for node in node_rows]) diff --git a/offline-backend-20260512/backend/app/gateway/routers/roundtable_artifact_submissions.py b/offline-backend-20260512/backend/app/gateway/routers/roundtable_artifact_submissions.py index 7a0afc20..3d7cdde9 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/roundtable_artifact_submissions.py +++ b/offline-backend-20260512/backend/app/gateway/routers/roundtable_artifact_submissions.py @@ -19,6 +19,7 @@ from pydantic import AliasChoices, BaseModel, ConfigDict, Field, StrictInt, Stri from app.gateway.deps import get_current_user from deerflow.persistence.roundtable_artifact_submissions import RoundtableArtifactSubmissionStore +from deerflow.persistence.roundtable_artifact_submissions.sql import SubmissionVersionConflict router = APIRouter(prefix="/api/roundtable/tasks", tags=["roundtable-artifact-submissions"]) @@ -53,6 +54,7 @@ class ArtifactSubmissionRequest(BaseModel): submitted: bool submission_id: StrictStr | None = Field(default=None, max_length=64, validation_alias=AliasChoices("submissionId", "submission_id")) task_delivery_id: StrictStr | None = Field(default=None, max_length=255, validation_alias=AliasChoices("taskDeliveryId", "task_delivery_id")) + expected_version: int | None = Field(default=None, ge=0, validation_alias=AliasChoices("expectedVersion", "expected_version")) artifact: ArtifactReferenceInput @field_validator("task_id", "submission_id", "task_delivery_id", mode="before") @@ -82,6 +84,7 @@ class ArtifactSubmissionResponse(BaseModel): id: str task_id: str = Field(serialization_alias="taskId") + version: int = Field(default=0) submitted: bool submitted_at: str | None = Field(default=None, serialization_alias="submittedAt") cancelled_at: str | None = Field(default=None, serialization_alias="cancelledAt") @@ -131,6 +134,7 @@ def _response(row: dict[str, Any]) -> ArtifactSubmissionResponse: return ArtifactSubmissionResponse( id=str(row["id"]), task_id=str(row["task_id"]), + version=int(row.get("version", 0)), submitted=bool(row["submitted"]), submitted_at=row.get("submitted_at"), cancelled_at=row.get("cancelled_at"), @@ -178,19 +182,43 @@ async def set_artifact_submission( # a thread reference a saved state could never be previewed correctly. if not artifact["thread_id"]: raise HTTPException(status_code=422, detail="artifact.threadId is required when submitted is true") - row = await store.submit( - path_task_id, - artifact, - task_delivery_id=body.task_delivery_id, - actor_id=actor_id, - ) + try: + row = await store.submit( + path_task_id, + artifact, + task_delivery_id=body.task_delivery_id, + actor_id=actor_id, + expected_version=body.expected_version, + ) + except SubmissionVersionConflict as conflict: + raise HTTPException( + status_code=409, + detail={ + "code": "version_conflict", + "message": "提交产物版本冲突：另一写入已先提交,请刷新后重试。", + "current_version": conflict.row.get("version", 0), + "submission": _response(conflict.row).model_dump(by_alias=True, exclude_none=False), + }, + ) from conflict else: - row = await store.cancel( - path_task_id, - artifact, - submission_id=body.submission_id, - actor_id=actor_id, - ) + try: + row = await store.cancel( + path_task_id, + artifact, + submission_id=body.submission_id, + actor_id=actor_id, + expected_version=body.expected_version, + ) + except SubmissionVersionConflict as conflict: + raise HTTPException( + status_code=409, + detail={ + "code": "version_conflict", + "message": "取消产物版本冲突：另一写入已先提交,旧取消不能撤销新提交。", + "current_version": conflict.row.get("version", 0), + "submission": _response(conflict.row).model_dump(by_alias=True, exclude_none=False), + }, + ) from conflict if row is None: raise HTTPException(status_code=404, detail="Artifact submission not found") return _response(row) diff --git a/offline-backend-20260512/backend/app/gateway/routers/roundtable_drafts.py b/offline-backend-20260512/backend/app/gateway/routers/roundtable_drafts.py index 75574acf..54d44aaf 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/roundtable_drafts.py +++ b/offline-backend-20260512/backend/app/gateway/routers/roundtable_drafts.py @@ -32,6 +32,7 @@ from uuid import uuid4 from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel, Field +from deerflow.persistence.roundtable_drafts.sql import DraftConcurrentWriteError from deerflow.runtime.user_context import get_effective_user_id logger = logging.getLogger(__name__) @@ -47,6 +48,8 @@ class DraftMetaResponse(BaseModel): task_id: str | None = None title: str = "" furthest_step: int = 1 + # 乐观锁版本号：PUT 时经 expected_version 回传，冲突 → 409 + current_version。 + version: int = 0 # 本次会商所用业务链名（历史卡片展示）；无链条（自由/recommend 模式）时为 None。 chain_title: str | None = None created_at: datetime | str | None = None @@ -88,6 +91,9 @@ class DraftUpdateRequest(BaseModel): step1: dict[str, Any] | None = None step2: dict[str, Any] | None = None step3: dict[str, Any] | None = None + # 乐观锁：客户端基于的草稿版本号。缺省（旧客户端）= 盲写兼容； + # 提供时版本不匹配 → 409，body 带 current_version 供重拉合并。 + expected_version: int | None = Field(default=None, ge=0) class RecommendPickSchema(BaseModel): @@ -204,7 +210,18 @@ async def update_draft(request: Request, draft_id: str, body: DraftUpdateRequest # Build kwargs that distinguish "omitted" from "explicitly None". The store # uses a sentinel; here we pass only the fields the client actually sent. fields = body.model_dump(exclude_unset=True) - row = await store.update_draft(draft_id, user_id, **fields) + try: + row = await store.update_draft(draft_id, user_id, **fields) + except DraftConcurrentWriteError as err: + # 版本冲突：detail 里带 current_version，前端据此重拉最新草稿、本地合并后重试。 + raise HTTPException( + status_code=409, + detail={ + "error": "draft_version_conflict", + "message": f"草稿已被并发修改（expected={err.expected}, actual={err.actual}）", + "current_version": err.actual, + }, + ) from err if row is None: raise HTTPException(status_code=404, detail="Draft not found") return DraftResponse(**row) diff --git a/offline-backend-20260512/backend/app/gateway/routers/roundtable_jobs.py b/offline-backend-20260512/backend/app/gateway/routers/roundtable_jobs.py index 5116d0b8..452716a2 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/roundtable_jobs.py +++ b/offline-backend-20260512/backend/app/gateway/routers/roundtable_jobs.py @@ -18,6 +18,7 @@ Routes（prefix ``/api/roundtable-jobs``）:: from __future__ import annotations import asyncio +import hashlib import json import logging import time @@ -25,11 +26,11 @@ from datetime import UTC, datetime, timedelta from typing import Any from uuid import uuid4 -from fastapi import APIRouter, HTTPException, Request +from fastapi import APIRouter, HTTPException, Request, Response from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field -from app.gateway.roundtable_job_executor import StartParams +from app.gateway.roundtable_job_executor import build_start_snapshot from deerflow.agents.roundtable_orchestrator import SeatRef from deerflow.runtime.user_context import get_effective_user_id @@ -54,6 +55,10 @@ class JobAgent(BaseModel): class StartJobRequest(BaseModel): + # Phase 3 客户端幂等键：前端为每次「后台挂起」生成一个 requestId，网络重试**复用同一 + # requestId**。服务端据此审计 / 排查；「同草稿不双跑」本身由 active_dedupe_key 唯一索引 + # 保证（requestId 只是锦上添花的请求级指纹）。缺省（旧前端）→ 服务端生成一个。 + requestId: str | None = Field(default=None, max_length=64) draftId: str | None = Field(default=None, max_length=64) # 外部任务 id（taskId 深链会商）。非空 = task 作业：按 task_id 共享、不按 user 分权， # 其 draft_id 指向独立的 roundtable_task_drafts 表（最终报告写回那里）。 @@ -151,6 +156,20 @@ def _get_executor(request: Request): return executor +def _get_dispatcher(request: Request): + """Phase 3 dispatcher（可能为 None：store 未配置时）。nudge 前先判空。""" + return getattr(request.app.state, "roundtable_job_dispatcher", None) + + +def _request_hash(body: StartJobRequest) -> str: + """启动入参指纹（sha256）——审计/排查「同 requestId 但入参变了」的异常重试。""" + canonical = json.dumps( + body.model_dump(mode="json", exclude={"requestId"}), + ensure_ascii=False, sort_keys=True, default=str, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + def _pending_nodes(agents: list[JobAgent]) -> list[dict[str, Any]]: """启动即落一份「全 pending」的派活链路，使 GET / 首帧 SSE 立刻能渲染流程图。""" return [ @@ -194,32 +213,63 @@ def _seat_refs_from_nodes(nodes: list[dict[str, Any]] | None) -> list[SeatRef]: @router.post("", response_model=RoundtableJobResponse, status_code=201) -async def start_job(request: Request, body: StartJobRequest) -> RoundtableJobResponse: +async def start_job( + request: Request, body: StartJobRequest, response: Response +) -> RoundtableJobResponse: + """启动一个后台作业（Phase 3：唯一活跃键去重 + dispatcher 领取执行）。 + + 流程： + 1. 把整份启动入参序列化成 ``input_snapshot`` 落库（跨 worker 重建 StartParams 的前提）； + 2. ``try_create_or_get_active`` 直接 INSERT 并占用 ``active_dedupe_key`` 唯一索引 —— + 冲突（同草稿已有活跃作业）即幂等返回它（``created=False`` → HTTP 200），不再新建； + 3. 新建成功（HTTP 201，status=queued）后 ``nudge`` dispatcher，由它 ``claim_job`` + 原子领取并在本（或其它）worker 执行 —— **不再由路由直接拉起进程内任务**。 + + 「同一草稿至多一个活跃作业」由 DB 唯一索引强制，跨 worker / 并发 / 重试天然互斥， + 不依赖 GET_LOCK 命名锁或进程内 asyncio。 + """ store = _get_store(request) - executor = _get_executor(request) user_id = _current_user_id(request) if not body.agents: raise HTTPException(status_code=400, detail="agents is required") - # 防重复挂起（服务端幂等闸门）：同一草稿已有未终态作业（queued/running/awaiting_input） - # 时直接返回它、不再新建。前端的「手动挂起按钮 / 离开页面自动挂起 / beforeunload 刷新 - # 兜底」可能对同一研讨多次触发本端点 —— 客户端去重只能挡住同页内的重复，跨触发源 / - # 跨标签页的并发只能靠这里兜住，否则同一研讨会被双跑。 task_id = (body.taskId or "").strip() or None - if body.draftId: - # task 作业按 task_id 去重（跨用户共享）；个人作业按 user 去重。 - existing = await store.get_by_draft(body.draftId, user_id=user_id, task_id=task_id) - if existing is not None and (existing.get("status") or "") not in _TERMINAL: - return _to_response(existing) - + draft_id = (body.draftId or "").strip() or None agents = [SeatRef(a.agentId, a.name or a.agentId) for a in body.agents] + seed_message = (body.seedMessage or body.intent or "").strip() + + # 整份启动入参快照 —— dispatcher 在任意 worker 领取后据此无损重建 StartParams。 + snapshot = build_start_snapshot( + user_id=user_id, + agents=agents, + seed_message=seed_message, + model=body.model, + mode=body.orchestrationMode, + intent_text=body.intent, + thread_ids=body.threadIds, + coordinator_name=body.coordinatorName, + draft_id=draft_id, + task_id=task_id, + is_resume=False, + chain=body.chain, + orchestration_plan=body.orchestrationPlan, + seat_thinking_enabled=body.thinking_enabled, + seat_reasoning_effort=body.reasoning_effort, + seat_subagent_enabled=body.subagent_enabled, + seat_skill_directives=dict(body.seat_skill_directives or {}), + seat_modes=dict(body.seat_modes or {}), + gather_first=bool(body.gather_first), + business_code=(body.business_code or None), + ) + request_id = (body.requestId or "").strip() or uuid4().hex + job_id = uuid4().hex - row = await store.create( - id=job_id, - draft_id=body.draftId, + row, created = await store.try_create_or_get_active( + draft_id=draft_id, user_id=user_id, task_id=task_id, + id=job_id, orchestration_mode=body.orchestrationMode, coordinator_name=body.coordinatorName, thread_ids=body.threadIds, @@ -227,32 +277,21 @@ async def start_job(request: Request, body: StartJobRequest) -> RoundtableJobRes orchestration_plan=body.orchestrationPlan, dispatch_chain=_pending_nodes(body.agents), status="queued", + request_id=request_id, + request_hash=_request_hash(body), + input_snapshot=snapshot, ) - - executor.start_job( - StartParams( - job_id=job_id, - user_id=user_id, - agents=agents, - seed_message=(body.seedMessage or body.intent or "").strip(), - model=body.model, - mode=body.orchestrationMode, - intent_text=body.intent, - thread_ids=body.threadIds, - coordinator_name=body.coordinatorName, - draft_id=body.draftId, - task_id=task_id, - chain=body.chain, - orchestration_plan=body.orchestrationPlan, - seat_thinking_enabled=body.thinking_enabled, - seat_reasoning_effort=body.reasoning_effort, - seat_subagent_enabled=body.subagent_enabled, - seat_skill_directives=dict(body.seat_skill_directives or {}), - seat_modes=dict(body.seat_modes or {}), - gather_first=bool(body.gather_first), - business_code=(body.business_code or None), - ) - ) + if created: + # 新建（queued）：交给 dispatcher 领取执行。 + response.status_code = 201 + else: + # 命中已有活跃作业：幂等返回（不重复建 / 不重复跑）。 + response.status_code = 200 + # 任一分支都唤醒 dispatcher：新建需被领取；幂等命中时若既有作业还在 queued、未被任何 + # worker 领走（如 dispatcher 正在轮询间隙），nudge 确保它被立刻 claim，无感延迟。 + dispatcher = _get_dispatcher(request) + if dispatcher is not None: + dispatcher.nudge() return _to_response(row) @@ -310,48 +349,73 @@ async def stream_job(request: Request, job_id: str, task_id: str | None = None) async def resume_job( request: Request, job_id: str, body: ResumeJobRequest, task_id: str | None = None ) -> RoundtableJobResponse: + """回答澄清后续跑（Phase 3：条件 UPDATE 串行 + dispatcher 领取）。 + + ``queue_for_resume`` 用**原子条件 UPDATE**（WHERE status='awaiting_input'）把作业翻成 + queued —— 并发 / 重复的 resume 只有一个能成功（其余返回 None → 409），杜绝「同一澄清 + 被续跑两次」的双跑。成功后清租约 + 写含续跑语义的新 snapshot，``nudge`` dispatcher 由 + 它在任意 worker 领取执行；**路由不再直接拉起进程内任务**。 + """ store = _get_store(request) - executor = _get_executor(request) tid = (task_id or "").strip() or None user_id = _current_user_id(request) row = await store.get(job_id, user_id=user_id, task_id=tid) if row is None: raise HTTPException(status_code=404, detail="Job not found") - if (row.get("status") or "") != "awaiting_input": - raise HTTPException(status_code=409, detail="Job is not awaiting input") answer = body.answer.strip() if not answer: raise HTTPException(status_code=400, detail="answer is required") - # task 作业不分权：放开写入归属校验（user_id=None）；并以**原作业所有者**身份续跑， - # 保证编排上下文 / 写回 task-draft 用一致的 user 上下文。 + # task 作业不分权：放开写入归属校验（user_id=None）。 write_user_id = None if tid else user_id run_user_id = row.get("user_id") or user_id - updated = await store.update_progress( - job_id, user_id=write_user_id, status="running", pending_clarification=None - ) - executor.start_job( - StartParams( - job_id=job_id, - user_id=run_user_id, - agents=_seat_refs_from_nodes(row.get("dispatch_chain")), - seed_message=_RESUME_PREFIX + answer, - model=None, - mode=row.get("orchestration_mode") or "recommend", - intent_text=answer, - thread_ids=row.get("thread_ids"), - coordinator_name=row.get("coordinator_name") or "roundtable-coordinator", - draft_id=row.get("draft_id"), - task_id=tid or row.get("task_id"), - is_resume=True, - chain=row.get("chain"), - orchestration_plan=row.get("orchestration_plan"), - ) + # 续跑 snapshot：优先复用原 input_snapshot（含席位/模式/链条等完整入参），覆盖续跑三件套 + # （is_resume / seed_message / intent_text），并用作业行里**最新**的 thread_ids / + # coordinator_name（首跑建线程后由执行器写回，比原 snapshot 里的更新）。旧作业无 snapshot + # 时从作业行字段兜底重建。 + base_snapshot = row.get("input_snapshot") or {} + if not isinstance(base_snapshot, dict): + base_snapshot = {} + resume_snapshot: dict[str, Any] = { + "v": base_snapshot.get("v", 1), + "user_id": run_user_id, + "agents": base_snapshot.get("agents") + or [{"agent_id": n.get("agentId"), "name": n.get("name")} for n in (row.get("dispatch_chain") or [])], + "seed_message": _RESUME_PREFIX + answer, + "model": base_snapshot.get("model"), + "mode": row.get("orchestration_mode") or base_snapshot.get("mode") or "recommend", + "intent_text": answer, + "thread_ids": row.get("thread_ids"), + "coordinator_name": row.get("coordinator_name") or "roundtable-coordinator", + "draft_id": row.get("draft_id"), + "task_id": tid or row.get("task_id"), + "is_resume": True, + "chain": row.get("chain"), + "orchestration_plan": row.get("orchestration_plan"), + "enable_report": base_snapshot.get("enable_report", True), + "seat_thinking_enabled": base_snapshot.get("seat_thinking_enabled"), + "seat_reasoning_effort": base_snapshot.get("seat_reasoning_effort"), + "seat_subagent_enabled": base_snapshot.get("seat_subagent_enabled", False), + "seat_skill_directives": base_snapshot.get("seat_skill_directives") or {}, + "seat_modes": base_snapshot.get("seat_modes") or {}, + "gather_first": base_snapshot.get("gather_first", False), + "business_code": base_snapshot.get("business_code"), + } + + updated = await store.queue_for_resume( + job_id, input_snapshot=resume_snapshot, user_id=write_user_id, task_id=tid ) - return _to_response(updated or row) + if updated is None: + # 条件 UPDATE 未命中：作业已不在 awaiting_input（被并发 resume 抢先 / 已续跑 / 已终态）。 + raise HTTPException(status_code=409, detail="Job is not awaiting input") + + dispatcher = _get_dispatcher(request) + if dispatcher is not None: + dispatcher.nudge() + return _to_response(updated) async def _require_admin(request: Request) -> None: @@ -395,8 +459,19 @@ async def admin_cleanup(request: Request, body: CleanupRequest) -> CleanupRespon @router.post("/{job_id}/cancel", response_model=RoundtableJobResponse) async def cancel_job(request: Request, job_id: str, task_id: str | None = None) -> RoundtableJobResponse: + """取消作业（Phase 3：两段式 cancel）。 + + 1. ``request_cancel``：原子条件 UPDATE（可取消态 → ``cancel_requested``）—— 权威意图落库， + 重复 cancel 幂等（已 cancel_requested/cancelled/终态时返回 None，直接回现状）。 + 2. **无持租者**（queued / awaiting_input，执行器未在跑）→ 路由立刻 ``finalize_cancel`` + 收口为终态 ``cancelled``（清唯一活跃键 + 租约）。 + 3. **有持租者**（running）→ 保留 ``cancel_requested``，由持租者的 DB 看门狗收口 + ``cancelled``；本接口先回 ``cancel_requested``（SSE 随后见翻转）。租约过期前未能 + 收口（例如 worker 崩溃）时，dispatcher 会回收该取消请求。 + + 「已取消」终态永远由持租者或 finalize 收口，避免与执行器的进度写竞态。 + """ store = _get_store(request) - executor = _get_executor(request) tid = (task_id or "").strip() or None user_id = _current_user_id(request) @@ -404,7 +479,20 @@ async def cancel_job(request: Request, job_id: str, task_id: str | None = None) if row is None: raise HTTPException(status_code=404, detail="Job not found") - executor.cancel(job_id) # task 作业不分权：放开写入归属校验。 - updated = await store.update_progress(job_id, user_id=(None if tid else user_id), status="cancelled") - return _to_response(updated or row) + write_user_id = None if tid else user_id + requested = await store.request_cancel(job_id, user_id=write_user_id, task_id=tid) + if requested is None: + # 已不可取消（终态 / 已 cancel_requested）：幂等返回现状，不当作错误。 + fresh = await store.get(job_id, user_id=user_id, task_id=tid) + return _to_response(fresh or row) + + if not requested.get("lease_owner"): + # 无持租者：路由直接收口（queued / awaiting_input）。 + finalized = await store.finalize_cancel(job_id) + return _to_response(finalized or requested) + + # 有持租者（running）：不要直接 task.cancel()。直接取消会中断当前 worker 的看门狗， + # 使 DB 永久遗留在 cancel_requested；让 DB 看门狗以持租者身份收口，崩溃时由 dispatcher + # 在租约到期后回收，才能保证跨进程取消最终完成。 + return _to_response(requested) diff --git a/offline-backend-20260512/backend/app/gateway/routers/roundtable_task_drafts.py b/offline-backend-20260512/backend/app/gateway/routers/roundtable_task_drafts.py index 1bea7a18..5b4b3574 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/roundtable_task_drafts.py +++ b/offline-backend-20260512/backend/app/gateway/routers/roundtable_task_drafts.py @@ -28,6 +28,7 @@ from uuid import uuid4 from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel, Field +from deerflow.persistence.roundtable_drafts.sql import DraftConcurrentWriteError from deerflow.runtime.user_context import get_effective_user_id logger = logging.getLogger(__name__) @@ -42,6 +43,8 @@ class TaskDraftMetaResponse(BaseModel): task_id: str title: str = "" furthest_step: int = 1 + # 乐观锁版本号：PUT 时经 expected_version 回传，冲突 → 409 + current_version。 + version: int = 0 # 本次会商所用业务链名（历史卡片展示）；无链条（自由/recommend 模式）时为 None。 chain_title: str | None = None created_at: datetime | str | None = None @@ -77,6 +80,9 @@ class TaskDraftUpdateRequest(BaseModel): step1: dict[str, Any] | None = None step2: dict[str, Any] | None = None step3: dict[str, Any] | None = None + # 乐观锁：客户端基于的草稿版本号。缺省（旧客户端）= 盲写兼容； + # 提供时版本不匹配 → 409，body 带 current_version 供重拉合并。 + expected_version: int | None = Field(default=None, ge=0) # ── helpers ────────────────────────────────────────────────────────────── @@ -175,7 +181,18 @@ async def get_draft(request: Request, draft_id: str) -> TaskDraftResponse: async def update_draft(request: Request, draft_id: str, body: TaskDraftUpdateRequest) -> TaskDraftResponse: store = _get_store(request) fields = body.model_dump(exclude_unset=True) - row = await store.update_draft(draft_id, **fields) + try: + row = await store.update_draft(draft_id, **fields) + except DraftConcurrentWriteError as err: + # 版本冲突：detail 里带 current_version，前端据此重拉最新草稿、本地合并后重试。 + raise HTTPException( + status_code=409, + detail={ + "error": "draft_version_conflict", + "message": f"草稿已被并发修改（expected={err.expected}, actual={err.actual}）", + "current_version": err.actual, + }, + ) from err if row is None: raise HTTPException(status_code=404, detail="Draft not found") return TaskDraftResponse(**row) diff --git a/offline-backend-20260512/backend/config.yaml b/offline-backend-20260512/backend/config.yaml index 2e7fd637..da97f4cd 100644 --- a/offline-backend-20260512/backend/config.yaml +++ b/offline-backend-20260512/backend/config.yaml @@ -364,16 +364,23 @@ auth_login: # 点击后在新标签页打开下面的 url。enabled 关掉则不显示该入口。 database: - backend: sqlite + backend: postgres + postgres_url: $POSTGRES_DATABASE_URL sqlite_dir: .deer-flow/data pool_size: 20 # 基础连接池(postgres/mysql 生效;sqlite 忽略) max_overflow: 20 # 高峰期可临时超出 pool_size 的额外连接数,真实上限 = pool_size + max_overflow pool_timeout: 30 # 池满时等待空闲连接的秒数,超时报错 + pool_recycle: 1800 # 连接回收周期(秒);防火墙/pgbouncer/MySQL wait_timeout 静默断连兜底。多 worker 部署:每 worker 上限 = pool_size + max_overflow,总数 × worker 数 < DB max_connections echo_sql: false checkpointer: - type: sqlite - connection_string: .deer-flow/data/deerflow.db + type: postgres + connection_string: $POSTGRES_CHECKPOINT_URL + use_postgres: true + # PostgreSQL 优先；SQLite 仅作历史记录读取兜底，确认稳定后可移除。 + dual_read_enabled: true + dual_read_sqlite_paths: + - .deer-flow/data/deerflow.db # 记忆子系统:跨会话持久化用户画像与 agent 私有记忆。 # 三层覆盖优先级(高->低): diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/roundtable_orchestrator/progress.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/roundtable_orchestrator/progress.py index b2977fde..732068f6 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/agents/roundtable_orchestrator/progress.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/roundtable_orchestrator/progress.py @@ -54,6 +54,7 @@ class JobProgressWriter: event_sink: EventSink = None, on_dialogue: Any | None = None, recorder: Any | None = None, + lease_owner: str | None = None, ) -> None: self._store = store self._job_id = job_id @@ -65,6 +66,9 @@ class JobProgressWriter: # 每轮 leader/seat 模型返回后回调（dialogues, consensus）—— 执行器据此把对话 # 写进草稿（和手动跑同一存取路径）。 self._on_dialogue = on_dialogue + # Phase 3 租约持有者：进度落库随 ``update_progress(lease_owner=...)`` 下发，store 的 + # 租约门只放行持租者的写（一致性来自 DB 条件 UPDATE）。None = 兼容旧路径（不走租约门）。 + self._lease_owner = lease_owner self._state: dict[str, str] = {a.agent_id: _PENDING for a in agents} self._spoke_cycle: dict[str, int] = {} self._consensus = 0 @@ -100,7 +104,11 @@ class JobProgressWriter: fields.setdefault("dialogues", self._dialogues) if self._store is not None: try: - await self._store.update_progress(self._job_id, **fields) + # 带租约持有者时走 store 的租约门（原子条件 UPDATE）；返回 None 表示写被拒 + # （非持租者 / 已取消）——进度落库 best-effort，不据此中断编排。 + await self._store.update_progress( + self._job_id, lease_owner=self._lease_owner, **fields + ) except Exception: # noqa: BLE001 — 进度落库失败不能打断编排 logger.warning("roundtable job progress persist failed", exc_info=True) if self._event_sink is not None: diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/database_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/database_config.py index 2ed2100c..f5bdc0ed 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/config/database_config.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/database_config.py @@ -91,6 +91,17 @@ class DatabaseConfig(BaseModel): "SQLAlchemy default is 30." ), ) + pool_recycle: int = Field( + default=1800, + description=( + "Recycle pooled connections after this many seconds (postgres/mysql only). " + "Prevents stale-connection errors when a firewall, pgbouncer, or MySQL " + "wait_timeout silently drops idle connections. pool_pre_ping catches most " + "cases, but pool_recycle is the belt-and-suspenders safety net. " + "Multi-worker sizing: each worker opens up to (pool_size + max_overflow) " + "connections; total across workers must stay below the DB's max_connections." + ), + ) # -- Derived helpers (not user-configured) -- diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/engine.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/engine.py index a426077a..e7addcc0 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/engine.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/engine.py @@ -130,7 +130,7 @@ async def _run_alembic_upgrade(url: str) -> None: try: await asyncio.wait_for(asyncio.to_thread(_do_upgrade), timeout=timeout_s) logger.info("Alembic migrations applied (or already up-to-date)") - except asyncio.TimeoutError: + except TimeoutError: logger.error( "Alembic upgrade did not finish within %.0fs; continuing startup on create_all schema. " "Likely cause: MySQL metadata-lock wait — check SHOW FULL PROCESSLIST for a blocking transaction.", @@ -213,12 +213,18 @@ def _ensure_orm_columns_sync(sync_conn) -> None: This sweep compares each existing table against the ORM metadata and issues ``ALTER TABLE … ADD COLUMN`` for every missing column that can be added safely (nullable, or carrying a server_default). Indexes covering a newly - added column are created as well. Works on sqlite/mysql/postgres — the - column DDL is compiled through the active dialect, so portable types - (``PortableJSON``/``PortableLongText``/``BeijingDateTime``) resolve to the - right backend type. Everything is per-column best-effort: a failure is - logged and startup continues. + added column are created as well. Table-level ``UniqueConstraint`` objects + that are declared on the model but missing from the table are also created + (as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations + emit too, and works on SQLite/MySQL/PostgreSQL alike) — without this a newly + declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent + on an existing database, weakening dedupe. + Works on sqlite/mysql/postgres — the column DDL is compiled through the + active dialect, so portable types (``PortableJSON``/``PortableLongText``/ + ``BeijingDateTime``) resolve to the right backend type. Everything is + per-column best-effort: a failure is logged and startup continues. """ + from sqlalchemy import UniqueConstraint from sqlalchemy import inspect as sa_inspect from sqlalchemy.schema import CreateColumn @@ -273,6 +279,48 @@ def _ensure_orm_columns_sync(sync_conn) -> None: except Exception: logger.exception("Failed to auto-create index %s on %s", idx.name, table.name) + # ── 表级 UNIQUE 约束补齐 ────────────────────────────────────────────────── + # create_all 只为全新表建 UniqueConstraint；既有表上新增声明的唯一约束（如 + # roundtable_jobs.active_dedupe_key）会静默缺失，去重/幂等的正确性就靠它，故 + # 这里显式补建。迁移可能用 CREATE UNIQUE INDEX（同名）实现唯一性，故同时查 + # get_unique_constraints 与 get_indexes 的名字集合，避免重复建。 + for table in Base.metadata.sorted_tables: + if table.name not in existing_tables: + continue + named_uqs = [ + c for c in table.constraints + if isinstance(c, UniqueConstraint) and c.name is not None + ] + if not named_uqs: + continue + try: + existing_uq_names = { + uq["name"] for uq in inspector.get_unique_constraints(table.name) if uq.get("name") + } + except Exception: + existing_uq_names = set() + try: + existing_idx_names = { + ix["name"] for ix in inspector.get_indexes(table.name) if ix.get("name") + } + except Exception: + existing_idx_names = set() + for uq in named_uqs: + if uq.name in existing_uq_names or uq.name in existing_idx_names: + continue + # 用 CREATE UNIQUE INDEX（而非 ALTER TABLE ADD CONSTRAINT）：迁移本身也用 + # CREATE UNIQUE INDEX，且它在 SQLite/MySQL/PostgreSQL 都受支持（SQLite 不 + # 支持 ALTER TABLE ADD CONSTRAINT），功能等价地强制唯一性。 + cols = ", ".join(preparer.quote(c.name) for c in uq.columns) + try: + sync_conn.exec_driver_sql( + f"CREATE UNIQUE INDEX {preparer.quote(uq.name)} " + f"ON {preparer.quote(table.name)} ({cols})" + ) + logger.info("Auto-added unique index %s on %s", uq.name, table.name) + except Exception: + logger.exception("Failed to add unique index %s on %s", uq.name, table.name) + def _is_mysql_unknown_database_error(exc: Exception) -> bool: message = str(exc).lower() @@ -287,6 +335,7 @@ async def init_engine( pool_size: int = 5, max_overflow: int = 10, pool_timeout: int = 30, + pool_recycle: int = 1800, sqlite_dir: str = "", ) -> None: """Create the async engine and session factory, then auto-create tables. @@ -298,6 +347,8 @@ async def init_engine( pool_size: Base connection pool size (postgres/mysql). max_overflow: Extra connections allowed beyond pool_size under load (postgres/mysql). pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql). + pool_recycle: Recycle connections after N seconds (postgres/mysql). Prevents stale + connections dropped by firewalls / MySQL wait_timeout / pgbouncer. sqlite_dir: Directory to create for SQLite (ensured to exist). """ global _engine, _session_factory, _engine_loop @@ -366,6 +417,7 @@ async def init_engine( max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, + pool_recycle=pool_recycle, json_serializer=_json_serializer, ) elif backend == "mysql": @@ -376,7 +428,7 @@ async def init_engine( max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, - pool_recycle=1800, + pool_recycle=pool_recycle, json_serializer=_json_serializer, ) else: @@ -407,7 +459,7 @@ async def init_engine( await _auto_create_postgres_db(url) # Rebuild engine against the now-existing database await _engine.dispose() - _engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, json_serializer=_json_serializer) + _engine = create_async_engine(url, echo=echo, pool_size=pool_size, max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, pool_recycle=pool_recycle, json_serializer=_json_serializer) _session_factory = async_sessionmaker(_engine, expire_on_commit=False) async with _engine.begin() as conn: await conn.run_sync(Base.metadata.create_all) @@ -423,7 +475,7 @@ async def init_engine( max_overflow=max_overflow, pool_timeout=pool_timeout, pool_pre_ping=True, - pool_recycle=1800, + pool_recycle=pool_recycle, json_serializer=_json_serializer, ) _session_factory = async_sessionmaker(_engine, expire_on_commit=False) @@ -460,6 +512,7 @@ async def init_engine_from_config(config) -> None: pool_size=config.pool_size, max_overflow=getattr(config, "max_overflow", 10), pool_timeout=getattr(config, "pool_timeout", 30), + pool_recycle=getattr(config, "pool_recycle", 1800), sqlite_dir=config.sqlite_dir if config.backend == "sqlite" else "", ) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/intent_turns/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/intent_turns/__init__.py new file mode 100644 index 00000000..14e473f1 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/intent_turns/__init__.py @@ -0,0 +1,25 @@ +"""Intent-turn single-flight persistence (Phase 4). + +Exposes a store factory; ``app/gateway/deps.py`` registers it on +``app.state.intent_turn_store`` at startup. The intent router reads it from +``request.app.state`` (no cross-boundary default needed — intent runs only in the +gateway process). +""" + +from __future__ import annotations + +from deerflow.persistence.intent_turns.model import IntentTurnRow +from deerflow.persistence.intent_turns.sql import IntentTurnRepository + + +def make_intent_turn_store(session_factory) -> IntentTurnRepository | None: + if session_factory is None: + return None + return IntentTurnRepository(session_factory) + + +__all__ = [ + "IntentTurnRow", + "IntentTurnRepository", + "make_intent_turn_store", +] diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/intent_turns/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/intent_turns/model.py new file mode 100644 index 00000000..a641b243 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/intent_turns/model.py @@ -0,0 +1,60 @@ +"""ORM model for intent-determination turn single-flight (Phase 4). + +一条 ``intent_turns`` 行 = 第一步「确定意图」对话的**一个回合**对 ``(thread_id, +client_turn_id)`` 的占用记录。前端为每个回合生成稳定的 ``client_turn_id``，重试 / 双击 / +网络层重发都复用它；后端用 ``UNIQUE(thread_id, client_turn_id)`` 做跨 worker 单飞： + +* 同一 ``(thread_id, client_turn_id)`` 至多一条行 —— INSERT 冲突即幂等命中： + - 已 ``done`` → 原样回放缓存的终态帧，**不再启 run**（杜绝双发意图 / 重复澄清卡）； + - 在跑且租约未过期 → 该请求是重复的 in-flight 请求，丢弃（原请求仍在流式吐结果）； + - 在跑但租约过期（worker 崩溃）→ 抢占续跑； + - ``message_hash`` 不同（同一 client_turn_id 却是不同内容）→ 409 客户端 bug。 + +一致性只来自 DB 唯一键 + 条件 UPDATE（租约 CAS），不依赖进程内 ``RunManager`` +（后者是每进程内存态，多 worker 下看不到彼此的 run，无法单飞）。 +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from sqlalchemy import String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from deerflow.persistence.base import Base +from deerflow.persistence.types import BeijingDateTime, PortableLongText + + +class IntentTurnRow(Base): + __tablename__ = "intent_turns" + __table_args__ = ( + # 跨 worker 单飞的根：同一 (thread_id, client_turn_id) 至多一条。INSERT 冲突即幂等。 + UniqueConstraint("thread_id", "client_turn_id", name="uq_intent_turns_thread_client"), + ) + + id: Mapped[str] = mapped_column(String(64), primary_key=True) + thread_id: Mapped[str] = mapped_column(String(128), nullable=False, index=True) + # 前端每回合生成的稳定幂等键；重试 / 双击复用同一个。 + client_turn_id: Mapped[str] = mapped_column(String(64), nullable=False) + # 本回合 message 的 sha256：检测「同 client_turn_id 但内容变了」的客户端 bug（→ 409）。 + message_hash: Mapped[str] = mapped_column(String(64), nullable=False) + # running（在跑）/ done（已出终态帧，可回放）/ error。 + status: Mapped[str] = mapped_column( + String(16), nullable=False, default="running", server_default="running" + ) + # 终态帧的完整 JSON（done/error 时写入）：重复请求命中时原样回放，不再启 run。 + result_json: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) + error: Mapped[str | None] = mapped_column(Text, nullable=True) + # 租约：领走的 worker 标记；过期（worker 崩溃）后可被条件 UPDATE 抢占续跑。 + lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True) + lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True) + + created_at: Mapped[datetime] = mapped_column( + BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC) + ) + updated_at: Mapped[datetime] = mapped_column( + BeijingDateTime(), + nullable=False, + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + ) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/intent_turns/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/intent_turns/sql.py new file mode 100644 index 00000000..6cd58a3b --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/intent_turns/sql.py @@ -0,0 +1,225 @@ +"""SQLAlchemy-backed intent-turn single-flight storage (Phase 4). + +跨 worker 单飞 + 幂等回放，一致性只来自 DB（唯一键 + 条件 UPDATE），不依赖进程内 +``RunManager``（多 worker 下彼此不可见）。两条核心路径： + +* ``try_claim`` —— INSERT 占用 ``(thread_id, client_turn_id)``；唯一键冲突即幂等命中， + 按「已 done→回放 / 在跑且租约有效→丢弃 / 租约过期或 error→抢占续跑 / 内容不一致→409」分流。 +* ``record_result`` —— 持租者把终态帧写回（条件 UPDATE 要求租约匹配），清租约。 +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import Any +from uuid import uuid4 + +from sqlalchemy import select, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import async_sessionmaker + +from deerflow.persistence.intent_turns.model import IntentTurnRow + + +def _loads(value: Any) -> Any: + if isinstance(value, str) and value: + try: + return json.loads(value) + except (json.JSONDecodeError, TypeError): + return value + return value + + +class IntentTurnRepository: + def __init__(self, session_factory: async_sessionmaker) -> None: + self._sf = session_factory + + @staticmethod + def _row_to_dict(row: IntentTurnRow | None) -> dict[str, Any] | None: + if row is None: + return None + return { + "id": row.id, + "thread_id": row.thread_id, + "client_turn_id": row.client_turn_id, + "message_hash": row.message_hash, + "status": row.status, + "result_json": _loads(row.result_json), + "error": row.error, + "lease_owner": row.lease_owner, + "lease_until": row.lease_until.isoformat() if row.lease_until else None, + "created_at": row.created_at, + "updated_at": row.updated_at, + } + + async def get(self, thread_id: str, client_turn_id: str) -> dict[str, Any] | None: + stmt = select(IntentTurnRow).where( + IntentTurnRow.thread_id == thread_id, + IntentTurnRow.client_turn_id == client_turn_id, + ) + async with self._sf() as session: + row = (await session.execute(stmt)).scalar_one_or_none() + return self._row_to_dict(row) + + async def try_claim( + self, + *, + thread_id: str, + client_turn_id: str, + message_hash: str, + lease_owner: str, + lease_until: datetime, + ) -> dict[str, Any]: + """占用一个意图回合 —— 跨 worker 单飞 + 幂等的入口。 + + 返回 ``{"action": ..., "row": {...}}``，``action`` 取值： + + * ``"run"`` —— 本调用方赢得占用（新建 / 抢占续跑），应启 run 并在终态写回。 + * ``"replay"`` —— 该回合已 ``done``，原样回放 ``row.result_json``，**不再启 run**。 + * ``"duplicate"``—— 该回合仍在跑且租约有效（或刚被别处抢占），本请求是重复的 + in-flight 请求，应丢弃（原请求仍在流式吐结果）。 + * ``"conflict"`` —— 同一 ``client_turn_id`` 但 ``message_hash`` 不同 → 客户端 bug，409。 + """ + # 1) 先尝试 INSERT 占用。 + new_id = uuid4().hex + try: + async with self._sf() as session: + session.add( + IntentTurnRow( + id=new_id, + thread_id=thread_id, + client_turn_id=client_turn_id, + message_hash=message_hash, + status="running", + lease_owner=lease_owner, + lease_until=lease_until, + ) + ) + await session.commit() + row = await self.get(thread_id, client_turn_id) + return {"action": "run", "row": row} + except IntegrityError: + # 唯一键 (thread_id, client_turn_id) 冲突 → 幂等命中，读回既有行分流。 + pass + + existing = await self.get(thread_id, client_turn_id) + if existing is None: + # 极端竞态：冲突与回读之间行被清掉。降级为「可新建」再插一次（best-effort）。 + return {"action": "run", "row": None} + + # 2) 内容指纹不一致 → 客户端 bug（同 client_turn_id 却换了 message）。 + if existing.get("message_hash") != message_hash: + return {"action": "conflict", "row": existing} + + status = existing.get("status") or "running" + + # 3) 已完成 → 幂等回放终态帧。 + if status == "done": + return {"action": "replay", "row": existing} + + # 4) 在跑或出错 → 尝试条件 UPDATE 抢占续跑（error 强制；running 要求租约已过期）。 + reclaimed = await self._conditional_reclaim( + thread_id=thread_id, + client_turn_id=client_turn_id, + prev_status=status, + lease_owner=lease_owner, + lease_until=lease_until, + ) + if reclaimed is not None: + return {"action": "run", "row": reclaimed} + + # 5) 抢占失败：仍在跑且租约有效（或刚被别处抢占）→ 本请求是重复 in-flight，丢弃。 + return {"action": "duplicate", "row": existing} + + async def _conditional_reclaim( + self, + *, + thread_id: str, + client_turn_id: str, + prev_status: str, + lease_owner: str, + lease_until: datetime, + ) -> dict[str, Any] | None: + """原子条件 UPDATE 抢占：``error`` 强制续跑；``running`` 仅当租约已过期。 + + 成功则置 ``status='running'``、写新租约，返回行；否则 None（仍在跑 / 刚被抢占）。 + """ + now = datetime.now(UTC) + where = [ + IntentTurnRow.thread_id == thread_id, + IntentTurnRow.client_turn_id == client_turn_id, + ] + if prev_status == "error": + where.append(IntentTurnRow.status == "error") + else: + where.extend( + [ + IntentTurnRow.status == "running", + IntentTurnRow.lease_until.is_not(None), + IntentTurnRow.lease_until < now, + ] + ) + values = { + "status": "running", + "lease_owner": lease_owner, + "lease_until": lease_until, + "updated_at": now, + } + async with self._sf() as session: + result = await session.execute( + update(IntentTurnRow).where(*where).values(**values) + ) + if result.rowcount == 0: + return None + await session.commit() + row = ( + await session.execute( + select(IntentTurnRow).where( + IntentTurnRow.thread_id == thread_id, + IntentTurnRow.client_turn_id == client_turn_id, + ) + ) + ).scalar_one_or_none() + return self._row_to_dict(row) + + async def record_result( + self, + *, + thread_id: str, + client_turn_id: str, + lease_owner: str, + status: str, + result_json: Any = None, + error: str | None = None, + ) -> bool: + """持租者把终态帧写回（条件 UPDATE 要求租约匹配），并清租约。 + + 只有赢得 ``try_claim``（``action='run'``）的调用方才应调用，且 ``lease_owner`` + 必须与领取时一致 —— 防止「抢占续跑的新 worker」被「旧 worker 的迟到收尾」覆盖。 + """ + now = datetime.now(UTC) + values: dict[str, Any] = { + "status": status, + "lease_owner": None, + "lease_until": None, + "updated_at": now, + } + if result_json is not None: + values["result_json"] = json.dumps(result_json, ensure_ascii=False) + if error is not None: + values["error"] = error + where = [ + IntentTurnRow.thread_id == thread_id, + IntentTurnRow.client_turn_id == client_turn_id, + IntentTurnRow.lease_owner == lease_owner, + IntentTurnRow.status == "running", + ] + async with self._sf() as session: + result = await session.execute( + update(IntentTurnRow).where(*where).values(**values) + ) + if result.rowcount == 0: + return False + await session.commit() + return True diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_01_position_roundtable_node_version.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_01_position_roundtable_node_version.py new file mode 100644 index 00000000..3e492f69 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_01_position_roundtable_node_version.py @@ -0,0 +1,61 @@ +"""Add optimistic-lock version column to position_roundtable_nodes. + +Revision ID: 20260813_01 +Revises: 20260808_01 +Create Date: 2026-08-13 + +背景：``complete_node_turn`` 把「本节点 done → 下游 invalidate → 解锁下一 stage」 +拆成多个独立事务的 ``update_node``，并发下后写覆盖先写，节点状态被打错（卡 locked / +invalidated_by 残留），导致下一阶段 ``_handoff_context`` 拿不到上游产物（断链）。 +新增 ``version`` 列供 ``update_node`` 做乐观锁（``WHERE version=?``），并把状态转移 +收敛进单一 ``FOR UPDATE`` 事务（``recompute_chain_state``），彻底消除 TOCTOU。 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import inspect + +revision = "20260813_01" +down_revision = "20260808_01" +branch_labels = None +depends_on = None + +_TABLE = "position_roundtable_nodes" +_COLUMN = "version" + + +def _has_table(table_name: str) -> bool: + return table_name in inspect(op.get_bind()).get_table_names() + + +def _has_column(table_name: str, column_name: str) -> bool: + if not _has_table(table_name): + return False + return any( + column["name"] == column_name + for column in inspect(op.get_bind()).get_columns(table_name) + ) + + +def upgrade() -> None: + if not _has_table(_TABLE): + return + if not _has_column(_TABLE, _COLUMN): + op.add_column( + _TABLE, + sa.Column( + _COLUMN, + sa.Integer(), + nullable=False, + server_default="0", + ), + ) + + +def downgrade() -> None: + if not _has_table(_TABLE): + return + if _has_column(_TABLE, _COLUMN): + op.drop_column(_TABLE, _COLUMN) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_02_roundtable_drafts_version.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_02_roundtable_drafts_version.py new file mode 100644 index 00000000..e621c53d --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_02_roundtable_drafts_version.py @@ -0,0 +1,48 @@ +"""Add optimistic-lock ``version`` column to both roundtable draft tables. + +Revision ID: 20260813_02 +Revises: 20260813_01 +Create Date: 2026-08-13 + +为 ``roundtable_drafts``（个人草稿）与 ``roundtable_task_drafts``（任务草稿）各加一列 +``version INTEGER NOT NULL DEFAULT 0``，配合 ``update_draft(expected_version=...)`` 乐观锁， +消除后台作业 ``_write_run_to_draft`` 的 step2.runs 读-改-写丢失更新。 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import inspect + +revision = "20260813_02" +down_revision = "20260813_01" +branch_labels = None +depends_on = None + +_TABLES = ("roundtable_drafts", "roundtable_task_drafts") + + +def _has_column(table_name: str, column_name: str) -> bool: + bind = op.get_bind() + if table_name not in inspect(bind).get_table_names(): + return False + return any( + column["name"] == column_name + for column in inspect(bind).get_columns(table_name) + ) + + +def upgrade() -> None: + for table in _TABLES: + if not _has_column(table, "version"): + op.add_column( + table, + sa.Column("version", sa.Integer(), nullable=False, server_default="0"), + ) + + +def downgrade() -> None: + for table in _TABLES: + if _has_column(table, "version"): + op.drop_column(table, "version") diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_03_position_roundtable_session_version_commands.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_03_position_roundtable_session_version_commands.py new file mode 100644 index 00000000..b4b3f4cb --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_03_position_roundtable_session_version_commands.py @@ -0,0 +1,88 @@ +"""Add session version column and command idempotency table. + +Revision ID: 20260813_03 +Revises: 20260813_02 +Create Date: 2026-08-13 + +岗位会商并发根治（命令式状态机）的两项 DB 支撑： + +1. ``position_roundtable_sessions.version`` — session 级乐观锁版本号，保护 + 「确定意图 / 激活链 / 归档」等 session 级写操作，配合 ``apply_node_command`` + 的单一 ``FOR UPDATE`` 事务消除两个用户同时激活/归档同一 session 的竞态。 +2. ``position_roundtable_commands`` — 命令幂等记录表。网络重试 / 双击 / + beforeunload beacon 可能把同一命令送达多次；靠 ``UNIQUE(session_id, + node_key, command_id)`` 让数据库天然去重，第二次到达时直接返回上次 + ``result_json``，绝不重复推进状态机。``payload_hash`` 防止客户端误复用 + command_id 发送不同参数（同 command_id 但 hash 不同 → 409）。 + +``create_all`` 会为全新 DB 建表/补列；本迁移只补丁已存在的 DB。 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import inspect +from sqlalchemy.dialects.mysql import LONGTEXT + +revision = "20260813_03" +down_revision = "20260813_02" +branch_labels = None +depends_on = None + +_SESSIONS_TABLE = "position_roundtable_sessions" +_COMMANDS_TABLE = "position_roundtable_commands" + + +def _has_table(table_name: str) -> bool: + return table_name in inspect(op.get_bind()).get_table_names() + + +def _has_column(table_name: str, column_name: str) -> bool: + if not _has_table(table_name): + return False + return any( + column["name"] == column_name + for column in inspect(op.get_bind()).get_columns(table_name) + ) + + +def upgrade() -> None: + # 1) sessions.version + if _has_table(_SESSIONS_TABLE) and not _has_column(_SESSIONS_TABLE, "version"): + op.add_column( + _SESSIONS_TABLE, + sa.Column("version", sa.Integer(), nullable=False, server_default="0"), + ) + + # 2) position_roundtable_commands + # result_json 与模型的 PortableLongText 对齐：MySQL 用 LONGTEXT，其余用 TEXT。 + long_text = LONGTEXT() if op.get_bind().dialect.name == "mysql" else sa.Text() + if not _has_table(_COMMANDS_TABLE): + op.create_table( + _COMMANDS_TABLE, + sa.Column("id", sa.String(length=64), primary_key=True), + sa.Column("session_id", sa.String(length=64), nullable=False), + sa.Column("node_key", sa.String(length=256), nullable=False, server_default=""), + sa.Column("command_id", sa.String(length=64), nullable=False), + sa.Column("command_type", sa.String(length=32), nullable=False), + sa.Column("payload_hash", sa.String(length=64), nullable=False), + sa.Column("result_json", long_text, nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.UniqueConstraint( + "session_id", "node_key", "command_id", + name="uq_position_roundtable_commands_session_node_cmd", + ), + ) + op.create_index( + "ix_position_roundtable_commands_session_id", + _COMMANDS_TABLE, + ["session_id"], + ) + + +def downgrade() -> None: + if _has_table(_COMMANDS_TABLE): + op.drop_table(_COMMANDS_TABLE) + if _has_table(_SESSIONS_TABLE) and _has_column(_SESSIONS_TABLE, "version"): + op.drop_column(_SESSIONS_TABLE, "version") diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_04_roundtable_jobs_draft_persist_pending.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_04_roundtable_jobs_draft_persist_pending.py new file mode 100644 index 00000000..3155e80d --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_04_roundtable_jobs_draft_persist_pending.py @@ -0,0 +1,51 @@ +"""Add ``draft_persist_pending`` flag to roundtable jobs. + +Revision ID: 20260813_04 +Revises: 20260813_03 +Create Date: 2026-08-13 + +后台作业 ``_write_run_to_draft`` 的乐观锁重试全部耗尽后，不再退化为 +last-write-wins 盲写，而是把 ``draft_persist_pending`` 置 1；由 outbox 补偿器 +（Phase 3）负责投影结果到草稿，保证「研讨结果不丢」的一致性来自 DB 标记而非运气。 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import inspect + +revision = "20260813_04" +down_revision = "20260813_03" +branch_labels = None +depends_on = None + +_TABLE = "roundtable_jobs" + + +def _has_column(table_name: str, column_name: str) -> bool: + bind = op.get_bind() + if table_name not in inspect(bind).get_table_names(): + return False + return any( + column["name"] == column_name + for column in inspect(bind).get_columns(table_name) + ) + + +def upgrade() -> None: + if not _has_column(_TABLE, "draft_persist_pending"): + op.add_column( + _TABLE, + sa.Column( + "draft_persist_pending", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + + +def downgrade() -> None: + if _has_column(_TABLE, "draft_persist_pending"): + op.drop_column(_TABLE, "draft_persist_pending") diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_05_roundtable_jobs_lease_dedupe.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_05_roundtable_jobs_lease_dedupe.py new file mode 100644 index 00000000..1c7b1b3e --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_05_roundtable_jobs_lease_dedupe.py @@ -0,0 +1,115 @@ +"""Add lease / dedupe / snapshot columns to roundtable jobs (Phase 3). + +Revision ID: 20260813_05 +Revises: 20260813_04 +Create Date: 2026-08-13 + +把「后台作业不双跑 / 崩溃可接管 / resume·cancel 串行」的一致性收敛到 DB： + +- ``active_dedupe_key``：**唯一索引**。作业活跃（非终态）时存 + ``sha256(scope)``（scope = draft_id + task_id|user_id），进入终态时清 NULL。 + MySQL/SQLite 的唯一索引都允许多个 NULL —— 于是「同一 scope 至多一个活跃作业」 + 由唯一键强制，INSERT 冲突即幂等命中，**不再依赖 GET_LOCK 命名锁**。 +- ``request_id`` / ``request_hash``：前端启动请求的幂等键与入参指纹（审计 / + 排查用；去重本身由 active_dedupe_key 负责）。 +- ``input_snapshot``：启动入参的完整 JSON 快照 —— dispatcher 在**任意 worker** + 领取 queued/租约过期作业时据此重建 StartParams（跨进程可接管的前提）。 +- ``lease_owner`` / ``lease_until``：租约。条件 UPDATE 抢占（queued 或租约过期）， + 心跳续租；worker 崩溃后租约自然过期，其它 worker 接管 —— 恢复能力来自 DB 而非 + 进程内存。 +- ``attempt`` / ``version``：接管次数计数 + 行级 CAS 版本（条件 UPDATE 乐观锁）。 + +``draft_persist_pending`` 已在 20260813_04 加入，此处不重复。 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import inspect +from sqlalchemy.dialects import mysql + +revision = "20260813_05" +down_revision = "20260813_04" +branch_labels = None +depends_on = None + +_TABLE = "roundtable_jobs" +_DEDUPE_INDEX = "uq_roundtable_jobs_active_dedupe_key" + + +def _has_table(table_name: str) -> bool: + return table_name in inspect(op.get_bind()).get_table_names() + + +def _has_column(table_name: str, column_name: str) -> bool: + if not _has_table(table_name): + return False + return any( + column["name"] == column_name + for column in inspect(op.get_bind()).get_columns(table_name) + ) + + +def _has_index(table_name: str, index_name: str) -> bool: + if not _has_table(table_name): + return False + try: + return any(ix["name"] == index_name for ix in inspect(op.get_bind()).get_indexes(table_name)) + except Exception: # pragma: no cover - 方言差异兜底 + return False + + +def _longtext() -> sa.types.TypeEngine: + """SQLite/PostgreSQL 用 TEXT（无上限），MySQL 用 LONGTEXT（避免 64KB 截断）。""" + return sa.Text().with_variant(mysql.LONGTEXT(), "mysql") + + +def upgrade() -> None: + if not _has_table(_TABLE): + return + + if not _has_column(_TABLE, "request_id"): + op.add_column(_TABLE, sa.Column("request_id", sa.String(64), nullable=True)) + if not _has_column(_TABLE, "request_hash"): + op.add_column(_TABLE, sa.Column("request_hash", sa.String(64), nullable=True)) + if not _has_column(_TABLE, "active_dedupe_key"): + op.add_column(_TABLE, sa.Column("active_dedupe_key", sa.String(64), nullable=True)) + if not _has_index(_TABLE, _DEDUPE_INDEX): + op.create_index(_DEDUPE_INDEX, _TABLE, ["active_dedupe_key"], unique=True) + if not _has_column(_TABLE, "input_snapshot"): + op.add_column(_TABLE, sa.Column("input_snapshot", _longtext(), nullable=True)) + if not _has_column(_TABLE, "lease_owner"): + op.add_column(_TABLE, sa.Column("lease_owner", sa.String(64), nullable=True)) + if not _has_column(_TABLE, "lease_until"): + op.add_column(_TABLE, sa.Column("lease_until", sa.DateTime(), nullable=True)) + if not _has_column(_TABLE, "attempt"): + op.add_column( + _TABLE, + sa.Column("attempt", sa.Integer(), nullable=False, server_default=sa.text("0")), + ) + if not _has_column(_TABLE, "version"): + op.add_column( + _TABLE, + sa.Column("version", sa.Integer(), nullable=False, server_default=sa.text("0")), + ) + + +def downgrade() -> None: + if not _has_table(_TABLE): + return + + if _has_index(_TABLE, _DEDUPE_INDEX): + op.drop_index(_DEDUPE_INDEX, table_name=_TABLE) + for col in ( + "version", + "attempt", + "lease_until", + "lease_owner", + "input_snapshot", + "active_dedupe_key", + "request_hash", + "request_id", + ): + if _has_column(_TABLE, col): + op.drop_column(_TABLE, col) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_06_intent_turns.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_06_intent_turns.py new file mode 100644 index 00000000..36350fa1 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_06_intent_turns.py @@ -0,0 +1,77 @@ +"""intent_turns: intent-determination turn single-flight table (Phase 4). + +Revision ID: 20260813_06 +Revises: 20260813_05 +Create Date: 2026-08-13 + +新建 ``intent_turns`` 表：第一步「确定意图」对话每回合对 ``(thread_id, client_turn_id)`` +的占用记录。前端每回合生成稳定 ``client_turn_id``（重试 / 双击复用），后端用 +``UNIQUE(thread_id, client_turn_id)`` 跨 worker 单飞 + 幂等回放，杜绝并发下「同一回合 +被启两个 run → 双发意图 / 重复澄清卡」（用户报的「确定意图接口报错」）。 + +为什么需要它：进程内 ``RunManager`` 是每 worker 内存态，多 worker 下看不到彼此的 run， +同 thread 的两个请求若落到不同 worker 会各启一个 run（没有任何 409）—— 单飞只能靠 DB。 + +防变动既有库：全新部署由 ``create_all`` 建表；既有库由本迁移 + ``_ensure_orm_columns_sync`` +兜底（表不存在则建）。所有步骤带 ``_has_table``/``_has_index`` 守卫，可重复执行。 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "20260813_06" +down_revision = "20260813_05" +branch_labels = None +depends_on = None + +_TABLE = "intent_turns" +_UQ = "uq_intent_turns_thread_client" + + +def _has_table(name: str) -> bool: + bind = op.get_bind() + insp = sa.inspect(bind) + return name in insp.get_table_names() + + +def _has_index(table: str, name: str) -> bool: + bind = op.get_bind() + insp = sa.inspect(bind) + return any(ix["name"] == name for ix in insp.get_indexes(table)) + + +def upgrade() -> None: + if _has_table(_TABLE): + # 既有库（极少见，纯防御）：补唯一索引即可。 + if not _has_index(_TABLE, _UQ): + op.create_index(_UQ, _TABLE, ["thread_id", "client_turn_id"], unique=True) + return + + op.create_table( + _TABLE, + sa.Column("id", sa.String(length=64), nullable=False), + sa.Column("thread_id", sa.String(length=128), nullable=False), + sa.Column("client_turn_id", sa.String(length=64), nullable=False), + sa.Column("message_hash", sa.String(length=64), nullable=False), + sa.Column("status", sa.String(length=16), nullable=False, server_default=sa.text("running")), + sa.Column("result_json", sa.Text().with_variant(sa.dialects.mysql.LONGTEXT(), "mysql"), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("lease_owner", sa.String(length=64), nullable=True), + sa.Column("lease_until", sa.DateTime(), nullable=True), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_intent_turns_thread_id", _TABLE, ["thread_id"]) + op.create_index(_UQ, _TABLE, ["thread_id", "client_turn_id"], unique=True) + + +def downgrade() -> None: + if _has_index(_TABLE, _UQ): + op.drop_index(_UQ, table_name=_TABLE) + if _has_index(_TABLE, "ix_intent_turns_thread_id"): + op.drop_index("ix_intent_turns_thread_id", table_name=_TABLE) + if _has_table(_TABLE): + op.drop_table(_TABLE) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_07_artifact_submission_version.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_07_artifact_submission_version.py new file mode 100644 index 00000000..4a9bb57f --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260813_07_artifact_submission_version.py @@ -0,0 +1,42 @@ +"""Add version column to roundtable_artifact_submissions for CAS. + +Revision ID: 20260813_07 +Revises: 20260813_06 +Create Date: 2026-08-13 + +Phase 5 CAS: submit/cancel 带 expected_version,并发时旧 cancel 不得撤销新提交。 +version 列有 server_default='0',既有行自动得 0,无需数据回填。 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "20260813_07" +down_revision = "20260813_06" +branch_labels = None +depends_on = None + +_TABLE = "roundtable_artifact_submissions" +_COL = "version" + + +def _has_column(bind, table: str, column: str) -> bool: + inspector = sa.inspect(bind) + return any(col["name"] == column for col in inspector.get_columns(table)) + + +def upgrade() -> None: + bind = op.get_bind() + if not _has_column(bind, _TABLE, _COL): + op.add_column( + _TABLE, + sa.Column(_COL, sa.Integer(), nullable=False, server_default=sa.text("0")), + ) + + +def downgrade() -> None: + bind = op.get_bind() + if _has_column(bind, _TABLE, _COL): + op.drop_column(_TABLE, _COL) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py index fe13e6ed..48217797 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py @@ -40,6 +40,7 @@ from deerflow.persistence.concurrency.model import ConcurrencySampleRow from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow from deerflow.persistence.embed_sessions.model import EmbedSessionRow from deerflow.persistence.feedback.model import FeedbackRow +from deerflow.persistence.intent_turns.model import IntentTurnRow from deerflow.persistence.light_apps.model import LightAppRow from deerflow.persistence.llm_metrics.model import LlmCallMetricRow from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow @@ -147,6 +148,7 @@ __all__ = [ "TaskCopTaskRow", "TaskReportRow", "EmbedSessionRow", + "IntentTurnRow", "ThreadMetaRow", "ThreadShareRow", "ToolCallMetricRow", diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/__init__.py index eb01db3f..bc59080d 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/__init__.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/__init__.py @@ -3,12 +3,14 @@ from __future__ import annotations from deerflow.persistence.position_roundtable.model import ( + PositionRoundtableCommandRow, PositionRoundtableNodeRow, PositionRoundtableSessionRow, ) from deerflow.persistence.position_roundtable.sql import PositionRoundtableRepository __all__ = [ + "PositionRoundtableCommandRow", "PositionRoundtableNodeRow", "PositionRoundtableRepository", "PositionRoundtableSessionRow", diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/delivery.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/delivery.py new file mode 100644 index 00000000..893f44d8 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/delivery.py @@ -0,0 +1,117 @@ +"""Pure delivery-decision helpers for position-roundtable nodes. + +These functions decide whether a turn delivered a real Markdown report, whether +the delivery changed, and whether a node is the special intent position. They +are intentionally **pure** (no DB / no app imports) so the exact same logic can +run both: + +- in the API router (for read-only projections and validation), and +- inside ``PositionRoundtableRepository.apply_node_command`` — *under the row + locks* — where every state-machine decision must be computed from the locked + rows, never from a stale pre-transaction snapshot. + +Keeping one implementation avoids the classic drift where the router's decision +and the store's transition disagree under concurrency. +""" + +from __future__ import annotations + +from typing import Any + +_OUTPUTS_PREFIX = "mnt/user-data/outputs/" + + +def _artifact_path(artifact: dict[str, Any]) -> str: + return str(artifact.get("path") or artifact.get("artifact_url") or "").strip().lower() + + +def _artifact_name(artifact: dict[str, Any]) -> str: + return str(artifact.get("name") or artifact.get("filename") or "").strip().lower() + + +def _artifact_mime(artifact: dict[str, Any]) -> str: + return str(artifact.get("mime_type") or artifact.get("mimeType") or "").strip().lower() + + +def _in_outputs(path: str) -> bool: + """Only files under the thread outputs dir count as deliverables.""" + return not path or path.replace("\\", "/").lstrip("/").startswith(_OUTPUTS_PREFIX) + + +def is_markdown_artifact(artifact: Any) -> bool: + if not isinstance(artifact, dict): + return False + path = _artifact_path(artifact) + name = _artifact_name(artifact) + mime = _artifact_mime(artifact) + if not _in_outputs(path): + return False + return ( + "markdown" in mime + or path.endswith(".md") + or path.endswith(".markdown") + or name.endswith(".md") + or name.endswith(".markdown") + ) + + +def is_json_artifact(artifact: Any) -> bool: + if not isinstance(artifact, dict): + return False + path = _artifact_path(artifact) + name = _artifact_name(artifact) + mime = _artifact_mime(artifact) + if not _in_outputs(path): + return False + return "json" in mime or path.endswith(".json") or name.endswith(".json") + + +def artifact_signature(artifact: dict[str, Any]) -> tuple[str, str, str]: + """Use server-returned path/mtime/size to detect an actually rewritten file.""" + return ( + str(artifact.get("path") or artifact.get("artifact_url") or ""), + str(artifact.get("modified_at") or artifact.get("modifiedAt") or ""), + str(artifact.get("size_bytes") or artifact.get("sizeBytes") or ""), + ) + + +def markdown_delivery_changed( + previous_artifacts: Any, + incoming_artifacts: list[dict[str, Any]], +) -> bool: + """Whether this turn rewrote a Markdown deliverable, not just chatted. + + A completed node can receive normal follow-up Q&A. That must not bump its + effective delivery revision or invalidate every downstream stage unless a + Markdown report really changed. The standard artifact listing includes + ``modified_at`` and ``size_bytes``, which together make a stable lightweight + version marker without reading a potentially large file again. + """ + old = previous_artifacts if isinstance(previous_artifacts, list) else [] + old_signatures = { + artifact_signature(item) for item in old if is_markdown_artifact(item) + } + new_signatures = { + artifact_signature(item) for item in incoming_artifacts if is_markdown_artifact(item) + } + return bool(new_signatures - old_signatures) + + +def is_intent_position_node( + position_id: str | None, + intent_position_id: str | None = None, +) -> bool: + """Whether a node is the special intent position (answers need no report).""" + position = str(position_id or "").strip() + if intent_position_id: + return position == intent_position_id + # Existing persisted sessions predate the configurable directory. Retain + # their historical interpretation until they are archived/recreated. + return position.lower() in {"intelligence", "intel", "intent"} + + +def session_intent_position_id(chain_snapshot: Any) -> str | None: + """Read the frozen intent-position id from an activated chain snapshot.""" + chain = chain_snapshot if isinstance(chain_snapshot, dict) else {} + value = chain.get("_intent_position_id") + return value.strip() if isinstance(value, str) and value.strip() else None diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/model.py index 56419161..fd900dd0 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/model.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/model.py @@ -60,6 +60,10 @@ class PositionRoundtableSessionRow(Base): # intent_pending -> active -> completed / archived. The router owns legal # transitions; the DB keeps it as a concise, portable string. status: Mapped[str] = mapped_column(String(32), nullable=False, default="intent_pending", index=True) + # 乐观并发控制版本号：保护「确定意图 / 激活链 / 归档」等 session 级变更。 + # 每次 session 写操作自增；传入 expected_version 时做 ``WHERE version=?`` CAS， + # 命中 0 行即并发冲突（消除两个用户同时激活/归档同一 session 的竞态）。 + version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") created_at: Mapped[datetime] = mapped_column( BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC) ) @@ -101,6 +105,11 @@ class PositionRoundtableNodeRow(Base): # position Session removes every recovery snapshot through the FK cascade. conversation_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) revision: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + # 乐观并发控制版本号：每次 update_node 自增；传入 expected_version 时做 + # ``WHERE version=?`` 校验，命中 0 行即并发冲突。与 revision（业务交付计数） + # 刻意解耦——revision 仅在真正交付新 Markdown 报告时 +1，version 在任意字段 + # 变更时 +1，专用于消除「complete_node_turn 多步状态机」并发盲写丢失更新。 + version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") upstream_revision: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) rejection_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") last_rejection: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) @@ -114,3 +123,38 @@ class PositionRoundtableNodeRow(Base): default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), ) + + +class PositionRoundtableCommandRow(Base): + """命令幂等记录：同一次客户端操作（由 command_id 标识）只执行一次。 + + 网络重试 / 双击 / beforeunload beacon 可能把同一命令送达多次。靠 + ``UNIQUE(session_id, node_key, command_id)`` 让数据库天然去重：第二次到达时 + 命中已有记录，直接返回 ``result_json``（上次结果），绝不重复推进状态机。 + ``payload_hash`` 防止客户端误复用 command_id 发送不同参数——同 command_id 但 + payload 不同 → 409。session 级命令（activate/archive）node_key 存空串。 + """ + + __tablename__ = "position_roundtable_commands" + __table_args__ = ( + UniqueConstraint( + "session_id", "node_key", "command_id", + name="uq_position_roundtable_commands_session_node_cmd", + ), + ) + + id: Mapped[str] = mapped_column(String(64), primary_key=True) + session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + # 命令作用的节点；session 级命令（activate/archive/delete）存空串 ""。 + node_key: Mapped[str] = mapped_column(String(256), nullable=False, default="") + # 客户端生成的幂等键（一次点击/重试共用）。 + command_id: Mapped[str] = mapped_column(String(64), nullable=False) + # complete / bind / reject / conversation / activate / archive / delete + command_type: Mapped[str] = mapped_column(String(32), nullable=False) + # 规范化 payload 的 sha256；同 command_id 但 hash 不同 → 409（防误复用）。 + payload_hash: Mapped[str] = mapped_column(String(64), nullable=False) + # 上次执行结果的 JSON 序列化，重放时原样返回。 + result_json: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) + created_at: Mapped[datetime] = mapped_column( + BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC) + ) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/sql.py index cb72b107..30891365 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/sql.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/position_roundtable/sql.py @@ -2,6 +2,7 @@ from __future__ import annotations +import hashlib import json from datetime import UTC, datetime from typing import Any @@ -11,7 +12,14 @@ from sqlalchemy import delete, select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from deerflow.persistence.position_roundtable.delivery import ( + is_intent_position_node, + is_markdown_artifact, + markdown_delivery_changed, + session_intent_position_id, +) from deerflow.persistence.position_roundtable.model import ( + PositionRoundtableCommandRow, PositionRoundtableNodeRow, PositionRoundtableSessionRow, ) @@ -42,6 +50,85 @@ def _has_material(row: PositionRoundtableNodeRow) -> bool: return row.revision > 0 or bool(row.latest_answer) or bool(_loads(row.artifact_manifest)) +def _has_confirmed_intent_snapshot(intent: Any) -> bool: + """Return whether a persisted Step-1 snapshot may activate a chain. + + This duplicates only the storage-level shape check used by the router so + activation can re-check the prerequisite *after* locking the session row. + A router-only pre-check leaves a TOCTOU window where another user can + overwrite the intent before the chain topology is frozen. + """ + if not isinstance(intent, dict): + return False + required_sections = ("coreGoals", "riskWarnings", "keyPoints", "strategicSignificance") + if any(key in intent for key in required_sections): + if not all(isinstance(intent.get(key), list) for key in required_sections): + return False + return bool(intent["coreGoals"] or intent["strategicSignificance"]) + objective = intent.get("objective") or intent.get("goal") + return bool(isinstance(objective, str) and objective.strip() and isinstance(intent.get("constraints"), list)) + + +class NodeConcurrentWriteError(Exception): + """节点被并发改写，调用方持有的 version 已过期（乐观锁冲突）。 + + 由 ``update_node(expected_version=...)`` / ``recompute_chain_state`` 在检测到 + version 不匹配时抛出。调用方可捕获后重新读取最新状态重算（幂等重入）。 + """ + + def __init__( + self, + node_key: str, + *, + expected: int | None = None, + actual: int | None = None, + ) -> None: + self.node_key = node_key + self.expected = expected + self.actual = actual + super().__init__( + f"concurrent write detected on position node {node_key}" + + (f" (expected version {expected}, got {actual})" if expected is not None else "") + ) + + +class SessionWriteConflictError(Exception): + """Session-level state changed after a route's optimistic pre-check. + + The public routes translate this into HTTP 409. Keeping the exception at + the persistence boundary prevents a stale intent/task save from reviving + an archived session or changing the frozen input of an active chain. + """ + + def __init__(self, detail: str) -> None: + self.detail = detail + super().__init__(detail) + + +class CommandConflictError(Exception): + """命令在锁内判定为业务冲突（→ HTTP 409），事务已回滚、未改任何状态。 + + 与「幂等重放」严格区分：重放返回上次成功结果；冲突说明状态机拒绝该命令 + （节点未解锁 / 已绑定其它线程 / 同 command_id 携带了不同参数等）。 + ``detail`` 是面向用户的提示，router 直接转成 409。 + """ + + def __init__(self, detail: str) -> None: + self.detail = detail + super().__init__(detail) + + +def _payload_hash(command_type: str, payload: dict[str, Any]) -> str: + """规范化命令参数的 sha256，用于「同 command_id 是否同一参数」判定。""" + normalized = json.dumps( + {"type": command_type, "payload": payload}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + class PositionRoundtableRepository: def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: self._sf = session_factory @@ -70,6 +157,7 @@ class PositionRoundtableRepository: "action_plan_snapshot": _loads(row.action_plan_snapshot), "action_plan_conversation": _loads(row.action_plan_conversation), "status": row.status, + "version": row.version, "created_at": _time(row.created_at), "updated_at": _time(row.updated_at), } @@ -90,6 +178,7 @@ class PositionRoundtableRepository: "artifact_manifest": _loads(row.artifact_manifest) or [], "conversation_snapshot": _loads(row.conversation_snapshot), "revision": row.revision, + "version": row.version, "upstream_revision": _loads(row.upstream_revision), "rejection_count": row.rejection_count or 0, "last_rejection": _loads(row.last_rejection), @@ -294,11 +383,32 @@ class PositionRoundtableRepository: action_plan_snapshot: dict[str, Any] | None | object = _UNSET, action_plan_conversation: dict[str, Any] | None | object = _UNSET, status: str | object = _UNSET, + allowed_statuses: set[str] | None = None, ) -> dict[str, Any] | None: + """Update one session snapshot under the same row lock as commands. + + Snapshot persistence used to do an unlocked ORM ``get`` followed by a + write. That permitted a stale "confirm intent" save to land after a + concurrent activation/archival, or allowed two different snapshot + surfaces to overwrite each other's unseen values. Acquiring the + parent row lock serializes all session writes with node/session + commands. ``allowed_statuses`` additionally freezes task/intent + inputs once the business chain has become active. + """ async with self._sf() as session: - row = await session.get(PositionRoundtableSessionRow, session_id) + row = ( + await session.execute( + select(PositionRoundtableSessionRow) + .where(PositionRoundtableSessionRow.id == session_id) + .with_for_update() + ) + ).scalar_one_or_none() if row is None or not self._can_access(row, user_id): return None + if row.status == "archived": + raise SessionWriteConflictError("该岗位协同会话已归档，仅可查看历史记录") + if allowed_statuses is not None and row.status not in allowed_statuses: + raise SessionWriteConflictError("会话已启动，不能再修改已冻结的任务或意图") if task_snapshot is not _UNSET: row.task_snapshot = _dumps(task_snapshot) if intent_thread_id is not _UNSET: @@ -321,6 +431,7 @@ class PositionRoundtableRepository: row.action_plan_conversation = _dumps(action_plan_conversation) if status is not _UNSET: row.status = str(status) + row.version += 1 row.updated_at = datetime.now(UTC) await session.commit() await session.refresh(row) @@ -425,6 +536,7 @@ class PositionRoundtableRepository: upstream_revision: dict[str, Any] | object = _UNSET, last_rejection: dict[str, Any] | None | object = _UNSET, invalidated_by: str | None | object = _UNSET, + expected_version: int | None = None, increment_revision: bool = False, increment_rejection: bool = False, ) -> dict[str, Any] | None: @@ -436,9 +548,17 @@ class PositionRoundtableRepository: PositionRoundtableNodeRow.session_id == session_id, PositionRoundtableNodeRow.node_key == node_key, ).limit(1) + # 乐观锁：传入 expected_version 时加行锁并校验，杜绝「检查后到提交之间 + # 被并发改写」的 TOCTOU。recompute_chain_state 不走这里（它自管事务）。 + if expected_version is not None: + stmt = stmt.with_for_update() row = (await session.execute(stmt)).scalars().first() if row is None: return None + if expected_version is not None and row.version != expected_version: + raise NodeConcurrentWriteError( + node_key, expected=expected_version, actual=row.version + ) if thread_id is not _UNSET: row.thread_id = thread_id or None if status is not _UNSET: @@ -455,6 +575,8 @@ class PositionRoundtableRepository: row.last_rejection = _dumps(last_rejection) if invalidated_by is not _UNSET: row.invalidated_by = invalidated_by or None + # 任意字段变更都 bump version（乐观锁单调递增），与业务交付 revision 解耦。 + row.version += 1 if increment_revision: row.revision += 1 if increment_rejection: @@ -464,6 +586,734 @@ class PositionRoundtableRepository: await session.refresh(row) return self._node_dict(row) + async def recompute_chain_state( + self, + session_id: str, + node_key: str, + user_id: str, + *, + next_status: str, + latest_answer: str | None, + artifact_manifest: list[dict[str, Any]] | None, + upstream_revision: dict[str, Any] | None, + new_effective_delivery: bool, + ) -> dict[str, Any] | None: + """在单一 ``FOR UPDATE`` 事务内原子完成「本节点交付 → 下游失效 → 解锁下一 stage」。 + + 把原来 router 里由若干个独立提交的 ``update_node`` 拼成的状态转移收敛成一个 + 事务：先锁住本 session 全部节点行（MySQL ``SELECT ... FOR UPDATE``），再在锁 + 内重算所有节点状态，一次提交。同一 session 的并发 ``complete_node_turn`` 因此被 + 数据库行锁串行化，彻底消除 TOCTOU / 丢失更新导致的「下一 stage 卡 locked / + invalidated_by 残留 → 下一阶段拿不到上游产物」断链。 + + 幂等可重入：解锁不假设「最后一个 seat 负责」，而是遍历每个「全 done」stage 补 + 解锁其紧邻下一 stage 的 locked 节点；多次调用结果一致。``next_status`` / + ``new_effective_delivery`` 等业务判定由 router 基于调用前快照算好传入，本方法 + 只负责把它们原子地落到节点状态机上。 + """ + async with self._sf() as session: + parent = await session.get(PositionRoundtableSessionRow, session_id) + if parent is None or not self._can_access(parent, user_id): + return None + # 锁住本 session 全部节点行，串行化并发状态转移（行锁持有至 commit）。 + rows = ( + await session.execute( + select(PositionRoundtableNodeRow) + .where(PositionRoundtableNodeRow.session_id == session_id) + .order_by( + PositionRoundtableNodeRow.stage_index, + PositionRoundtableNodeRow.seat_index, + ) + .with_for_update() + ) + ).scalars().all() + if not rows: + return None + by_key = {r.node_key: r for r in rows} + current = by_key.get(node_key) + if current is None: + return None + + now = datetime.now(UTC) + # 本次交付前的 revision（用于判断「此前已交付过 → 新交付需失效下游」）。 + old_revision = current.revision + + # ① 落本节点交付状态。 + current.status = next_status + current.latest_answer = latest_answer + if artifact_manifest is not None: + current.artifact_manifest = _dumps(artifact_manifest) + if upstream_revision is not None: + current.upstream_revision = _dumps(upstream_revision) + current.invalidated_by = None + if new_effective_delivery: + current.revision += 1 + current.version += 1 + current.updated_at = now + + # ② + ③ 与命令式状态机共用同一套下游失效 / 幂等解锁实现，避免逻辑漂移。 + self._invalidate_downstream(rows, current, new_effective_delivery, old_revision, now) + self._recompute_unlocks(rows, now) + + await session.commit() + await session.refresh(current) + return self._node_dict(current) + + # ── command-style state machine ────────────────────────────────────────── + + async def apply_node_command( + self, + session_id: str, + node_key: str, + user_id: str, + *, + command_id: str, + command_type: str, + payload: dict[str, Any] | None = None, + expected_session_version: int | None = None, + ) -> dict[str, Any]: + """命令式状态机：所有节点级写命令（complete/bind/conversation/reject）的唯一入口。 + + 单事务内依次完成： + + ① ``SELECT ... FOR UPDATE`` 锁住 session 行（可选 ``expected_session_version`` + CAS，命中失败即 409）； + ② 命令幂等判定——``UNIQUE(session_id, node_key, command_id)`` 命中已有记录： + 参数一致（payload_hash 相同）→ 原样重放上次结果；参数不同 → 409； + ③ 锁住本 session 全部节点行，**基于锁内状态**做所有业务判定 + （能否完成 / 是否新交付 / 能否驳回），绝不信任调用方传入的过期快照； + ④ 应用状态转移（本节点更新 → 下游失效 → 幂等重算解锁）； + ⑤ 写入命令幂等记录（含结果 JSON），一次提交。 + + 返回 dict： + + - ``ok=True`` — ``replayed`` 标记是否幂等重放；``node`` 为目标节点新状态， + ``reject`` 命令额外返回 ``nodes``（全部节点）与 ``session``； + - ``ok=False`` — ``http_status``（404/409）+ ``detail``；事务已回滚，状态未变。 + + 一致性只来自数据库：并发命令被 session/节点行锁串行化，重试/双击/beacon 由 + 命令唯一键去重。进程内任何缓存都不参与正确性。 + """ + payload = payload or {} + payload_hash = _payload_hash(command_type, payload) + now = datetime.now(UTC) + + async def _run(session: AsyncSession) -> dict[str, Any]: + parent = ( + await session.execute( + select(PositionRoundtableSessionRow) + .where(PositionRoundtableSessionRow.id == session_id) + .with_for_update() + ) + ).scalar_one_or_none() + if parent is None or not self._can_access(parent, user_id): + return {"ok": False, "http_status": 404, "detail": "Position roundtable session not found"} + if expected_session_version is not None and parent.version != expected_session_version: + return { + "ok": False, + "http_status": 409, + "detail": "会话状态已更新，请刷新后重试", + "session": self._session_dict(parent), + } + # 归档后的会话拒绝一切写命令（锁内复检，杜绝路由预检与提交之间的窗口）。 + if parent.status == "archived": + return { + "ok": False, + "http_status": 409, + "detail": "该岗位协同会话已归档，仅可查看历史记录", + } + + # ② 命令幂等：同 command_id 已执行过 → 重放上次结果（或 409）。 + existing = ( + await session.execute( + select(PositionRoundtableCommandRow).where( + PositionRoundtableCommandRow.session_id == session_id, + PositionRoundtableCommandRow.node_key == (node_key or ""), + PositionRoundtableCommandRow.command_id == command_id, + ) + ) + ).scalars().first() + if existing is not None: + if existing.payload_hash != payload_hash: + return { + "ok": False, + "http_status": 409, + "detail": "相同操作标识携带了不同参数，已拒绝执行", + } + return { + "ok": True, + "replayed": True, + "command_type": command_type, + **(_loads(existing.result_json) or {}), + } + + # ③ 锁住全部节点行（行锁持有至 commit），串行化同 session 并发命令。 + rows = ( + await session.execute( + select(PositionRoundtableNodeRow) + .where(PositionRoundtableNodeRow.session_id == session_id) + .order_by( + PositionRoundtableNodeRow.stage_index, + PositionRoundtableNodeRow.seat_index, + ) + .with_for_update() + ) + ).scalars().all() + by_key = {r.node_key: r for r in rows} + + if command_type == "complete": + result = self._apply_complete(by_key, node_key, parent, payload, now) + elif command_type == "bind": + result = self._apply_bind(by_key, node_key, payload, now) + elif command_type == "conversation": + result = self._apply_conversation(by_key, node_key, payload, now) + elif command_type == "reject": + result = self._apply_reject(by_key, node_key, parent, payload, user_id, now) + else: + return {"ok": False, "http_status": 400, "detail": f"unknown command type: {command_type}"} + + if not result.get("ok", True): + return result + + # ⑤ 写命令幂等记录（与状态转移同事务提交）。 + stored = {k: v for k, v in result.items() if k != "ok"} + session.add( + PositionRoundtableCommandRow( + id=uuid4().hex, + session_id=session_id, + node_key=node_key or "", + command_id=command_id, + command_type=command_type, + payload_hash=payload_hash, + result_json=_dumps(stored), + created_at=now, + ) + ) + parent.version += 1 + parent.updated_at = now + result["replayed"] = False + result["command_type"] = command_type + return result + + try: + async with self._sf() as session: + outcome = await _run(session) + if outcome.get("ok", False): + await session.commit() + else: + await session.rollback() + return outcome + except IntegrityError: + # 命令唯一键并发竞态：另一连接已用同一 command_id 提交。读回其结果重放。 + async with self._sf() as session: + existing = ( + await session.execute( + select(PositionRoundtableCommandRow).where( + PositionRoundtableCommandRow.session_id == session_id, + PositionRoundtableCommandRow.node_key == (node_key or ""), + PositionRoundtableCommandRow.command_id == command_id, + ) + ) + ).scalars().first() + if existing is not None: + if existing.payload_hash != payload_hash: + return { + "ok": False, + "http_status": 409, + "detail": "相同操作标识携带了不同参数，已拒绝执行", + } + return { + "ok": True, + "replayed": True, + "command_type": command_type, + **(_loads(existing.result_json) or {}), + } + raise + + # ── command handlers（全部在 FOR UPDATE 锁内调用，入参为锁定的行） ───────── + + def _apply_complete( + self, + by_key: dict[str, PositionRoundtableNodeRow], + node_key: str, + parent: PositionRoundtableSessionRow, + payload: dict[str, Any], + now: datetime, + ) -> dict[str, Any]: + """complete-turn：锁内判定能否完成 / 是否新交付，原子落状态机。""" + current = by_key.get(node_key) + if current is None: + return {"ok": False, "http_status": 404, "detail": "Position roundtable node not found"} + if current.status == "locked": + return {"ok": False, "http_status": 409, "detail": "上游节点尚未完成，当前节点未解锁"} + if current.invalidated_by: + return { + "ok": False, + "http_status": 409, + "detail": "上游产物已被驳回或重做，当前节点需等待前序节点重新完成", + } + + # 请求未带产物清单 → 沿用锁内的当前清单（绝不用调用方的过期快照兜底）。 + incoming = payload.get("artifact_manifest") + current_manifest = _loads(current.artifact_manifest) or [] + effective_artifacts = incoming if isinstance(incoming, list) and incoming else current_manifest + latest_answer = payload.get("latest_answer") + upstream_revision = payload.get("upstream_revision") + + has_markdown = any(is_markdown_artifact(item) for item in effective_artifacts) + markdown_changed = markdown_delivery_changed(current_manifest, effective_artifacts) + requires_fresh_delivery = current.status in {"rejected", "stale"} + intent_position_id = session_intent_position_id(_loads(parent.chain_snapshot)) + is_intent_node = is_intent_position_node(current.position_id, intent_position_id) + next_status = ( + "done" + if is_intent_node + else "done" + if has_markdown and (not requires_fresh_delivery or markdown_changed) + else "running" + ) + # 有效 revision 是「交付 revision」，不是每一次对话应答。已完成节点上的普通 + # 追问，不能仅因其旧 Markdown 仍出现在清单里就失效所有下游 stage。 + new_effective_delivery = ( + next_status == "done" + and (is_intent_node or current.status != "done" or markdown_changed) + ) + old_revision = current.revision + + # ④ 落本节点交付状态。 + current.status = next_status + current.latest_answer = latest_answer + current.artifact_manifest = _dumps(effective_artifacts) + if upstream_revision is not None: + current.upstream_revision = _dumps(upstream_revision) + current.invalidated_by = None + if new_effective_delivery: + current.revision += 1 + current.version += 1 + current.updated_at = now + + self._invalidate_downstream(by_key.values(), current, new_effective_delivery, old_revision, now) + self._recompute_unlocks(by_key.values(), now) + + return {"ok": True, "node": self._node_dict(current)} + + def _apply_bind( + self, + by_key: dict[str, PositionRoundtableNodeRow], + node_key: str, + payload: dict[str, Any], + now: datetime, + ) -> dict[str, Any]: + """bind-thread：锁内校验解锁/失效/线程占用后绑定。""" + current = by_key.get(node_key) + if current is None: + return {"ok": False, "http_status": 404, "detail": "Position roundtable node not found"} + thread_id = str(payload.get("thread_id") or "").strip() + if not thread_id: + return {"ok": False, "http_status": 400, "detail": "thread_id is required"} + if current.status == "locked": + return {"ok": False, "http_status": 409, "detail": "上游节点尚未完成，当前节点未解锁"} + if current.invalidated_by: + return { + "ok": False, + "http_status": 409, + "detail": "上游产物已被驳回或重做，当前节点需等待前序节点重新完成", + } + if current.thread_id and current.thread_id != thread_id: + return {"ok": False, "http_status": 409, "detail": "该节点已绑定另一条对话线程"} + current.thread_id = thread_id + if current.status == "ready": + current.status = "running" + current.version += 1 + current.updated_at = now + return {"ok": True, "node": self._node_dict(current)} + + def _apply_conversation( + self, + by_key: dict[str, PositionRoundtableNodeRow], + node_key: str, + payload: dict[str, Any], + now: datetime, + ) -> dict[str, Any]: + """conversation：对话快照持久化（UI 恢复用），锁内校验线程归属。""" + current = by_key.get(node_key) + if current is None: + return {"ok": False, "http_status": 404, "detail": "Position roundtable node not found"} + thread_id = payload.get("thread_id") + if thread_id and current.thread_id and current.thread_id != thread_id: + return {"ok": False, "http_status": 409, "detail": "Position node is bound to another thread"} + if thread_id and not current.thread_id: + current.thread_id = thread_id + current.conversation_snapshot = _dumps(payload.get("conversation_snapshot")) + current.version += 1 + current.updated_at = now + return {"ok": True, "node": self._node_dict(current)} + + def _apply_reject( + self, + by_key: dict[str, PositionRoundtableNodeRow], + node_key: str, + parent: PositionRoundtableSessionRow, + payload: dict[str, Any], + user_id: str, + now: datetime, + ) -> dict[str, Any]: + """reject：驳回一个已交付节点并失效全部下游 stage（锁内判定可驳回性）。""" + rows = sorted( + by_key.values(), key=lambda r: (r.stage_index, r.seat_index) + ) + target = by_key.get(node_key) + if target is None: + return {"ok": False, "http_status": 404, "detail": "Position roundtable node not found"} + if target.status == "locked": + return {"ok": False, "http_status": 409, "detail": "当前节点尚未解锁，暂无可驳回的交付产物"} + if ( + target.revision <= 0 + and not target.latest_answer + and not _loads(target.artifact_manifest) + ): + return {"ok": False, "http_status": 409, "detail": "当前节点暂无可驳回的交付产物"} + + rejection = { + "title": str(payload.get("title") or "").strip() or "产物驳回", + "reason": str(payload.get("reason") or "").strip(), + "rejected_by": user_id, + "rejected_at": now.isoformat(), + } + summary_snapshot = _loads(parent.summary_snapshot) + if isinstance(summary_snapshot, dict) and summary_snapshot.get("status") == "done": + parent.summary_snapshot = _dumps( + { + **summary_snapshot, + "status": "stale", + "invalidated_by": target.node_key, + "invalidated_at": now.isoformat(), + } + ) + action_plan_snapshot = _loads(parent.action_plan_snapshot) + if isinstance(action_plan_snapshot, dict) and action_plan_snapshot.get("status") == "done": + parent.action_plan_snapshot = _dumps( + { + **action_plan_snapshot, + "status": "stale", + "invalidated_by": target.node_key, + "invalidated_at": now.isoformat(), + } + ) + if parent.status == "completed": + parent.status = "active" + + target.status = "rejected" + target.last_rejection = _dumps(rejection) + target.rejection_count += 1 + target.invalidated_by = None + target.version += 1 + target.updated_at = now + + for row in rows: + if row.stage_index <= target.stage_index: + continue + if _has_material(row) or row.status in {"running", "done", "stale", "rejected", "error"}: + row.status = "stale" + else: + row.status = "locked" + row.invalidated_by = target.node_key + row.version += 1 + row.updated_at = now + + return { + "ok": True, + "session": self._session_dict(parent), + "nodes": [self._node_dict(row) for row in rows], + } + + @staticmethod + def _invalidate_downstream( + rows: Any, + current: PositionRoundtableNodeRow, + new_effective_delivery: bool, + old_revision: int, + now: datetime, + ) -> None: + """新交付且本节点此前已交付过 → 后续 stage 全部失效（需重做）。""" + if not (new_effective_delivery and old_revision > 0): + return + for downstream in rows: + if ( + downstream.stage_index > current.stage_index + and downstream.status in {"ready", "running", "done", "stale", "rejected", "error"} + ): + downstream.status = "stale" if _has_material(downstream) else "locked" + downstream.invalidated_by = current.node_key + downstream.version += 1 + downstream.updated_at = now + + @staticmethod + def _recompute_unlocks(rows: Any, now: datetime) -> None: + """幂等重算 stage 解锁：每个「全部 done」stage 解锁其紧邻下一 stage。 + + locked → ready；stale 保持 stale（产物过期需重做）但清掉 invalidated_by + （上游已就绪）。多次调用结果一致，不依赖调用时序。 + """ + row_list = list(rows) + stage_indices = sorted({r.stage_index for r in row_list}) + for stage in stage_indices: + stage_rows = [r for r in row_list if r.stage_index == stage] + if not stage_rows or not all(r.status == "done" for r in stage_rows): + continue + later = [t for t in stage_indices if t > stage] + if not later: + continue + next_stage = min(later) + for node in row_list: + if node.stage_index != next_stage: + continue + if node.status == "locked": + node.status = "ready" + node.invalidated_by = None + node.version += 1 + node.updated_at = now + elif node.status == "stale": + node.invalidated_by = None + node.version += 1 + node.updated_at = now + + async def apply_session_command( + self, + session_id: str, + user_id: str, + *, + command_id: str, + command_type: str, + payload: dict[str, Any] | None = None, + expected_session_version: int | None = None, + ) -> dict[str, Any]: + """session 级命令（activate / archive / delete）的命令式入口。 + + 与 ``apply_node_command`` 同一套原则：单事务锁 session 行（``FOR UPDATE``） + + 命令唯一键幂等 + 锁内判定。返回结构同 ``apply_node_command`` + （``ok`` / ``replayed`` / ``http_status`` / ``detail`` / ``session`` / ``nodes``）。 + + - ``activate`` — payload 需带 ``chain_id`` / ``chain_snapshot`` / ``nodes``。 + 锁内判定：已激活且链相同 → 幂等返回现有状态；链不同 → 409。 + - ``archive`` — 已归档 → 幂等返回；否则置 archived。 + - ``delete`` — 删除节点与会话（命令记录无外键，保留供重放）。 + """ + payload = payload or {} + payload_hash = _payload_hash(command_type, payload) + now = datetime.now(UTC) + + async def _replay_check(session: AsyncSession) -> dict[str, Any] | None: + existing = ( + await session.execute( + select(PositionRoundtableCommandRow).where( + PositionRoundtableCommandRow.session_id == session_id, + PositionRoundtableCommandRow.node_key == "", + PositionRoundtableCommandRow.command_id == command_id, + ) + ) + ).scalars().first() + if existing is None: + return None + if existing.payload_hash != payload_hash: + return { + "ok": False, + "http_status": 409, + "detail": "相同操作标识携带了不同参数，已拒绝执行", + } + return { + "ok": True, + "replayed": True, + "command_type": command_type, + **(_loads(existing.result_json) or {}), + } + + async def _run(session: AsyncSession) -> dict[str, Any]: + # delete 之后 session 行已不存在，重试必须在锁 session 之前查命令记录。 + replayed = await _replay_check(session) + if replayed is not None: + return replayed + + parent = ( + await session.execute( + select(PositionRoundtableSessionRow) + .where(PositionRoundtableSessionRow.id == session_id) + .with_for_update() + ) + ).scalar_one_or_none() + if parent is None or not self._can_access(parent, user_id): + return {"ok": False, "http_status": 404, "detail": "Position roundtable session not found"} + if expected_session_version is not None and parent.version != expected_session_version: + return { + "ok": False, + "http_status": 409, + "detail": "会话状态已更新，请刷新后重试", + "session": self._session_dict(parent), + } + + if command_type == "activate": + result = await self._apply_activate(session, parent, payload, now) + elif command_type == "archive": + if parent.status == "archived": + result = { + "ok": True, + "session": self._session_dict(parent), + "nodes": [], + "already": True, + } + else: + parent.status = "archived" + parent.version += 1 + parent.updated_at = now + result = { + "ok": True, + "session": self._session_dict(parent), + "nodes": [], + } + elif command_type == "delete": + await session.execute( + delete(PositionRoundtableNodeRow).where( + PositionRoundtableNodeRow.session_id == session_id + ) + ) + await session.execute( + delete(PositionRoundtableSessionRow).where( + PositionRoundtableSessionRow.id == session_id + ) + ) + result = {"ok": True, "deleted": True} + else: + return {"ok": False, "http_status": 400, "detail": f"unknown command type: {command_type}"} + + if not result.get("ok", True): + return result + + stored = {k: v for k, v in result.items() if k != "ok"} + session.add( + PositionRoundtableCommandRow( + id=uuid4().hex, + session_id=session_id, + node_key="", + command_id=command_id, + command_type=command_type, + payload_hash=payload_hash, + result_json=_dumps(stored), + created_at=now, + ) + ) + result["replayed"] = False + result["command_type"] = command_type + return result + + try: + async with self._sf() as session: + outcome = await _run(session) + if outcome.get("ok", False): + await session.commit() + else: + await session.rollback() + return outcome + except IntegrityError: + # 命令唯一键并发竞态：读回已提交的那条命令结果重放。 + async with self._sf() as session: + replayed = await _replay_check(session) + if replayed is not None: + return replayed + raise + + async def _apply_activate( + self, + session: AsyncSession, + parent: PositionRoundtableSessionRow, + payload: dict[str, Any], + now: datetime, + ) -> dict[str, Any]: + """激活业务链：锁内判定「已激活 → 链相同幂等 / 链不同 409」，否则建节点。""" + existing_rows = ( + await session.execute( + select(PositionRoundtableNodeRow) + .where(PositionRoundtableNodeRow.session_id == parent.id) + .order_by( + PositionRoundtableNodeRow.stage_index, + PositionRoundtableNodeRow.seat_index, + ) + .with_for_update() + ) + ).scalars().all() + + if parent.status == "archived": + return { + "ok": False, + "http_status": 409, + "detail": "该岗位协同会话已归档，仅可查看历史记录", + } + + chain_id = str(payload.get("chain_id") or "") + if existing_rows: + # 已激活：同一业务链 → 幂等返回现有状态；不同链 → 冲突。 + if parent.chain_id == chain_id: + return { + "ok": True, + "session": self._session_dict(parent), + "nodes": [self._node_dict(row) for row in existing_rows], + "already": True, + } + return { + "ok": False, + "http_status": 409, + "detail": "该会话已激活其他业务链条，不能重复激活", + } + + if not _has_confirmed_intent_snapshot(_loads(parent.intent_snapshot)): + return { + "ok": False, + "http_status": 400, + "detail": "请先完成并保存任务意图", + } + + chain_snapshot = payload.get("chain_snapshot") or {} + nodes = payload.get("nodes") or [] + if not nodes: + return {"ok": False, "http_status": 400, "detail": "activate requires nodes"} + + parent.chain_id = chain_id + parent.chain_snapshot = _dumps(chain_snapshot) + parent.status = "active" + parent.version += 1 + parent.updated_at = now + + node_rows = [ + PositionRoundtableNodeRow( + id=uuid4().hex, + session_id=parent.id, + node_key=str(node["node_key"]), + stage_index=int(node["stage_index"]), + seat_index=int(node["seat_index"]), + agent_id=str(node["agent_id"]), + position_id=node.get("position_id") or None, + status=str(node.get("status") or "locked"), + latest_answer=node.get("latest_answer") or None, + artifact_manifest=_dumps(node.get("artifact_manifest") or []), + revision=int(node.get("revision") or 0), + upstream_revision=( + _dumps(node.get("upstream_revision")) + if node.get("upstream_revision") is not None + else None + ), + created_at=now, + updated_at=now, + ) + for node in nodes + ] + session.add_all(node_rows) + # 同事务内 flush 拿到完整行状态，供命令记录与返回值使用。 + await session.flush() + return { + "ok": True, + "session": self._session_dict(parent), + "nodes": [self._node_dict(row) for row in node_rows], + } + async def reject_node( self, session_id: str, diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/base.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/base.py index e45743a0..b3d01ff7 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/base.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/base.py @@ -27,8 +27,14 @@ class RoundtableArtifactSubmissionStore(ABC): *, task_delivery_id: str | None = None, actor_id: str | None = None, + expected_version: int | None = None, ) -> dict[str, Any]: - """Create or update the idempotent submission record for an artifact.""" + """Create or update the idempotent submission record for an artifact. + + When *expected_version* is given it must match the current row version; + a mismatch indicates another writer committed first and the caller + should re-read and retry (HTTP 409). + """ raise NotImplementedError @abstractmethod @@ -39,6 +45,7 @@ class RoundtableArtifactSubmissionStore(ABC): *, submission_id: str | None = None, actor_id: str | None = None, + expected_version: int | None = None, ) -> dict[str, Any] | None: """Mark a submission as cancelled without deleting its audit record.""" raise NotImplementedError diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/memory.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/memory.py index a1cf4986..ea7bda10 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/memory.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/memory.py @@ -54,7 +54,10 @@ class MemoryRoundtableArtifactSubmissionStore(RoundtableArtifactSubmissionStore) *, task_delivery_id: str | None = None, actor_id: str | None = None, + expected_version: int | None = None, ) -> dict[str, Any]: + from deerflow.persistence.roundtable_artifact_submissions.sql import SubmissionVersionConflict + key = submission_key(task_id, artifact) row = self._rows.get((task_id, key)) values = _artifact_values(artifact) @@ -67,6 +70,7 @@ class MemoryRoundtableArtifactSubmissionStore(RoundtableArtifactSubmissionStore) **values, "task_delivery_id": task_delivery_id, "submitted": True, + "version": 0, "submitted_at": now, "cancelled_at": None, "created_by": actor_id, @@ -79,6 +83,9 @@ class MemoryRoundtableArtifactSubmissionStore(RoundtableArtifactSubmissionStore) self._rows[(task_id, key)] = row return deepcopy(row) + if expected_version is not None and row.get("version", 0) != expected_version: + raise SubmissionVersionConflict(deepcopy(row)) + changed = ( not row["submitted"] or row.get("task_delivery_id") != task_delivery_id @@ -91,6 +98,7 @@ class MemoryRoundtableArtifactSubmissionStore(RoundtableArtifactSubmissionStore) { "task_delivery_id": task_delivery_id, "submitted": True, + "version": row.get("version", 0) + 1, "submitted_at": now, "cancelled_at": None, "updated_by": actor_id, @@ -108,16 +116,24 @@ class MemoryRoundtableArtifactSubmissionStore(RoundtableArtifactSubmissionStore) *, submission_id: str | None = None, actor_id: str | None = None, + expected_version: int | None = None, ) -> dict[str, Any] | None: + from deerflow.persistence.roundtable_artifact_submissions.sql import SubmissionVersionConflict + key = submission_key(task_id, artifact) row = self._rows.get((task_id, key)) if row is None or (submission_id is not None and row["id"] != submission_id): return None + + if expected_version is not None and row.get("version", 0) != expected_version: + raise SubmissionVersionConflict(deepcopy(row)) + if row["submitted"]: now = _now() row.update( { "submitted": False, + "version": row.get("version", 0) + 1, "cancelled_at": now, "updated_by": actor_id, "cancelled_by": actor_id, diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/model.py index f973b4bc..96519b4b 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/model.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/model.py @@ -4,6 +4,7 @@ from __future__ import annotations from datetime import UTC, datetime +import sqlalchemy as sa from sqlalchemy import Boolean, Index, Integer, String, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column @@ -41,6 +42,9 @@ class RoundtableArtifactSubmissionRow(Base): task_delivery_id: Mapped[str | None] = mapped_column(String(255), nullable=True) submitted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + # Phase 5 CAS: 每次状态变更 +1。submit/cancel 可带 expected_version 做乐观锁, + # FOR UPDATE + version 检查确保「旧 cancel 不得撤销新提交」。 + version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0")) submitted_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True) cancelled_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True) created_by: Mapped[str | None] = mapped_column(String(64), nullable=True) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/sql.py index 6f397a81..b4e33e44 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/sql.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_artifact_submissions/sql.py @@ -15,6 +15,21 @@ from deerflow.persistence.roundtable_artifact_submissions.memory import submissi from deerflow.persistence.roundtable_artifact_submissions.model import RoundtableArtifactSubmissionRow +class SubmissionVersionConflict(Exception): + """Stale ``expected_version`` — another writer committed first. + + Carries the freshest row dict so the router can return it in the 409 body, + letting the client merge and retry. + """ + + def __init__(self, row: dict[str, Any]) -> None: + self.row = row + super().__init__( + f"submission version conflict: expected_version does not match " + f"current version={row.get('version')}" + ) + + def _timestamp(value: datetime | None) -> str | None: return value.isoformat() if isinstance(value, datetime) else value @@ -34,6 +49,7 @@ def _to_dict(row: RoundtableArtifactSubmissionRow) -> dict[str, Any]: "revision": row.revision, "task_delivery_id": row.task_delivery_id, "submitted": row.submitted, + "version": row.version, "submitted_at": _timestamp(row.submitted_at), "cancelled_at": _timestamp(row.cancelled_at), "created_by": row.created_by, @@ -92,34 +108,63 @@ class RoundtableArtifactSubmissionRepository(RoundtableArtifactSubmissionStore): *, task_delivery_id: str | None = None, actor_id: str | None = None, + expected_version: int | None = None, ) -> dict[str, Any]: key = submission_key(task_id, artifact) - stmt = select(RoundtableArtifactSubmissionRow).where( - RoundtableArtifactSubmissionRow.task_id == task_id, - RoundtableArtifactSubmissionRow.idempotency_key == key, - ) - async with self._sf() as session: - result = await session.execute(stmt) - row = result.scalars().first() - now = datetime.now(UTC) - created = row is None - if row is None: - row = RoundtableArtifactSubmissionRow( - id=uuid4().hex, - task_id=task_id, - idempotency_key=key, - task_delivery_id=task_delivery_id, - submitted=True, - submitted_at=now, - created_by=actor_id, - updated_by=actor_id, - submitted_by=actor_id, - created_at=now, - updated_at=now, + # 至多重试一次:第一轮 INSERT 遇唯一键冲突 → 第二轮 FOR UPDATE 读回做 CAS 更新。 + for _attempt in range(2): + async with self._sf() as session: + result = await session.execute( + select(RoundtableArtifactSubmissionRow) + .where( + RoundtableArtifactSubmissionRow.task_id == task_id, + RoundtableArtifactSubmissionRow.idempotency_key == key, + ) + .with_for_update() ) - _apply_artifact(row, artifact) - session.add(row) - elif _requires_submission_update(row, artifact, task_delivery_id): + row = result.scalars().first() + now = datetime.now(UTC) + + if row is None: + # 新提交:INSERT,version=0。 + row = RoundtableArtifactSubmissionRow( + id=uuid4().hex, + task_id=task_id, + idempotency_key=key, + task_delivery_id=task_delivery_id, + submitted=True, + submitted_at=now, + created_by=actor_id, + updated_by=actor_id, + submitted_by=actor_id, + created_at=now, + updated_at=now, + version=0, + ) + _apply_artifact(row, artifact) + session.add(row) + try: + await session.commit() + except IntegrityError: + # 并发 INSERT 赢家已写入,rollback 后重试 FOR UPDATE 读回。 + await session.rollback() + continue + await session.refresh(row) + return _to_dict(row) + + # 行已存在 —— CAS 检查:expected_version 不匹配说明在此期间另一个 + # 写入已提交(如新 submit 把 version 从 0 bump 到 1),旧 cancel 的 + # expected_version=0 必须失败,不得撤销新提交。 + # 注意:不调 session.rollback()——rollback 会 expire 所有 ORM 对象, + # 导致 _to_dict 访问属性时触发 lazy-load → MissingGreenlet。 + # async with 退出时会自动回滚未提交事务并释放 FOR UPDATE 锁。 + if expected_version is not None and row.version != expected_version: + raise SubmissionVersionConflict(_to_dict(row)) + + if not _requires_submission_update(row, artifact, task_delivery_id): + # 幂等:内容无变化,不 bump version。 + return _to_dict(row) + _apply_artifact(row, artifact) row.task_delivery_id = task_delivery_id row.submitted = True @@ -129,34 +174,12 @@ class RoundtableArtifactSubmissionRepository(RoundtableArtifactSubmissionStore): row.submitted_by = actor_id row.cancelled_by = None row.updated_at = now - try: + row.version = row.version + 1 await session.commit() - except IntegrityError: - # Two callers can both observe a missing row. The unique key - # turns that race into a normal idempotent retry instead of a - # 500. If their non-key metadata differs, apply this request - # after loading the winner so PUT still has upsert semantics. - await session.rollback() - if not created: - raise - result = await session.execute(stmt) - row = result.scalars().first() - if row is None: - raise - if _requires_submission_update(row, artifact, task_delivery_id): - now = datetime.now(UTC) - _apply_artifact(row, artifact) - row.task_delivery_id = task_delivery_id - row.submitted = True - row.submitted_at = now - row.cancelled_at = None - row.updated_by = actor_id - row.submitted_by = actor_id - row.cancelled_by = None - row.updated_at = now - await session.commit() - await session.refresh(row) - return _to_dict(row) + await session.refresh(row) + return _to_dict(row) + + raise RuntimeError("submit: retry loop exhausted (unexpected)") async def cancel( self, @@ -165,24 +188,36 @@ class RoundtableArtifactSubmissionRepository(RoundtableArtifactSubmissionStore): *, submission_id: str | None = None, actor_id: str | None = None, + expected_version: int | None = None, ) -> dict[str, Any] | None: key = submission_key(task_id, artifact) - stmt = select(RoundtableArtifactSubmissionRow).where( - RoundtableArtifactSubmissionRow.task_id == task_id, - RoundtableArtifactSubmissionRow.idempotency_key == key, - ) async with self._sf() as session: - result = await session.execute(stmt) + result = await session.execute( + select(RoundtableArtifactSubmissionRow) + .where( + RoundtableArtifactSubmissionRow.task_id == task_id, + RoundtableArtifactSubmissionRow.idempotency_key == key, + ) + .with_for_update() + ) row = result.scalars().first() if row is None or (submission_id is not None and row.id != submission_id): return None - if row.submitted: - now = datetime.now(UTC) - row.submitted = False - row.cancelled_at = now - row.updated_by = actor_id - row.cancelled_by = actor_id - row.updated_at = now - await session.commit() - await session.refresh(row) + + if expected_version is not None and row.version != expected_version: + raise SubmissionVersionConflict(_to_dict(row)) + + if not row.submitted: + # 幂等:已取消,不 bump version。 + return _to_dict(row) + + now = datetime.now(UTC) + row.submitted = False + row.cancelled_at = now + row.updated_by = actor_id + row.cancelled_by = actor_id + row.updated_at = now + row.version = row.version + 1 + await session.commit() + await session.refresh(row) return _to_dict(row) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_drafts/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_drafts/model.py index f7af5964..c1cf6c6a 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_drafts/model.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_drafts/model.py @@ -47,6 +47,9 @@ class RoundtableDraftRow(Base): default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), ) + # 乐观并发控制版本号：每次 update_draft 自增；传入 expected_version 时校验，命中 + # 0 行即并发冲突（防 step2.runs 读-改-写丢失更新——后台作业与前台 PUT 并发）。 + version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") class RoundtableRecommendHistoryRow(Base): diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_drafts/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_drafts/sql.py index 2ca3ddcb..c60f6558 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_drafts/sql.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_drafts/sql.py @@ -15,7 +15,7 @@ import re from datetime import UTC, datetime from typing import Any -from sqlalchemy import select +from sqlalchemy import case, select, update from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from deerflow.persistence.roundtable_drafts.model import ( @@ -66,6 +66,23 @@ def _dumps(value: Any) -> str | None: return json.dumps(value, ensure_ascii=False) +class DraftConcurrentWriteError(Exception): + """乐观锁冲突：``update_draft(expected_version=...)`` 时版本号不匹配。 + + 由后台作业 ``_write_run_to_draft`` 的读-改-写重试循环捕获：冲突时重新读取最新草稿、 + 重新合并 ``step2.runs[]``、再试。防多用户/多 worker 并发写同一草稿时后写覆盖先写。 + """ + + def __init__(self, draft_id: str, *, expected: int | None = None, actual: int | None = None) -> None: + self.draft_id = draft_id + self.expected = expected + self.actual = actual + super().__init__( + f"concurrent write detected on roundtable draft {draft_id} " + f"(expected version {expected}, got {actual})" + ) + + class RoundtableDraftRepository: def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: self._sf = session_factory @@ -80,6 +97,8 @@ class RoundtableDraftRepository: "task_id": row.task_id, "title": row.title, "furthest_step": row.furthest_step, + # 乐观锁版本号：列表也携带，供前端本地副本对账（冲突时重拉合并）。 + "version": row.version, # 所用业务链名（历史卡片展示用）；无链条 → None → 前端显示「自由模式」。 "chain_title": _extract_chain_title(row.step2), "created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at, @@ -145,26 +164,60 @@ class RoundtableDraftRepository: step2=_UNSET, step3=_UNSET, task_id=_UNSET, + expected_version: int | None = None, ) -> dict[str, Any] | None: - async with self._sf() as session: - row = await session.get(RoundtableDraftRow, draft_id) + """原子条件 UPDATE：版本判定与字段写入发生在**同一条 SQL** 里。 + + 一致性只来自数据库：``UPDATE ... WHERE id AND user_id AND version=`` 配合 + rowcount 判定，多 worker 并发下不存在「读-改-写」的检查后竞争窗口。 + ``rowcount == 0`` 时再回读区分「不存在/非本人」（返回 None）与 + 「版本冲突」（抛 :class:`DraftConcurrentWriteError`）。 + """ + values: dict[str, Any] = { + "version": RoundtableDraftRow.version + 1, + "updated_at": datetime.now(UTC), + } + if task_id is not _UNSET and task_id is not None: + values["task_id"] = str(task_id) + if title is not _UNSET and title is not None: + values["title"] = str(title).strip() + if furthest_step is not _UNSET and furthest_step is not None: + # furthest_step 单调不回退：迟到的自动保存不得把已推进的步骤拉回去。 + step_value = int(furthest_step) + values["furthest_step"] = case( + (RoundtableDraftRow.furthest_step >= step_value, RoundtableDraftRow.furthest_step), + else_=step_value, + ) + if step1 is not _UNSET: + values["step1"] = _dumps(step1) + if step2 is not _UNSET: + values["step2"] = _dumps(step2) + if step3 is not _UNSET: + values["step3"] = _dumps(step3) + + where = [ + RoundtableDraftRow.id == draft_id, # 个人草稿严格按 user 隔离（task 聊天记录已迁出到独立存储）。 - if row is None or row.user_id != user_id: + RoundtableDraftRow.user_id == user_id, + ] + if expected_version is not None: + where.append(RoundtableDraftRow.version == expected_version) + + async with self._sf() as session: + result = await session.execute( + update(RoundtableDraftRow).where(*where).values(**values) + ) + if result.rowcount == 0: + if expected_version is not None: + # 回读区分冲突与不存在（回读在冲突分支，正常路径零额外开销）。 + existing = await session.get(RoundtableDraftRow, draft_id) + if existing is not None and existing.user_id == user_id: + raise DraftConcurrentWriteError( + draft_id, expected=expected_version, actual=existing.version + ) return None - if task_id is not _UNSET and task_id is not None: - row.task_id = str(task_id) - if title is not _UNSET and title is not None: - row.title = str(title).strip() - if furthest_step is not _UNSET and furthest_step is not None: - row.furthest_step = int(furthest_step) - if step1 is not _UNSET: - row.step1 = _dumps(step1) - if step2 is not _UNSET: - row.step2 = _dumps(step2) - if step3 is not _UNSET: - row.step3 = _dumps(step3) - row.updated_at = datetime.now(UTC) await session.commit() + row = await session.get(RoundtableDraftRow, draft_id) await session.refresh(row) return self._draft_to_dict(row) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_jobs/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_jobs/model.py index 03af942d..c48feef7 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_jobs/model.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_jobs/model.py @@ -19,7 +19,8 @@ from __future__ import annotations from datetime import UTC, datetime -from sqlalchemy import Integer, String, Text +import sqlalchemy as sa +from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column from deerflow.persistence.base import Base @@ -28,6 +29,13 @@ from deerflow.persistence.types import BeijingDateTime, PortableLongText class RoundtableJobRow(Base): __tablename__ = "roundtable_jobs" + __table_args__ = ( + # 「同一 scope 至多一个活跃作业」由该唯一键强制（Phase 3）：活跃时存 + # sha256(draft_id + task_id|user_id)，进入终态时清 NULL；MySQL/SQLite 唯一索引 + # 都允许多个 NULL，故终态历史作业不占键。INSERT 冲突即幂等命中 —— 一致性来自 + # DB 唯一键，不再依赖 GET_LOCK 命名锁。 + UniqueConstraint("active_dedupe_key", name="uq_roundtable_jobs_active_dedupe_key"), + ) id: Mapped[str] = mapped_column(String(64), primary_key=True) # The draft/session this job runs for. Indexed for "active job of draft" lookups. @@ -37,6 +45,24 @@ class RoundtableJobRow(Base): # **不按 user 分权**（任何人打开同一 task 都能看在跑的研讨）；其 draft_id 指向独立的 # ``roundtable_task_drafts`` 表。为空 = 普通个人作业（按 user 分权）。 task_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True) + # ── Phase 3：唯一活跃键 + 租约 + dispatcher ────────────────────────────── + # 前端启动请求的幂等键（同键重试返回同一作业）与入参指纹（审计/排查）。 + request_id: Mapped[str | None] = mapped_column(String(64), nullable=True) + request_hash: Mapped[str | None] = mapped_column(String(64), nullable=True) + # 活跃去重键：活跃（非终态）时 = sha256(draft_id + task_id|user_id)，终态清 NULL。 + # 唯一索引（__table_args__）保证「同一 scope 至多一个活跃作业」——INSERT 冲突即幂等。 + active_dedupe_key: Mapped[str | None] = mapped_column(String(64), nullable=True) + # 启动入参完整 JSON 快照：dispatcher 在任意 worker 领取 queued/租约过期作业时 + # 据此重建 StartParams（跨进程可接管的前提；resume 续跑也重写本列）。 + input_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) + # 租约：条件 UPDATE 抢占（queued 或租约过期）+ 心跳续租；worker 崩溃后租约自然过期， + # 其它 worker 接管。恢复能力来自 DB 而非进程内存。 + lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True) + lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True) + # 被 dispatcher 领取执行的次数（接管计数）。 + attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0")) + # 行级 CAS 版本：条件 UPDATE 乐观锁，防止并发写互相覆盖。 + version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0")) # Job status machine (§2.1): queued / running / awaiting_input / done / error / cancelled. status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True) # Fine-grained phase (§2.2): initializing / leader_thinking / dispatching / @@ -71,6 +97,11 @@ class RoundtableJobRow(Base): step3: Mapped[str | None] = mapped_column(PortableLongText, nullable=True) # Terminal error detail when status=error. error: Mapped[str | None] = mapped_column(Text, nullable=True) + # 草稿回写 pending 标记：``_write_run_to_draft`` 乐观锁重试耗尽后置 1（绝不退化 + # last-write-wins 盲写）；由 outbox 补偿器投影结果到草稿后清 0。一致性来自 DB 标记。 + draft_persist_pending: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False, server_default=sa.false() + ) # 时间列必须用 BeijingDateTime（项目规范，与 ai_writing_sessions / roundtable_drafts 对齐）。 created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)) updated_at: Mapped[datetime] = mapped_column( diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_jobs/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_jobs/sql.py index 7ff979db..efe442be 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_jobs/sql.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_jobs/sql.py @@ -12,11 +12,14 @@ threading a user id through, while gateway routes pass it explicitly. from __future__ import annotations +import hashlib import json +import logging from datetime import UTC, datetime from typing import Any -from sqlalchemy import delete, select +from sqlalchemy import delete, or_, select, update +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from deerflow.persistence.roundtable_jobs.model import RoundtableJobRow @@ -24,8 +27,36 @@ from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id _UNSET = object() +logger = logging.getLogger(__name__) + # 终态：列表/清理判定用。 TERMINAL_STATUSES = ("done", "error", "cancelled") +# 非终态（活跃）：唯一活跃键去重 / 领取判定用。 +ACTIVE_STATUSES = ("queued", "running", "awaiting_input") +# cancel 可命中的状态（queued/running/awaiting_input → cancel_requested）。 +CANCELLABLE_STATUSES = ("queued", "running", "awaiting_input") + + +def compute_active_dedupe_key( + draft_id: str | None, user_id: str | None, task_id: str | None +) -> str | None: + """活跃去重键：``sha256(scope)``。scope = draft_id + (task_id | user_id)。 + + - task 作业（task_id 非空）：按 (draft_id, task_id) 去重 —— 跨用户共享； + - 普通作业：按 (draft_id, user_id) 去重； + - draft_id 为空 → 无法去重，返回 None（直接创建，保持原行为）。 + + 该键写入 ``active_dedupe_key`` 唯一索引：作业活跃时占用，进入终态时清 NULL + （MySQL/SQLite 唯一索引都允许多个 NULL）——「同一 scope 至多一个活跃作业」 + 由 DB 唯一键强制，INSERT 冲突即幂等命中。 + """ + if not draft_id: + return None + if task_id: + scope = f"task:{draft_id}:{task_id}" + else: + scope = f"user:{draft_id}:{user_id or ''}" + return hashlib.sha256(scope.encode("utf-8")).hexdigest() def _loads(value: Any) -> Any: @@ -52,15 +83,72 @@ class RoundtableJobRepository: @staticmethod def _row_to_dict(row: RoundtableJobRow) -> dict[str, Any]: d = row.to_dict() - for k in ("created_at", "updated_at"): + for k in ("created_at", "updated_at", "lease_until"): if isinstance(d.get(k), datetime): d[k] = d[k].isoformat() - for k in ("dispatch_chain", "thread_ids", "chain", "orchestration_plan", "dialogues", "step3"): + for k in ( + "dispatch_chain", + "thread_ids", + "chain", + "orchestration_plan", + "dialogues", + "step3", + "input_snapshot", + ): d[k] = _loads(d.get(k)) return d # ── create ──────────────────────────────────────────────────────────── + @staticmethod + def _make_row( + *, + id: str, + resolved_user_id: str | None, + draft_id: str | None, + task_id: str | None, + orchestration_mode: str, + coordinator_name: str | None, + thread_ids: dict[str, str] | None, + chain: dict[str, Any] | None, + orchestration_plan: dict[str, Any] | None, + dispatch_chain: list[dict[str, Any]] | None, + status: str, + phase: str, + request_id: str | None = None, + request_hash: str | None = None, + active_dedupe_key: str | None = None, + input_snapshot: dict[str, Any] | None = None, + ) -> RoundtableJobRow: + now = datetime.now(UTC) + return RoundtableJobRow( + id=id, + draft_id=draft_id, + user_id=resolved_user_id, + task_id=(task_id or None), + request_id=(request_id or None), + request_hash=(request_hash or None), + active_dedupe_key=(active_dedupe_key or None), + input_snapshot=_dumps(input_snapshot), + status=status, + phase=phase, + cycle=0, + dispatch_chain=_dumps(dispatch_chain), + active_agent_id=None, + consensus_percentage=0, + pending_clarification=None, + thread_ids=_dumps(thread_ids), + coordinator_name=coordinator_name, + orchestration_mode=orchestration_mode, + chain=_dumps(chain), + orchestration_plan=_dumps(orchestration_plan), + error=None, + attempt=0, + version=0, + created_at=now, + updated_at=now, + ) + async def create( self, *, @@ -78,27 +166,19 @@ class RoundtableJobRepository: phase: str = "initializing", ) -> dict[str, Any]: resolved_user_id = resolve_user_id(user_id, method_name="RoundtableJobRepository.create") - now = datetime.now(UTC) - row = RoundtableJobRow( + row = self._make_row( id=id, + resolved_user_id=resolved_user_id, draft_id=draft_id, - user_id=resolved_user_id, - task_id=(task_id or None), + task_id=task_id, + orchestration_mode=orchestration_mode, + coordinator_name=coordinator_name, + thread_ids=thread_ids, + chain=chain, + orchestration_plan=orchestration_plan, + dispatch_chain=dispatch_chain, status=status, phase=phase, - cycle=0, - dispatch_chain=_dumps(dispatch_chain), - active_agent_id=None, - consensus_percentage=0, - pending_clarification=None, - thread_ids=_dumps(thread_ids), - coordinator_name=coordinator_name, - orchestration_mode=orchestration_mode, - chain=_dumps(chain), - orchestration_plan=_dumps(orchestration_plan), - error=None, - created_at=now, - updated_at=now, ) async with self._sf() as session: session.add(row) @@ -106,6 +186,102 @@ class RoundtableJobRepository: await session.refresh(row) return self._row_to_dict(row) + async def try_create_or_get_active( + self, + *, + draft_id: str | None, + user_id: str | None | _AutoSentinel, + task_id: str | None, + id: str, + orchestration_mode: str = "recommend", + coordinator_name: str | None = None, + thread_ids: dict[str, str] | None = None, + chain: dict[str, Any] | None = None, + orchestration_plan: dict[str, Any] | None = None, + dispatch_chain: list[dict[str, Any]] | None = None, + status: str = "queued", + phase: str = "initializing", + request_id: str | None = None, + request_hash: str | None = None, + input_snapshot: dict[str, Any] | None = None, + ) -> tuple[dict[str, Any], bool]: + """原子地「创建活跃作业 → 冲突则幂等返回已有活跃作业」（Phase 3 唯一活跃键版）。 + + 一致性来自 DB **唯一索引** ``uq_roundtable_jobs_active_dedupe_key``：活跃作业占用 + ``active_dedupe_key = sha256(draft_id + task_id|user_id)``，进入终态时清 NULL。于是 + 「同一 scope 至多一个活跃作业」由唯一键强制——直接 INSERT，冲突（IntegrityError）即 + 说明已有活跃作业，读回它幂等返回（``created=False``）。**不再依赖 GET_LOCK 命名锁**： + 唯一键在 MySQL / PostgreSQL / SQLite 上语义一致，跨进程/跨 worker 天然互斥。 + + 极小竞态兜底：INSERT 冲突后读回时若胜者恰好已进终态（键已释放），则重试一次 INSERT。 + + 返回 ``(job_dict, created)``。``draft_id`` 为空时无法去重，直接创建（保持原行为）。 + """ + resolved_user_id = resolve_user_id( + user_id, method_name="RoundtableJobRepository.try_create_or_get_active" + ) + dedupe_key = compute_active_dedupe_key(draft_id, resolved_user_id, task_id) + + def _build_row() -> RoundtableJobRow: + return self._make_row( + id=id, + resolved_user_id=resolved_user_id, + draft_id=draft_id, + task_id=task_id, + orchestration_mode=orchestration_mode, + coordinator_name=coordinator_name, + thread_ids=thread_ids, + chain=chain, + orchestration_plan=orchestration_plan, + dispatch_chain=dispatch_chain, + status=status, + phase=phase, + request_id=request_id, + request_hash=request_hash, + active_dedupe_key=dedupe_key, + input_snapshot=input_snapshot, + ) + + # 无 draft_id → 无法去重，直接创建（保持原有行为）。 + if dedupe_key is None: + row = _build_row() + async with self._sf() as session: + session.add(row) + await session.commit() + await session.refresh(row) + return self._row_to_dict(row), True + + for _attempt in range(2): + try: + async with self._sf() as session: + row = _build_row() + session.add(row) + await session.commit() + await session.refresh(row) + return self._row_to_dict(row), True + except IntegrityError: + pass # 唯一键冲突 → 已有活跃作业（或极小竞态，见下）。 + existing = await self._get_by_dedupe_key(dedupe_key) + if existing is not None: + return self._row_to_dict(existing), False + # 胜者在冲突与我们读回之间进了终态（键已释放）→ 重试 INSERT 可成功。 + + # 两次都冲突且读不回 —— 理论上不可达；保守读回/报错，绝不双建。 + existing = await self._get_by_dedupe_key(dedupe_key) + if existing is not None: + return self._row_to_dict(existing), False + raise RuntimeError( + f"roundtable job dedupe conflict but no active row found for key {dedupe_key}" + ) + + async def _get_by_dedupe_key(self, dedupe_key: str) -> RoundtableJobRow | None: + """按唯一活跃键读回占用该键的（活跃）作业行；无则 None。""" + stmt = select(RoundtableJobRow).where( + RoundtableJobRow.active_dedupe_key == dedupe_key + ) + async with self._sf() as session: + return (await session.execute(stmt)).scalars().first() + # ── update progress ───────────────────────────────────────────────────── async def update_progress( @@ -125,8 +301,80 @@ class RoundtableJobRepository: dialogues=_UNSET, step3=_UNSET, error=_UNSET, + lease_owner: str | None = None, ) -> dict[str, Any] | None: + """更新进度字段。 + + **租约门（Phase 3）**：传入 ``lease_owner`` 时走**原子条件 UPDATE**——要求 DB 里 + ``lease_owner`` 匹配且状态不在 ``cancel_requested/cancelled``，否则返回 None（写被 + 拒）。这保证：① 只有持租者能推进度；② 一旦 cancel 落库（cancel_requested），迟到的 + 进度写不会覆盖取消意图；③ 多 worker 下非持租者的过期写直接失效。一致性来自 DB 条件 + UPDATE，而非读后判断。 + + **终态处理**：写入 status ∈ 终态（done/error/cancelled）时，同时清空 + ``active_dedupe_key``（释放唯一活跃键，允许该 scope 再建作业）与租约字段。 + + 不传 ``lease_owner`` 时保持原「按 user 分权 + 读改写」行为（向后兼容旧调用/测试）。 + """ resolved_user_id = resolve_user_id(user_id, method_name="RoundtableJobRepository.update_progress") + + # ── 租约门：原子条件 UPDATE ── + if lease_owner is not None: + values: dict[str, Any] = {"updated_at": datetime.now(UTC)} + if status is not _UNSET: + values["status"] = status + if phase is not _UNSET: + values["phase"] = phase + if cycle is not _UNSET: + values["cycle"] = int(cycle) + if dispatch_chain is not _UNSET: + values["dispatch_chain"] = _dumps(dispatch_chain) + if active_agent_id is not _UNSET: + values["active_agent_id"] = active_agent_id + if consensus_percentage is not _UNSET: + values["consensus_percentage"] = int(consensus_percentage) + if pending_clarification is not _UNSET: + values["pending_clarification"] = pending_clarification + if thread_ids is not _UNSET: + values["thread_ids"] = _dumps(thread_ids) + if coordinator_name is not _UNSET: + values["coordinator_name"] = coordinator_name + if dialogues is not _UNSET: + values["dialogues"] = _dumps(dialogues) + if step3 is not _UNSET: + values["step3"] = _dumps(step3) + if error is not _UNSET: + values["error"] = error + # 终态：释放唯一活跃键 + 清租约。 + if status is not _UNSET and status in TERMINAL_STATUSES: + values["active_dedupe_key"] = None + values["lease_owner"] = None + values["lease_until"] = None + # awaiting_input：任务已暂停、执行协程退出 —— 必须释放租约，否则 resume 前 + # 租约未过期没人能重新领取。活跃键保留（作业仍未终态，继续占着 scope）。 + elif status is not _UNSET and status == "awaiting_input": + values["lease_owner"] = None + values["lease_until"] = None + values["version"] = RoundtableJobRow.version + 1 + + where = [ + RoundtableJobRow.id == job_id, + RoundtableJobRow.lease_owner == lease_owner, + RoundtableJobRow.status.not_in(["cancel_requested", "cancelled"]), + ] + if resolved_user_id is not None: + where.append(RoundtableJobRow.user_id == resolved_user_id) + async with self._sf() as session: + result = await session.execute( + update(RoundtableJobRow).where(*where).values(**values) + ) + if result.rowcount == 0: + return None + await session.commit() + row = await session.get(RoundtableJobRow, job_id) + return self._row_to_dict(row) if row is not None else None + + # ── 兼容路径：按 user 分权 + 读改写（不持租约的旧调用）── async with self._sf() as session: row = await session.get(RoundtableJobRow, job_id) if row is None: @@ -157,11 +405,319 @@ class RoundtableJobRepository: row.step3 = _dumps(step3) if error is not _UNSET: row.error = error + # 终态同样要释放唯一活跃键（兼容路径没有租约也要清键）。 + if status is not _UNSET and status in TERMINAL_STATUSES: + row.active_dedupe_key = None + row.lease_owner = None + row.lease_until = None + # awaiting_input 同样释放租约（与租约门路径一致）。 + elif status is not _UNSET and status == "awaiting_input": + row.lease_owner = None + row.lease_until = None row.updated_at = datetime.now(UTC) await session.commit() await session.refresh(row) return self._row_to_dict(row) + async def mark_draft_persist_pending(self, job_id: str) -> bool: + """把草稿回写 pending 标记置 1（``_write_run_to_draft`` 重试耗尽时调用）。 + + 一致性来自 DB：绝不退化为 last-write-wins 盲写，而是留下可被 outbox 补偿器 + （Phase 3）发现的标记。返回是否命中了作业行。 + """ + async with self._sf() as session: + result = await session.execute( + update(RoundtableJobRow) + .where(RoundtableJobRow.id == job_id) + .values(draft_persist_pending=True, updated_at=datetime.now(UTC)) + ) + if result.rowcount == 0: + return False + await session.commit() + return True + + # ── Phase 3：租约领取 / 续租 / 释放（dispatcher） ──────────────────────── + + async def list_claimable(self, *, now: datetime, limit: int = 20) -> list[str]: + """列出可领取的作业 id：``queued``（新作业/续跑入队）或 ``running`` 但租约已过期 + （持租 worker 崩溃/卡死）。按 updated_at 升序（先来先领），最多 limit 个。 + + 这只是「候选列表」；真正的领取靠 {@link claim_job} 的条件 UPDATE —— 多 worker 并发 + 扫到同一候选时，只有一个 UPDATE 能命中（一致性来自 DB）。 + """ + stmt = ( + select(RoundtableJobRow.id) + .where( + or_( + RoundtableJobRow.status == "queued", + (RoundtableJobRow.status == "running") + & (RoundtableJobRow.lease_until.is_not(None)) + & (RoundtableJobRow.lease_until < now), + ) + ) + .order_by(RoundtableJobRow.updated_at.asc()) + .limit(limit) + ) + async with self._sf() as session: + result = await session.execute(stmt) + return [r[0] for r in result.all()] + + async def list_cancel_reapable(self, *, now: datetime, limit: int = 20) -> list[str]: + """列出可以由 dispatcher 收口的取消请求。 + + ``cancel_requested`` 是取消的权威意图，但持租 worker 可能在看门狗收口前崩溃。 + 仅当作业没有租约或租约已过期时才允许 dispatcher 无租约收口；仍持有有效租约的 + 作业继续由原 worker 的看门狗处理，避免抢占仍在正常退出的执行器。 + """ + stmt = ( + select(RoundtableJobRow.id) + .where( + RoundtableJobRow.status == "cancel_requested", + or_( + RoundtableJobRow.lease_owner.is_(None), + RoundtableJobRow.lease_until.is_(None), + RoundtableJobRow.lease_until < now, + ), + ) + .order_by(RoundtableJobRow.updated_at.asc()) + .limit(limit) + ) + async with self._sf() as session: + result = await session.execute(stmt) + return [r[0] for r in result.all()] + + async def claim_job( + self, job_id: str, *, lease_owner: str, lease_until: datetime + ) -> dict[str, Any] | None: + """原子条件 UPDATE 领取一个作业：仅当它仍是可领取状态时才成功。 + + 成功条件（DB 保证，跨 worker 互斥）：``status='queued'``，或 ``status='running'`` + 且租约已过期。成功则置 ``status='running'``、写 ``lease_owner/lease_until``、 + ``attempt+1``、``version+1``，返回行；否则返回 None（别的 worker 先领了 / 状态已变）。 + """ + values = { + "status": "running", + "lease_owner": lease_owner, + "lease_until": lease_until, + "attempt": RoundtableJobRow.attempt + 1, + "version": RoundtableJobRow.version + 1, + "updated_at": datetime.now(UTC), + } + where = [ + RoundtableJobRow.id == job_id, + or_( + RoundtableJobRow.status == "queued", + (RoundtableJobRow.status == "running") + & (RoundtableJobRow.lease_until.is_not(None)) + & (RoundtableJobRow.lease_until < datetime.now(UTC)), + ), + ] + async with self._sf() as session: + result = await session.execute( + update(RoundtableJobRow).where(*where).values(**values) + ) + if result.rowcount == 0: + return None + await session.commit() + row = await session.get(RoundtableJobRow, job_id) + if row is None: + return None + # Phase 7 observability: attempt >= 2 means this job was previously + # claimed by another worker whose lease expired (crash / OOM / GC + # stall) and is now being reclaimed. This is the key signal for + # diagnosing worker instability under load. + if row.attempt >= 2: + logger.warning( + "roundtable_jobs lease takeover: job=%s reclaimed by %s " + "(attempt=%d — previous worker's lease expired)", + job_id, lease_owner, row.attempt, + ) + return self._row_to_dict(row) + + async def renew_lease( + self, job_id: str, *, lease_owner: str, lease_until: datetime + ) -> bool: + """心跳续租：仅持租者可延长 ``lease_until``。返回是否续租成功。""" + async with self._sf() as session: + result = await session.execute( + update(RoundtableJobRow) + .where( + RoundtableJobRow.id == job_id, + RoundtableJobRow.lease_owner == lease_owner, + RoundtableJobRow.status == "running", + ) + .values(lease_until=lease_until, updated_at=datetime.now(UTC)) + ) + if result.rowcount == 0: + return False + await session.commit() + return True + + async def release_lease(self, job_id: str, *, lease_owner: str | None = None) -> bool: + """释放租约（进入 awaiting_input 等「暂停但非终态」时调用）。 + + 置 ``lease_owner/lease_until`` 为 NULL，使该作业可被任意 worker 重新领取（续跑）。 + 传入 ``lease_owner`` 时仅当匹配才释放（防止误放他人的租约）。 + """ + where = [RoundtableJobRow.id == job_id] + if lease_owner is not None: + where.append(RoundtableJobRow.lease_owner == lease_owner) + async with self._sf() as session: + result = await session.execute( + update(RoundtableJobRow) + .where(*where) + .values(lease_owner=None, lease_until=None, updated_at=datetime.now(UTC)) + ) + if result.rowcount == 0: + return False + await session.commit() + return True + + # ── Phase 3：resume / cancel 条件状态机 ────────────────────────────────── + + async def queue_for_resume( + self, + job_id: str, + *, + input_snapshot: dict[str, Any] | None = None, + user_id: str | None | _AutoSentinel = AUTO, + task_id: str | None = None, + ) -> dict[str, Any] | None: + """原子条件 UPDATE：``awaiting_input → queued``（续跑入队），仅第一个成功方有效。 + + 多 worker / 重复点击下可能有多个 resume 请求并发；条件 UPDATE（WHERE status= + 'awaiting_input'）保证只有一个能把它翻成 queued，其余返回 None（路由据此拒绝）。 + 成功后清租约（交给 dispatcher 重新领取），并把含续跑语义的新 snapshot 写回。 + """ + resolved_user_id = resolve_user_id( + user_id, method_name="RoundtableJobRepository.queue_for_resume" + ) + values: dict[str, Any] = { + "status": "queued", + "phase": "initializing", + "pending_clarification": None, + "lease_owner": None, + "lease_until": None, + "version": RoundtableJobRow.version + 1, + "updated_at": datetime.now(UTC), + } + if input_snapshot is not None: + values["input_snapshot"] = _dumps(input_snapshot) + where = [RoundtableJobRow.id == job_id, RoundtableJobRow.status == "awaiting_input"] + if task_id is not None: + where.append(RoundtableJobRow.task_id == task_id) + elif resolved_user_id is not None: + where.append(RoundtableJobRow.user_id == resolved_user_id) + async with self._sf() as session: + result = await session.execute( + update(RoundtableJobRow).where(*where).values(**values) + ) + if result.rowcount == 0: + return None + await session.commit() + row = await session.get(RoundtableJobRow, job_id) + return self._row_to_dict(row) if row is not None else None + + async def request_cancel( + self, + job_id: str, + *, + user_id: str | None | _AutoSentinel = AUTO, + task_id: str | None = None, + ) -> dict[str, Any] | None: + """原子条件 UPDATE：可取消态（queued/running/awaiting_input）→ ``cancel_requested``。 + + cancel 的**第一步**（权威意图落库）。成功后路由再 best-effort 取消本进程任务； + 实际终态 ``cancelled`` 由持租者看门狗（检测到 cancel_requested）或 ``finalize_cancel`` + 写入 —— 保证「已取消」终态由持租者收口，避免竞态。重复 cancel 幂等（已 cancel_requested/ + cancelled 时返回 None，路由据此回 409 或返回现状）。 + """ + resolved_user_id = resolve_user_id( + user_id, method_name="RoundtableJobRepository.request_cancel" + ) + values: dict[str, Any] = { + "status": "cancel_requested", + "version": RoundtableJobRow.version + 1, + "updated_at": datetime.now(UTC), + } + where = [ + RoundtableJobRow.id == job_id, + RoundtableJobRow.status.in_(list(CANCELLABLE_STATUSES)), + ] + if task_id is not None: + where.append(RoundtableJobRow.task_id == task_id) + elif resolved_user_id is not None: + where.append(RoundtableJobRow.user_id == resolved_user_id) + async with self._sf() as session: + result = await session.execute( + update(RoundtableJobRow).where(*where).values(**values) + ) + if result.rowcount == 0: + return None + await session.commit() + row = await session.get(RoundtableJobRow, job_id) + return self._row_to_dict(row) if row is not None else None + + async def finalize_cancel( + self, + job_id: str, + *, + lease_owner: str | None = None, + ) -> dict[str, Any] | None: + """把 ``cancel_requested``（或孤儿 queued/running）收口为终态 ``cancelled``。 + + 由持租者看门狗（检测 cancel_requested 后）或路由（作业尚在 queued、无持租者时）调用。 + 条件 UPDATE：仅当 ``status='cancel_requested'`` 时翻转；同时清唯一活跃键 + 租约。 + 传入 ``lease_owner`` 时要求租约匹配（持租者收口），否则不加租约条件（queued 无租约）。 + """ + values: dict[str, Any] = { + "status": "cancelled", + "active_dedupe_key": None, + "lease_owner": None, + "lease_until": None, + "version": RoundtableJobRow.version + 1, + "updated_at": datetime.now(UTC), + } + where = [RoundtableJobRow.id == job_id, RoundtableJobRow.status == "cancel_requested"] + if lease_owner is not None: + where.append(RoundtableJobRow.lease_owner == lease_owner) + async with self._sf() as session: + result = await session.execute( + update(RoundtableJobRow).where(*where).values(**values) + ) + if result.rowcount == 0: + return None + await session.commit() + row = await session.get(RoundtableJobRow, job_id) + return self._row_to_dict(row) if row is not None else None + + # ── Phase 3：outbox 补偿器（draft_persist_pending 投影） ───────────────── + + async def list_draft_persist_pending(self, *, limit: int = 50) -> list[dict[str, Any]]: + """列出 ``draft_persist_pending=1`` 的作业行（补偿器待投影清单）。""" + stmt = ( + select(RoundtableJobRow) + .where(RoundtableJobRow.draft_persist_pending.is_(True)) + .order_by(RoundtableJobRow.updated_at.asc()) + .limit(limit) + ) + async with self._sf() as session: + result = await session.execute(stmt) + return [self._row_to_dict(r) for r in result.scalars()] + + async def clear_draft_persist_pending(self, job_id: str) -> bool: + """投影成功后清 ``draft_persist_pending`` 标记。返回是否命中。""" + async with self._sf() as session: + result = await session.execute( + update(RoundtableJobRow) + .where(RoundtableJobRow.id == job_id) + .values(draft_persist_pending=False, updated_at=datetime.now(UTC)) + ) + if result.rowcount == 0: + return False + await session.commit() + return True + # ── read ──────────────────────────────────────────────────────────────── async def get( diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_task_drafts/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_task_drafts/model.py index 707ba655..ec601c00 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_task_drafts/model.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_task_drafts/model.py @@ -50,3 +50,6 @@ class RoundtableTaskDraftRow(Base): default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC), ) + # 乐观并发控制版本号：每次 update_draft 自增；传入 expected_version 时校验，命中 + # 0 行即并发冲突（防 step2.runs 读-改-写丢失更新——多用户/多 worker 并发）。 + version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_task_drafts/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_task_drafts/sql.py index fad0ef0a..33bdafa2 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_task_drafts/sql.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/roundtable_task_drafts/sql.py @@ -19,9 +19,10 @@ import re from datetime import UTC, datetime from typing import Any -from sqlalchemy import select +from sqlalchemy import case, select, update from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from deerflow.persistence.roundtable_drafts.sql import DraftConcurrentWriteError from deerflow.persistence.roundtable_task_drafts.model import RoundtableTaskDraftRow _UNSET = object() @@ -77,6 +78,8 @@ class RoundtableTaskDraftRepository: "task_id": row.task_id, "title": row.title, "furthest_step": row.furthest_step, + # 乐观锁版本号：列表也携带，供前端本地副本对账（冲突时重拉合并）。 + "version": row.version, # 所用业务链名（历史卡片展示用）；无链条 → None → 前端显示「自由模式」。 "chain_title": _extract_chain_title(row.step2), "created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else row.created_at, @@ -159,24 +162,52 @@ class RoundtableTaskDraftRepository: step1=_UNSET, step2=_UNSET, step3=_UNSET, + expected_version: int | None = None, ) -> dict[str, Any] | None: - """Partial update by id. ``user_id`` is ignored (signature-compat only).""" + """Partial update by id (``user_id`` ignored — signature-compat only). + + 与个人草稿一致：原子条件 UPDATE，版本判定与写入在同一条 SQL 里， + 多 worker 并发下无检查后竞争窗口。``rowcount == 0`` 时回读区分 + 「不存在」与「版本冲突」。 + """ + values: dict[str, Any] = { + "version": RoundtableTaskDraftRow.version + 1, + "updated_at": datetime.now(UTC), + } + if title is not _UNSET and title is not None: + values["title"] = str(title).strip() + if furthest_step is not _UNSET and furthest_step is not None: + # furthest_step 单调不回退：迟到的自动保存不得把已推进的步骤拉回去。 + step_value = int(furthest_step) + values["furthest_step"] = case( + (RoundtableTaskDraftRow.furthest_step >= step_value, RoundtableTaskDraftRow.furthest_step), + else_=step_value, + ) + if step1 is not _UNSET: + values["step1"] = _dumps(step1) + if step2 is not _UNSET: + values["step2"] = _dumps(step2) + if step3 is not _UNSET: + values["step3"] = _dumps(step3) + + where = [RoundtableTaskDraftRow.id == draft_id] + if expected_version is not None: + where.append(RoundtableTaskDraftRow.version == expected_version) + async with self._sf() as session: - row = await session.get(RoundtableTaskDraftRow, draft_id) - if row is None: + result = await session.execute( + update(RoundtableTaskDraftRow).where(*where).values(**values) + ) + if result.rowcount == 0: + if expected_version is not None: + existing = await session.get(RoundtableTaskDraftRow, draft_id) + if existing is not None: + raise DraftConcurrentWriteError( + draft_id, expected=expected_version, actual=existing.version + ) return None - if title is not _UNSET and title is not None: - row.title = str(title).strip() - if furthest_step is not _UNSET and furthest_step is not None: - row.furthest_step = int(furthest_step) - if step1 is not _UNSET: - row.step1 = _dumps(step1) - if step2 is not _UNSET: - row.step2 = _dumps(step2) - if step3 is not _UNSET: - row.step3 = _dumps(step3) - row.updated_at = datetime.now(UTC) await session.commit() + row = await session.get(RoundtableTaskDraftRow, draft_id) await session.refresh(row) return self._to_dict(row) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/runtime/runs/worker.py b/offline-backend-20260512/backend/packages/harness/deerflow/runtime/runs/worker.py index 01e205b1..5e36391b 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/runtime/runs/worker.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/runtime/runs/worker.py @@ -15,10 +15,10 @@ internal checkpoint callbacks that are not exposed in the Python public API. from __future__ import annotations -import asyncio -import copy -import inspect -import logging +import asyncio +import copy +import inspect +import logging from dataclasses import dataclass, field from functools import lru_cache from typing import TYPE_CHECKING, Any, Literal, cast @@ -28,31 +28,31 @@ from langgraph.checkpoint.base import empty_checkpoint if TYPE_CHECKING: from langchain_core.messages import HumanMessage -from deerflow.agents.middlewares.title_middleware import ( - finalize_thread_title, - log_title_debug, -) -from deerflow.config.app_config import AppConfig -from deerflow.runtime.serialization import serialize -from deerflow.runtime.stream_bridge import StreamBridge +from deerflow.agents.middlewares.title_middleware import ( + finalize_thread_title, + log_title_debug, +) +from deerflow.config.app_config import AppConfig +from deerflow.runtime.serialization import serialize +from deerflow.runtime.stream_bridge import StreamBridge from .manager import RunManager, RunRecord from .schemas import RunStatus -logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) # Valid stream_mode values for LangGraph's graph.astream() -_VALID_LG_MODES = {"values", "updates", "checkpoints", "tasks", "debug", "messages", "custom"} - - -def _log_background_title_finalize_failure(thread_id: str): - def _done(task: asyncio.Task[bool]) -> None: - try: - task.result() - except Exception: - logger.debug("Failed to finalize background title for thread %s", thread_id, exc_info=True) - - return _done +_VALID_LG_MODES = {"values", "updates", "checkpoints", "tasks", "debug", "messages", "custom"} + + +def _log_background_title_finalize_failure(thread_id: str): + def _done(task: asyncio.Task[bool]) -> None: + try: + task.result() + except Exception: + logger.debug("Failed to finalize background title for thread %s", thread_id, exc_info=True) + + return _done def _build_runtime_context( @@ -364,7 +364,40 @@ async def run_agent( else: await run_manager.set_status(run_id, RunStatus.interrupted) else: - await run_manager.set_status(run_id, RunStatus.success) + # LLMErrorHandlingMiddleware intentionally converts an exhausted + # provider failure into a user-safe AIMessage instead of raising. + # That keeps a malformed upstream response out of the SSE transport, + # but previously made this worker record the run as ``success``. + # The UI consequently had no error event to show and looked as if + # generation had silently stopped halfway through. Inspect the + # terminal checkpoint before declaring success so the persisted run + # state and the user-visible SSE outcome stay consistent. + terminal_llm_error = await _terminal_llm_error_reason( + checkpointer=checkpointer, + thread_id=thread_id, + ) + if terminal_llm_error is not None: + error_message = "本轮模型生成未能完成，已保留当前对话内容，请稍后重试。" + logger.warning( + "Run %s ended with LLM fallback message (reason=%s)", + run_id, + terminal_llm_error, + ) + await run_manager.set_status( + run_id, + RunStatus.error, + error=error_message, + ) + await bridge.publish( + run_id, + "error", + { + "message": error_message, + "name": "ModelGenerationError", + }, + ) + else: + await run_manager.set_status(run_id, RunStatus.success) except asyncio.CancelledError: action = record.abort_action @@ -414,47 +447,47 @@ async def run_agent( except Exception: logger.warning("Failed to persist run completion for %s (non-fatal)", run_id, exc_info=True) - # Sync title from checkpoint to threads_meta.display_name - if checkpointer is not None and thread_store is not None: - try: - ckpt_config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}} - ckpt_tuple = await checkpointer.aget_tuple(ckpt_config) - if ckpt_tuple is not None: - ckpt = getattr(ckpt_tuple, "checkpoint", {}) or {} - channel_values = ckpt.get("channel_values", {}) or {} - title = channel_values.get("title") - if title: - await thread_store.update_display_name(thread_id, title) - if ( - record.status == RunStatus.success - and channel_values.get("title_provisional") is True - ): - log_title_debug( - "Run %s: scheduling background title generation for thread %s", - run_id, - thread_id, - ) - title_task = asyncio.create_task( - finalize_thread_title( - checkpointer=checkpointer, - thread_store=thread_store, - thread_id=thread_id, - source_state=copy.deepcopy(channel_values), - app_config=ctx.app_config, - ), - name=f"title-finalize:{thread_id}", - ) - title_task.add_done_callback(_log_background_title_finalize_failure(thread_id)) - elif record.status == RunStatus.success: - log_title_debug( - "Run %s: background title generation not scheduled for thread %s because title_provisional=%r title=%r", - run_id, - thread_id, - channel_values.get("title_provisional"), - channel_values.get("title"), - ) - except Exception: - logger.debug("Failed to sync title for thread %s (non-fatal)", thread_id) + # Sync title from checkpoint to threads_meta.display_name + if checkpointer is not None and thread_store is not None: + try: + ckpt_config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}} + ckpt_tuple = await checkpointer.aget_tuple(ckpt_config) + if ckpt_tuple is not None: + ckpt = getattr(ckpt_tuple, "checkpoint", {}) or {} + channel_values = ckpt.get("channel_values", {}) or {} + title = channel_values.get("title") + if title: + await thread_store.update_display_name(thread_id, title) + if ( + record.status == RunStatus.success + and channel_values.get("title_provisional") is True + ): + log_title_debug( + "Run %s: scheduling background title generation for thread %s", + run_id, + thread_id, + ) + title_task = asyncio.create_task( + finalize_thread_title( + checkpointer=checkpointer, + thread_store=thread_store, + thread_id=thread_id, + source_state=copy.deepcopy(channel_values), + app_config=ctx.app_config, + ), + name=f"title-finalize:{thread_id}", + ) + title_task.add_done_callback(_log_background_title_finalize_failure(thread_id)) + elif record.status == RunStatus.success: + log_title_debug( + "Run %s: background title generation not scheduled for thread %s because title_provisional=%r title=%r", + run_id, + thread_id, + channel_values.get("title_provisional"), + channel_values.get("title"), + ) + except Exception: + logger.debug("Failed to sync title for thread %s (non-fatal)", thread_id) # Update threads_meta status based on run outcome if thread_store is not None: @@ -484,6 +517,65 @@ async def _call_checkpointer_method(checkpointer: Any, async_name: str, sync_nam return result +async def _terminal_llm_error_reason( + *, + checkpointer: Any | None, + thread_id: str, +) -> str | None: + """Return the terminal LLM fallback reason, if this run ended in one. + + ``LLMErrorHandlingMiddleware`` persists its fallback as an AI message with + an internal marker. A normal agent graph treats that message as a valid + result and completes without an exception, so the worker must check the + saved conversation rather than relying solely on ``astream`` raising. + """ + if checkpointer is None: + return None + + try: + checkpoint_config = { + "configurable": { + "thread_id": thread_id, + "checkpoint_ns": "", + } + } + checkpoint_tuple = await _call_checkpointer_method( + checkpointer, + "aget_tuple", + "get_tuple", + checkpoint_config, + ) + if checkpoint_tuple is None: + return None + checkpoint = getattr(checkpoint_tuple, "checkpoint", {}) or {} + channel_values = checkpoint.get("channel_values", {}) + messages = channel_values.get("messages", []) if isinstance(channel_values, dict) else [] + if not isinstance(messages, list): + return None + + # Walk backwards to the latest assistant message. Tool/human messages + # may be appended around it by middleware, but an older fallback should + # never turn a later successful response into an error. + for message in reversed(messages): + message_type = message.get("type") if isinstance(message, dict) else getattr(message, "type", None) + if message_type not in {"ai", "assistant"}: + continue + from deerflow.agents.middlewares.llm_error_handling_middleware import ( + is_llm_error_message, + ) + + return is_llm_error_message(message) + except Exception: + # A status check must never turn a completed answer into a failed run + # merely because a custom checkpointer cannot expose its latest tuple. + logger.debug( + "Could not inspect terminal LLM state for thread %s", + thread_id, + exc_info=True, + ) + return None + + async def _rollback_to_pre_run_checkpoint( *, checkpointer: Any, diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/clarification_tool.py b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/clarification_tool.py index 7420260a..652a94ac 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/clarification_tool.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/clarification_tool.py @@ -3,7 +3,13 @@ from typing import Literal from langchain.tools import tool -@tool("ask_clarification", parse_docstring=True, return_direct=True) +# ClarificationMiddleware owns the stop behaviour: it returns a Command that +# writes the ToolMessage and explicitly routes to END. ``return_direct=True`` +# makes the prebuilt tool node short-circuit before that Command update is +# checkpointed in some LangGraph versions, leaving an assistant tool call +# without its matching ToolMessage. The next user reply then fails provider +# validation with "tool_calls must be followed by tool messages". +@tool("ask_clarification", parse_docstring=True, return_direct=False) def ask_clarification_tool( question: str, clarification_type: Literal[ diff --git a/offline-backend-20260512/backend/skill-packages/position-roundtable-action-create.skill b/offline-backend-20260512/backend/skill-packages/position-roundtable-action-create.skill new file mode 100644 index 00000000..785bd065 Binary files /dev/null and b/offline-backend-20260512/backend/skill-packages/position-roundtable-action-create.skill differ diff --git a/offline-backend-20260512/backend/tests/test_position_roundtable_sessions.py b/offline-backend-20260512/backend/tests/test_position_roundtable_sessions.py index b159196d..6cebce60 100644 --- a/offline-backend-20260512/backend/tests/test_position_roundtable_sessions.py +++ b/offline-backend-20260512/backend/tests/test_position_roundtable_sessions.py @@ -38,7 +38,10 @@ async def store(tmp_path): import deerflow.persistence.position_roundtable.model # noqa: F401 await conn.run_sync(Base.metadata.create_all) - return PositionRoundtableRepository(async_sessionmaker(engine, expire_on_commit=False)) + try: + yield PositionRoundtableRepository(async_sessionmaker(engine, expire_on_commit=False)) + finally: + await engine.dispose() def test_reject_node_request_allows_an_optional_reason(): @@ -1206,6 +1209,8 @@ async def test_builtin_summary_and_action_plan_snapshots_are_invalidated_by_reje def test_rejected_or_stale_node_requires_a_new_markdown_delivery() -> None: + from deerflow.persistence.position_roundtable.delivery import markdown_delivery_changed + old = [ { "name": "report.md", @@ -1214,8 +1219,8 @@ def test_rejected_or_stale_node_requires_a_new_markdown_delivery() -> None: "size_bytes": 100, } ] - assert not position_roundtable_router._markdown_delivery_changed(old, old) - assert position_roundtable_router._markdown_delivery_changed( + assert not markdown_delivery_changed(old, old) + assert markdown_delivery_changed( old, [{**old[0], "modified_at": "2026-07-21T00:05:00+00:00", "size_bytes": 120}], ) @@ -1358,6 +1363,17 @@ async def test_locked_node_cannot_bind_a_thread_before_its_stage_is_ready(): "thread_id": None, } + async def apply_node_command( + self, session_id, node_key, user_id, *, command_id, command_type, payload=None, + expected_session_version=None, + ): + # Mirrors the in-lock rejection a locked node produces in the real store. + return { + "ok": False, + "http_status": 409, + "detail": "上游节点尚未完成，当前节点未解锁", + } + request = SimpleNamespace( app=SimpleNamespace(state=SimpleNamespace(position_roundtable_store=LockedStore())), state=SimpleNamespace(user=SimpleNamespace(id="user-a")),