diff --git a/.gitignore b/.gitignore index b22d5f03..32e39254 100644 --- a/.gitignore +++ b/.gitignore @@ -93,6 +93,8 @@ offline-backend-20260512/backend/tests/* # Keep the model configuration regression coverage versioned even though # this deployment repository otherwise ignores newly added backend tests. !offline-backend-20260512/backend/tests/test_model_management.py +# Keep fixed-question exact-match and streaming regression coverage versioned. +!offline-backend-20260512/backend/tests/test_fixed_questions.py deploy .history diff --git a/frontend-web/src/App.tsx b/frontend-web/src/App.tsx index 2833dea3..0e684bf7 100644 --- a/frontend-web/src/App.tsx +++ b/frontend-web/src/App.tsx @@ -102,16 +102,17 @@ function AppProviders({ children }: { children: React.ReactNode }) { forcedTheme={forcedTheme} > - - - - {!positionStandaloneMode && } - {!positionStandaloneMode && } - - {children} - - - + + + + + {!positionStandaloneMode && } + + {children} + + + + ); diff --git a/frontend-web/src/components/page-sidebar.tsx b/frontend-web/src/components/page-sidebar.tsx index 5119a2ba..dd85aeeb 100644 --- a/frontend-web/src/components/page-sidebar.tsx +++ b/frontend-web/src/components/page-sidebar.tsx @@ -10,6 +10,7 @@ import { FolderOpen, Gauge, ListTree, + MessageSquareText, MousePointerClick, BriefcaseBusiness, Settings, @@ -237,6 +238,14 @@ function SecondaryNavSidebar({ headerHidden }: { headerHidden: boolean }) { 按钮管理 )} + {isAdmin && ( + navigate("/page/strategy/admin/fixed-questions")} + > + + 问题配置 + + )} {isAdmin && ( navigate("/page/strategy/admin/position-roles")} @@ -558,6 +567,12 @@ function LegacyAccordionSidebar({ 按钮管理 )} + {isAdmin && ( + navigate("/page/strategy/admin/fixed-questions")}> + + 问题配置 + + )} {isAdmin && ( navigate("/page/strategy/admin/position-roles")}> diff --git a/frontend-web/src/components/workspace/llmwiki-knowledge-selector.tsx b/frontend-web/src/components/workspace/llmwiki-knowledge-selector.tsx index b1d0333b..afdbd6f3 100644 --- a/frontend-web/src/components/workspace/llmwiki-knowledge-selector.tsx +++ b/frontend-web/src/components/workspace/llmwiki-knowledge-selector.tsx @@ -76,6 +76,7 @@ function SelectionContent({ error, source, variant, + readOnly, }: { rows: LlmWikiKnowledgeBase[]; value: string[]; @@ -86,6 +87,7 @@ function SelectionContent({ error: unknown; source: SelectorSource; variant: SelectorVariant; + readOnly: boolean; }) { const selected = useMemo(() => new Set(value), [value]); const filtered = useMemo(() => { @@ -97,6 +99,7 @@ function SelectionContent({ }, [rows, search]); const groups = useMemo(() => groupRows(filtered), [filtered]); const toggle = (id: string) => { + if (readOnly) return; onChange(selected.has(id) ? value.filter((item) => item !== id) : [...value, id]); }; @@ -168,6 +171,7 @@ function SelectionContent({ toggle(item.id)} onSelect={(event) => event.preventDefault()} className={cn( @@ -183,6 +187,7 @@ function SelectionContent({ type="button" role="checkbox" aria-checked={selected.has(item.id)} + disabled={readOnly} onClick={() => toggle(item.id)} className={cn( "my-0.5 flex min-h-11 w-full items-center gap-2 rounded-md px-2 py-1.5 text-left outline-none transition-colors hover:bg-muted/60 focus-visible:ring-2 focus-visible:ring-primary", @@ -205,15 +210,21 @@ function SelectionContent({ - 已选择 {value.length} 个 - - onChange(filtered.map((item) => item.id))}> - 全选 - - onChange([])}> - 清空 - - + + {readOnly ? `本对话已选 ${value.length} 个` : `已选择 ${value.length} 个`} + + {readOnly ? ( + 对话开始后不可更改 + ) : ( + + onChange(filtered.map((item) => item.id))}> + 全选 + + onChange([])}> + 清空 + + + )} > ); @@ -226,6 +237,7 @@ export function LlmWikiKnowledgeSelector({ agentId, autoSelectAll = false, disabled = false, + readOnly = false, className, label = "知识空间", variant = "popover", @@ -236,6 +248,8 @@ export function LlmWikiKnowledgeSelector({ agentId?: string | null; autoSelectAll?: boolean; disabled?: boolean; + /** Keep the selected knowledge spaces visible without allowing the thread scope to change. */ + readOnly?: boolean; className?: string; label?: string; variant?: SelectorVariant; @@ -293,6 +307,7 @@ export function LlmWikiKnowledgeSelector({ error={error} source={source} variant={variant} + readOnly={readOnly} /> ); @@ -315,9 +330,14 @@ export function LlmWikiKnowledgeSelector({ ); } - const selectedName = value.length === 1 - ? rows.find((item) => item.id === value[0])?.name - : null; + const selectedNames = value + .map((id) => rows.find((item) => item.id === id)?.name) + .filter((name): name is string => !!name); + const selectedLabel = selectedNames.length > 0 + ? selectedNames.join("、") + : value.length > 0 + ? `已选 ${value.length} 个知识空间` + : label; return ( @@ -332,9 +352,10 @@ export function LlmWikiKnowledgeSelector({ className, )} disabled={disabled || loading} + title={value.length > 0 ? selectedLabel : undefined} > {loading ? : } - {selectedName || label} + {selectedLabel} {value.length > 0 && ( {value.length} diff --git a/frontend-web/src/components/workspace/workspace-nav-menu.tsx b/frontend-web/src/components/workspace/workspace-nav-menu.tsx index 25865950..e15573ad 100644 --- a/frontend-web/src/components/workspace/workspace-nav-menu.tsx +++ b/frontend-web/src/components/workspace/workspace-nav-menu.tsx @@ -11,6 +11,7 @@ import { GaugeIcon, HistoryIcon, MessageCircleQuestionIcon, + MessageSquareTextIcon, ScrollTextIcon, Settings2Icon, SettingsIcon, @@ -268,6 +269,16 @@ export function WorkspaceNavMenu() { 推荐问题配置 )} + {isAdmin && ( + + navigate("/page/strategy/admin/fixed-questions") + } + > + + 问题配置 + + )} {isAdmin && logSystem?.enabled && logSystem.url && ( diff --git a/frontend-web/src/core/auth/index.ts b/frontend-web/src/core/auth/index.ts index 36eb5542..64ad9bad 100644 --- a/frontend-web/src/core/auth/index.ts +++ b/frontend-web/src/core/auth/index.ts @@ -49,6 +49,17 @@ function normalizeIdentityPart(value: unknown): string { return typeof value === "string" ? value.trim().toLowerCase() : ""; } +function identityFromLoginName(value: unknown): string | null { + const normalized = normalizeIdentityPart(value); + if (!normalized) return null; + // Explicit email login is domain-specific. Bare username login is the + // stable business principal even if the backend's synthetic email domain or + // internal UUID changes. + return normalized.includes("@") + ? `email:${normalized}` + : `account:${normalized}`; +} + function readUserInfoIdentity(): string | null { let info: Record | null = null; try { @@ -81,20 +92,24 @@ function readUserInfoIdentity(): string | null { /** * A stable cross-tab identity key for "did the account really change?" checks. * - * Some external login paths can re-issue a backend ``user_id`` while resolving - * to the same business account. Prefer business-facing identifiers - * (email/username/userInfo) and keep ``user_id`` only as the final fallback. + * Some external login paths can re-issue a backend ``user_id`` or use a + * different synthetic email domain while resolving to the same business + * account. Prefer the raw login username returned by the backend, then the + * email local-part, and keep ``user_id`` only as the final fallback. */ export function getAccountIdentityKey( auth: LoginByUsernameResponse | null = getStoredAuth(), ): string | null { if (!auth) return null; - const email = normalizeIdentityPart(auth.email); - if (email) return `email:${email}`; + const loginIdentity = identityFromLoginName(auth.username); + if (loginIdentity) return loginIdentity; - const username = normalizeIdentityPart(auth.username); - if (username) return `username:${username}`; + const email = normalizeIdentityPart(auth.email); + if (email) { + const localPart = email.split("@", 1)[0]; + return localPart ? `account:${localPart}` : `email:${email}`; + } const userInfoIdentity = readUserInfoIdentity(); if (userInfoIdentity) return `userInfo:${userInfoIdentity}`; diff --git a/frontend-web/src/core/auth/sync.tsx b/frontend-web/src/core/auth/sync.tsx index 65321bd3..b4fc5c95 100644 --- a/frontend-web/src/core/auth/sync.tsx +++ b/frontend-web/src/core/auth/sync.tsx @@ -1,9 +1,15 @@ "use client"; -import { useEffect, useRef } from "react"; +import { Fragment, useEffect, useRef, useState, type ReactNode } from "react"; +import { useLocation, useNavigate } from "react-router-dom"; +import { useQueryClient } from "@tanstack/react-query"; import { toast } from "sonner"; -import { getAccountIdentityKey } from "@/core/auth"; +import { + AUTH_STATE_CHANGED_EVENT, + getAccountDisplayName, + getAccountIdentityKey, +} from "@/core/auth"; import { isEmbedActive, isIsolatedSession } from "@/core/embed/embed-session"; const AUTH_STORAGE_KEY = "deerflow.auth"; @@ -14,14 +20,37 @@ function readIdentityKey(): string | null { return getAccountIdentityKey(); } +function freshRouteForAccount(pathname: string): string { + // A thread URL belongs to the old account. Keep its workspace/agent context + // but move to a valid new conversation before the user can submit into the + // stale thread. Other pages can stay in place and are remounted below. + const chatMatch = pathname.match(/^(.*\/chats)\/[^/]+\/?$/); + return chatMatch ? `${chatMatch[1]}/new` : pathname; +} + /** - * Mount once at the app root. Watches localStorage auth changes from other - * tabs and lets the user decide when to refresh on account switches. + * Cross-tab auth boundary for normal pages. + * + * Same-account token re-issuance is ignored. A genuine account change clears + * account-scoped query state, remounts the page, and moves an open chat to its + * fresh-chat route without a full browser reload. Embedded/isolated tabs keep + * their sessionStorage identity and never participate in this synchronization. */ -export function AuthSyncWatcher() { +export function AuthSyncWatcher({ + children, + disabled = false, +}: { + children?: ReactNode; + disabled?: boolean; +}) { const lastIdentityRef = useRef(readIdentityKey()); + const [revision, setRevision] = useState(0); + const queryClient = useQueryClient(); + const navigate = useNavigate(); + const location = useLocation(); useEffect(() => { + if (disabled) return; // 嵌入(iframe)态 / 独立会话态(?isolate=1):鉴权走按标签页的 sessionStorage,本就不与其它标签页 // 共享,且不应被其它标签页的 localStorage 账号变更打扰 —— 直接不挂跨标签监听。 if (isEmbedActive() || isIsolatedSession()) return; @@ -36,18 +65,27 @@ export function AuthSyncWatcher() { lastIdentityRef.current = nextIdentity; + void queryClient.cancelQueries(); + queryClient.clear(); + window.dispatchEvent(new Event(AUTH_STATE_CHANGED_EVENT)); + setRevision((value) => value + 1); + if (nextIdentity === null) { + toast.info("账号已在其它页面退出", { + id: IDENTITY_TOAST_ID, + description: "当前页面已同步退出状态。", + }); + navigate("/login", { replace: true }); return; } - toast.warning("检测到账号在其它标签页发生变化", { + const freshRoute = freshRouteForAccount(location.pathname); + if (freshRoute !== location.pathname) { + navigate(freshRoute, { replace: true }); + } + toast.info("账号已在页面间自动同步", { id: IDENTITY_TOAST_ID, - description: "当前标签页仍是原会话。如需与新账号同步,请刷新页面。", - duration: Infinity, - action: { - label: "刷新同步", - onClick: () => window.location.reload(), - }, + description: `当前账号:${getAccountDisplayName()}。无需刷新页面。`, }); } @@ -68,14 +106,24 @@ export function AuthSyncWatcher() { settleTimer = setTimeout(evaluateSettledIdentity, SETTLE_DELAY_MS); } + function handleSameTabAuthChange() { + // `storage` never fires in the tab that called setStoredAuth/clearStoredAuth. + // Keep the comparison baseline current so a later same-account token + // renewal from another normal page is not compared with the pre-login + // `null` identity and falsely classified as an account switch. + lastIdentityRef.current = readIdentityKey(); + } + window.addEventListener("storage", handleStorage); + window.addEventListener(AUTH_STATE_CHANGED_EVENT, handleSameTabAuthChange); return () => { window.removeEventListener("storage", handleStorage); + window.removeEventListener(AUTH_STATE_CHANGED_EVENT, handleSameTabAuthChange); if (settleTimer) { clearTimeout(settleTimer); } }; - }, []); + }, [disabled, location.pathname, navigate, queryClient]); - return null; + return children ? {children} : null; } diff --git a/frontend-web/src/core/auth/use-live-account.ts b/frontend-web/src/core/auth/use-live-account.ts index 8cfedd8c..b855973f 100644 --- a/frontend-web/src/core/auth/use-live-account.ts +++ b/frontend-web/src/core/auth/use-live-account.ts @@ -8,7 +8,7 @@ import { getAccountIdentityKey, getStoredAuth, } from "@/core/auth"; -import { isEmbedActive } from "@/core/embed/embed-session"; +import { isEmbedActive, isIsolatedSession } from "@/core/embed/embed-session"; const AUTH_STORAGE_KEY = "deerflow.auth"; @@ -65,7 +65,9 @@ export function useLiveAccount(): LiveAccount { refresh(); } } - if (!isEmbedActive()) { + // Normal pages share localStorage identity across tabs. Embedded and + // isolated pages deliberately keep their per-tab sessionStorage identity. + if (!isEmbedActive() && !isIsolatedSession()) { window.addEventListener("storage", onStorage); } window.addEventListener(AUTH_STATE_CHANGED_EVENT, refresh); diff --git a/frontend-web/src/core/threads/hooks.ts b/frontend-web/src/core/threads/hooks.ts index 5c235bb1..7d26057e 100644 --- a/frontend-web/src/core/threads/hooks.ts +++ b/frontend-web/src/core/threads/hooks.ts @@ -696,17 +696,15 @@ export function useThreadStream({ } return; } - // A "thread not found" 404 almost always means the URL points at a - // thread from a previous login session — either this tab outlived a - // logout/re-login or the thread genuinely belongs to a different user. - // Re-frame the error around the *current* logged-in identity and bounce - // the user to a fresh chat instead of leaving them stuck on the stale - // route looking at a raw HTTP 404. - if (isThreadNotAccessibleError(message)) { - const userLabel = buildCurrentUserLabel(); - toast.info( - `当前登录用户为 ${userLabel},原对话不存在或无权访问,正在为你刷新页面…`, - ); + // A "thread not found" 404 means this route cannot accept another turn. + // Move to a fresh chat without a full browser reload; the old reload path + // made every click look like a crash and could loop indefinitely when a + // different tab's cookie kept overriding this tab's Bearer token. + if (isThreadNotAccessibleError(message)) { + const userLabel = buildCurrentUserLabel(); + toast.info( + `当前登录用户为 ${userLabel},原对话不存在或无权访问,已为你打开新对话。`, + ); // Prefer staying inside an agent chat context when the user is on an // agent thread URL — otherwise fall back to the global new chat. const hash = typeof window !== "undefined" ? window.location.hash : ""; @@ -717,18 +715,7 @@ export function useThreadStream({ : iframeMatch ? "/page/workspace/chats_iframe/new" : "/page/workspace/chats/new"; - // First navigate to a safe URL so the post-reload page lands on a - // valid route. Then full-reload so TanStack Query cache, context - // providers and any other state captured under the previous user's - // identity get rebuilt against the current login. - if (typeof window !== "undefined") { - window.location.hash = target; - window.setTimeout(() => { - window.location.reload(); - }, 900); - } else { - navigate(target, { replace: true }); - } + navigate(target, { replace: true }); return; } // 上一轮还没结束又发了新问题(409)或并发达到上限(429): diff --git a/frontend-web/src/core/threads/thread-access-error.ts b/frontend-web/src/core/threads/thread-access-error.ts index dd803081..68ec23dd 100644 --- a/frontend-web/src/core/threads/thread-access-error.ts +++ b/frontend-web/src/core/threads/thread-access-error.ts @@ -13,8 +13,9 @@ import { getStoredAuth } from "@/core/auth"; * 404 just leaves the user staring at a scary error. Both the chat stream * (``core/threads/hooks.ts``) and the AI-writing stream * (``open-canvas/contexts/AIWritingContext.tsx``) funnel their stream errors - * through here so the UX is identical: detect the login-change 404 and prompt - * the user to refresh against the current identity instead of erroring out. + * through here so the UX is identical: detect the login-change 404 and reset + * the stale in-memory session against the current identity instead of showing + * a raw error or reloading the browser page. */ /** Normalize an unknown stream error into a human-readable message string. */ @@ -77,22 +78,15 @@ export function buildCurrentUserLabel(): string { } /** - * Re-frame a login-change 404 around the current identity and reload the page. + * Re-frame a login-change 404 around the current identity and reset its view. * - * Shows a friendly toast naming the current login, then full-reloads after a - * short delay so the toast is visible and so TanStack Query cache / context - * providers / any state captured under the previous user get rebuilt against - * the current login. Used by streams whose session id lives in component state - * (not the URL) — a plain reload lands them back on a fresh, valid page. + * Streams whose session id lives in component state pass a local reset callback + * so they can discard the inaccessible session without a browser reload. */ -export function notifyThreadAccessChangedAndReload(): void { +export function notifyThreadAccessChanged(onReset: () => void): void { const userLabel = buildCurrentUserLabel(); toast.info( - `当前登录用户为 ${userLabel},原会话不存在或无权访问,正在为你刷新页面…`, + `当前登录用户为 ${userLabel},原会话不存在或无权访问,已为你重置为新会话。`, ); - if (typeof window !== "undefined") { - window.setTimeout(() => { - window.location.reload(); - }, 900); - } + onReset(); } diff --git a/frontend-web/src/open-canvas/contexts/AIWritingContext.tsx b/frontend-web/src/open-canvas/contexts/AIWritingContext.tsx index 2b0b9f0a..de4e300f 100644 --- a/frontend-web/src/open-canvas/contexts/AIWritingContext.tsx +++ b/frontend-web/src/open-canvas/contexts/AIWritingContext.tsx @@ -18,7 +18,7 @@ import { getAPIClient } from '@/core/api'; import { getStreamErrorMessage, isThreadNotAccessibleError, - notifyThreadAccessChangedAndReload, + notifyThreadAccessChanged, } from '@/core/threads/thread-access-error'; import { useCanvasGraphContext } from '../contexts/CanvasGraphContext'; @@ -299,10 +299,13 @@ const AIWritingStreamHost = memo(function AIWritingStreamHost({ // thread belongs to a previous login session — the user switched accounts // (or this tab outlived a re-login), so the current account legitimately // can't access the old thread. Instead of dumping the raw HTTP 404 into - // the panel, re-frame it around the current identity and reload — same UX - // the chat stream gives (see core/threads/hooks.ts). + // the panel, re-frame it around the current identity and discard the + // stale component-state session without reloading the browser page. if (isThreadNotAccessibleError(message)) { - notifyThreadAccessChangedAndReload(); + notifyThreadAccessChanged(() => { + continueTargetRef.current = null; + dispatchRef.current?.({ type: 'reset' }); + }); return; } dispatchRef.current?.({ type: 'set_error', message }); @@ -410,6 +413,13 @@ export function AIWritingProvider({ children }: { children: ReactNode }) { // (刷新会断开 SSE,后端默认 on_disconnect=cancel 取消该 run,不留孤儿)。 const threadId = state.sessionId ?? undefined; + const resetAfterThreadAccessError = useCallback(() => { + setArtifact(null); + seenInterruptRef.current = null; + continueTargetRef.current = null; + dispatch({ type: 'reset' }); + }, [setArtifact]); + useEffect(() => { const s = state.status; const savable = s === 'done' || s === 'error' || s.startsWith('awaiting_'); @@ -473,12 +483,12 @@ export function AIWritingProvider({ children }: { children: ReactNode }) { } catch (err) { const message = getStreamErrorMessage(err); if (isThreadNotAccessibleError(message)) { - notifyThreadAccessChangedAndReload(); + notifyThreadAccessChanged(resetAfterThreadAccessError); return; } dispatch({ type: 'set_error', message }); } - }, [state.status]); + }, [state.status, resetAfterThreadAccessError]); const submitIntervention = useCallback(async (payload: UserInterventionPayload) => { if (!state.sessionId || !state.currentPause) return; @@ -585,14 +595,14 @@ export function AIWritingProvider({ children }: { children: ReactNode }) { } catch (err) { const message = getStreamErrorMessage(err); if (isThreadNotAccessibleError(message)) { - notifyThreadAccessChangedAndReload(); + notifyThreadAccessChanged(resetAfterThreadAccessError); return; } dispatch({ type: 'set_error', message }); } }, [state.sessionId, state.currentPause, state.materials, state.outline, state.currentDraftTitle, state.currentDraftMarkdown, state.reviewResult, - state.status]); + state.status, resetAfterThreadAccessError]); const pauseWriting = useCallback(() => { // 真停:取消当前 run(graph 已完成的阶段已落 checkpoint)。继续时从该 checkpoint 续跑。 @@ -626,13 +636,13 @@ export function AIWritingProvider({ children }: { children: ReactNode }) { } catch (err) { const message = getStreamErrorMessage(err); if (isThreadNotAccessibleError(message)) { - notifyThreadAccessChangedAndReload(); + notifyThreadAccessChanged(resetAfterThreadAccessError); return; } dispatch({ type: 'set_error', message }); } })(); - }, [state.request]); + }, [state.request, resetAfterThreadAccessError]); // Restore an unfinished history session: make it active, load its checkpoint, then let // the host skip completed awaiting_* stages or re-run the current unfinished stage. diff --git a/frontend-web/src/pages/AgentChatPage.tsx b/frontend-web/src/pages/AgentChatPage.tsx index 0d9d57a3..6339b226 100644 --- a/frontend-web/src/pages/AgentChatPage.tsx +++ b/frontend-web/src/pages/AgentChatPage.tsx @@ -775,6 +775,17 @@ export default function AgentChatPage({ onContextChange={(context) => setSettings("context", context) } + knowledgeSelector={ + + } onFollowupsVisibilityChange={setShowFollowups} onSubmit={handleSubmit} onStop={handleStop} diff --git a/frontend-web/src/pages/ChatPage.tsx b/frontend-web/src/pages/ChatPage.tsx index 08c09d7b..c60da639 100644 --- a/frontend-web/src/pages/ChatPage.tsx +++ b/frontend-web/src/pages/ChatPage.tsx @@ -837,20 +837,19 @@ export default function ChatPage() { : undefined } knowledgeSelector={ - isLandingState ? ( - - ) : undefined + } onFollowupsVisibilityChange={setShowFollowups} onSubmit={handleSubmit} diff --git a/frontend-web/src/pages/FixedQuestionsPage.tsx b/frontend-web/src/pages/FixedQuestionsPage.tsx new file mode 100644 index 00000000..98e276da --- /dev/null +++ b/frontend-web/src/pages/FixedQuestionsPage.tsx @@ -0,0 +1,310 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { Navigate } from "react-router-dom"; +import { + Info, + MessageSquareText, + Plus, + Save, + Search, + Trash2, + Zap, +} from "lucide-react"; +import { toast } from "sonner"; + +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import { Textarea } from "@/components/ui/textarea"; +import { getStoredAuth } from "@/core/auth"; +import { + type FixedQuestion, + useFixedQuestionActions, + useFixedQuestions, +} from "@/strategy-components/api/fixed-questions"; + +let sequence = 1; + +function newId(): string { + if (typeof crypto !== "undefined" && "randomUUID" in crypto) { + return crypto.randomUUID(); + } + return `fixed-${Date.now()}-${sequence++}`; +} + +function serialize(rows: FixedQuestion[]): string { + return JSON.stringify( + rows + .map(({ id, question, answer, enabled, tokensPerSecond, sortOrder }) => ({ + id, + question, + answer, + enabled, + tokensPerSecond, + sortOrder, + })) + .sort((a, b) => a.id.localeCompare(b.id)), + ); +} + +export default function FixedQuestionsPage() { + const isAdmin = getStoredAuth()?.system_role === "admin"; + const { data: serverQuestions = [], isLoading } = useFixedQuestions(); + const { save } = useFixedQuestionActions(); + const [rows, setRows] = useState([]); + const [loadedKey, setLoadedKey] = useState(null); + const [query, setQuery] = useState(""); + + const sourceKey = useMemo( + () => serialize(serverQuestions), + [serverQuestions], + ); + useEffect(() => { + if (loadedKey !== sourceKey) { + setRows(serverQuestions); + setLoadedKey(sourceKey); + } + }, [loadedKey, serverQuestions, sourceKey]); + + const dirty = useMemo(() => serialize(rows) !== sourceKey, [rows, sourceKey]); + const search = query.trim().toLowerCase(); + const visibleRows = useMemo( + () => + search + ? rows.filter( + (row) => + row.question.toLowerCase().includes(search) || + row.answer.toLowerCase().includes(search), + ) + : rows, + [rows, search], + ); + const enabledCount = rows.filter( + (row) => row.enabled && row.question.trim() && row.answer.trim(), + ).length; + + if (!isAdmin) { + return ; + } + + const patchRow = (id: string, patch: Partial) => { + setRows((current) => + current.map((row) => (row.id === id ? { ...row, ...patch } : row)), + ); + }; + + const addRow = () => { + setRows((current) => [ + { + id: newId(), + question: "", + answer: "", + enabled: true, + tokensPerSecond: 100, + sortOrder: current.length, + }, + ...current, + ]); + }; + + const removeRow = (id: string) => { + setRows((current) => current.filter((row) => row.id !== id)); + }; + + const handleSave = async () => { + const cleaned = rows.map((row, index) => ({ + ...row, + question: row.question.trim(), + tokensPerSecond: Math.round(row.tokensPerSecond), + sortOrder: index, + })); + for (const row of cleaned) { + if (!row.question) { + toast.error("问题不能为空"); + return; + } + if (!row.answer.trim()) { + toast.error(`问题「${row.question}」的回答不能为空`); + return; + } + if ( + !Number.isFinite(row.tokensPerSecond) || + row.tokensPerSecond < 1 || + row.tokensPerSecond > 1000 + ) { + toast.error(`问题「${row.question}」的流式速度须为 1–1000 token/s`); + return; + } + } + const seen = new Set(); + for (const row of cleaned) { + if (seen.has(row.question)) { + toast.error(`问题「${row.question}」重复,请删除或修改重复项`); + return; + } + seen.add(row.question); + } + try { + await save.mutateAsync(cleaned); + toast.success("问题配置已保存并生效"); + } catch (error) { + toast.error(error instanceof Error ? error.message : "保存失败"); + } + }; + + return ( + + + + + + + + + + + 问题配置 + + {rows.length} 条 / {enabledCount} 条启用 + + + + 为完全相同的问题配置固定回答,命中时直接流式返回,不调用大模型。 + + + + + + + 新增问题 + + + + {save.isPending ? "保存中…" : "保存配置"} + + + + + + + 匹配区分大小写、标点和空格;停用的配置会保留但不参与匹配。带附件或图片的提问不会使用固定回答。 + + + + + + + + setQuery(event.target.value)} + placeholder="搜索问题或回答…" + className="pl-9" + /> + + + + 启用项保存后立即生效 + + + + {isLoading ? ( + + 加载中… + + ) : visibleRows.length === 0 ? ( + + {rows.length === 0 + ? "暂无问题配置,点击「新增问题」开始配置" + : "没有匹配的问题或回答"} + + ) : ( + + {visibleRows.map((row, index) => ( + + + + + {index + 1} + + 固定问答 + + + + 流式速度 + + patchRow(row.id, { + tokensPerSecond: Number(event.target.value), + }) + } + className="h-8 w-24" + aria-label="流式返回速度" + /> + token/s + + + + patchRow(row.id, { enabled }) + } + /> + {row.enabled ? "已启用" : "已停用"} + + removeRow(row.id)} + className="flex size-8 items-center justify-center rounded-lg text-muted-foreground transition-colors hover:bg-destructive/10 hover:text-destructive" + > + + + + + + + 问题 + + patchRow(row.id, { question: event.target.value }) + } + placeholder="输入需要精确匹配的问题" + className="min-h-28 resize-y" + /> + + + 回答 + + patchRow(row.id, { answer: event.target.value }) + } + placeholder="输入命中后流式返回的回答(支持 Markdown)" + className="min-h-28 resize-y" + /> + + + + ))} + + )} + + + ); +} diff --git a/frontend-web/src/pages/StrategyRoutes.tsx b/frontend-web/src/pages/StrategyRoutes.tsx index bcce5f92..76b6419b 100644 --- a/frontend-web/src/pages/StrategyRoutes.tsx +++ b/frontend-web/src/pages/StrategyRoutes.tsx @@ -45,6 +45,7 @@ import LightAppListPage from "./LightAppListPage"; import LightAppManagementPage from "./LightAppManagementPage"; import MenuManagementPage from "./MenuManagementPage"; import LoginWhitelistPage from "./LoginWhitelistPage"; +import FixedQuestionsPage from "./FixedQuestionsPage"; import SensitiveWordsPage from "./SensitiveWordsPage"; import SkillAddressesPage from "./SkillAddressesPage"; import TaskButtonsPage from "./TaskButtonsPage"; @@ -443,6 +444,16 @@ export default function StrategyRoutes() { ) } /> + + ) : ( + + ) + } + /> { + if (clarificationSeen) return; setInteractiveMessages((prev) => prev.map((m) => m.id === agentMsgId @@ -505,6 +511,7 @@ export function useStep1Intent(options: UseStep1IntentOptions): UseStep1IntentRe ); }, onClarification: (clarification, index) => { + clarificationSeen = true; // 实时把每条澄清问题灌进气泡 —— 用户能边流边看到「协助过程」逐个出现, // 而不是等终态帧(终态帧只带第 1 个问题)。同时往步骤链补一条 // 「追问 N:<问题>」,与「强化步骤链」诉求对齐。 diff --git a/frontend-web/src/strategy-components/api/fixed-questions.ts b/frontend-web/src/strategy-components/api/fixed-questions.ts new file mode 100644 index 00000000..90e29cda --- /dev/null +++ b/frontend-web/src/strategy-components/api/fixed-questions.ts @@ -0,0 +1,122 @@ +/** Administrator-managed exact-match fixed question/answer API. */ + +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { apiFetch } from "@/core/api/fetch-client"; +import { getBackendBaseURL } from "@/core/config"; +import { OptType, writeYLog } from "@/core/log/y-log"; + +const BASE = () => `${getBackendBaseURL()}/api/fixed-questions`; + +export interface FixedQuestion { + id: string; + question: string; + answer: string; + enabled: boolean; + tokensPerSecond: number; + sortOrder: number; + updatedBy?: string | null; + updatedAt?: string | null; +} + +interface ServerFixedQuestion { + id: string; + question: string; + answer: string; + enabled: boolean; + tokens_per_second: number; + sort_order: number; + updated_by?: string | null; + updated_at?: string | null; +} + +function fromServer(row: ServerFixedQuestion): FixedQuestion { + return { + id: row.id, + question: row.question ?? "", + answer: row.answer ?? "", + enabled: row.enabled ?? true, + tokensPerSecond: row.tokens_per_second ?? 100, + sortOrder: row.sort_order ?? 0, + updatedBy: row.updated_by ?? null, + updatedAt: row.updated_at ?? null, + }; +} + +function toServer(row: FixedQuestion): ServerFixedQuestion { + return { + id: row.id, + question: row.question, + answer: row.answer, + enabled: row.enabled, + tokens_per_second: row.tokensPerSecond, + sort_order: row.sortOrder, + }; +} + +async function readJson(response: Response, context: string): Promise { + if (!response.ok) { + const data = await response.json().catch(() => null as unknown); + const detail = + data && typeof data === "object" && "detail" in data + ? String((data as { detail: unknown }).detail) + : `${context} failed (${response.status})`; + throw new Error(detail); + } + return (await response.json()) as T; +} + +export async function listFixedQuestions(): Promise { + const response = await apiFetch(BASE(), { method: "GET" }); + const data = await readJson<{ questions: ServerFixedQuestion[] }>( + response, + "list fixed questions", + ); + return (data.questions ?? []).map(fromServer); +} + +export async function replaceFixedQuestions( + questions: FixedQuestion[], +): Promise { + const response = await apiFetch(BASE(), { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ questions: questions.map(toServer) }), + }); + const data = await readJson<{ questions: ServerFixedQuestion[] }>( + response, + "save fixed questions", + ); + const saved = (data.questions ?? []).map(fromServer); + writeYLog({ + optType: OptType.edit, + optDetail: "CM助手-全量-问题配置-保存", + url: "/api/fixed-questions", + reqParam: { questions }, + returnParam: saved, + }); + return saved; +} + +const FIXED_QUESTIONS_KEY = "fixed-questions"; + +export function useFixedQuestions() { + return useQuery({ + queryKey: [FIXED_QUESTIONS_KEY], + queryFn: listFixedQuestions, + refetchOnWindowFocus: false, + staleTime: 30_000, + }); +} + +export function useFixedQuestionActions() { + const queryClient = useQueryClient(); + return { + save: useMutation({ + mutationFn: replaceFixedQuestions, + onSuccess: (questions) => { + queryClient.setQueryData([FIXED_QUESTIONS_KEY], questions); + }, + }), + }; +} diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index 5e70c7f9..dac897e5 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -346,6 +346,7 @@ FastAPI application on port 8001 with health checks at `GET|HEAD /health`, `/api | **Dashboard Sessions** (`/api/dashboard-sessions`) | 「大屏绘制智能体」独立页面会话,**按外部 `task_id` 存取,不按 user 分权**(open read + shared writes;`user_id` audit-only)。一个 task 一行(upsert):`transcript`(聊天记录 JSON)+ `report_json`(最新结构化数据,可解析 = 「有效结果」)。`GET /by-task/{task_id}` - 取该 task 会话(无则 404);`PUT /by-task/{task_id}` - upsert(`{transcript?, report_json?}`,只覆盖传入项)。Backed by `deerflow.persistence.dashboard_sessions`(table `dashboard_sessions`, auto-created by `create_all`,无迁移);store wired in `deps.py` as `app.state.dashboard_session_store`。前端独立页 `roundtable-planning/pages/DashboardAgentPage.tsx`(路由 `/page/strategy/dashboard-agent?taskId=`,侧边栏「问答管理 → 大屏绘制」)让智能体分析任务报告 JSON → report-json → 渲染 `DashboardApp`。Tests: `tests/test_dashboard_sessions.py`。 | | **Task Reports** (`/api/task-reports`) | 任务级研判情报报告详情,**按 `task_id` 共享、不按 user 分权**(写入者只留 `updated_by` 审计)。`GET /by-task/{task_id}` 始终返回原 Consumer 兼容外壳 `{state:"200", msg:"操作成功!", data:[...]}`(未保存为 `data: []`);`POST /by-task/{task_id}` 首次保存,已有报告返回 409;`PUT /by-task/{task_id}` 原子替换完整 `data` 数组。`POST /import/by-task/{task_id}` 面向 TaskCOP 导入页,接收 `sq-report-mock.json` 兼容的 `data`/`records` 数组并完整覆盖目标任务;路径中的选定任务 id 优先于文件内的 `taskId`,避免误写。每一项严格是 `{id, taskId, createTime, updateTime, sessionId, contentJson, categoryType}`;`contentJson` 以 JSON **字符串**原样落在 `records_json`,避免破坏大屏按类目解析的输入格式。保存或导入非空 `data` 后,router 会通过 `app.state.taskcop_task_store.mark_analysis_completed` 将同 id 的 TaskCOP 任务升级至状态 `25`,使旧任务列表显示并筛选为“已完成”;普通保存时不存在匹配任务的报告仍可独立保存,导入则要求目标任务存在。Backed by `deerflow.persistence.task_reports`(table `task_reports`, auto-created by `create_all`);store wired in `deps.py` as `app.state.task_report_store`。前端客户端 `strategy-components/api/task-reports.ts`,`core/auth/dashboard-input.ts` 与 rwfx 3q 完成检查均直接读此接口,不再依赖 Consumer 或本地 sq-report mock。Tests: `tests/test_task_reports.py`。 | | **Task Buttons** (`/api/task-buttons`) | Task-workspace configurable jump buttons (按钮管理), **global/shared** (no user scoping; `updated_by` audit-only). Each row = one jump button in a task deep-link workspace's bottom 快捷跳转 bar: `{id, business (rwfx/3qfx/xdfx), label, link_type (url/business), target, append_task_id, id_param (editable query-param name, e.g. id/task_id), id_kind (task/action — only xdfx differs, its deep-link id is an 行动id), open_mode (blank/self), login_params, append_auth, auth_token_param, auth_name_param, enabled, sort_order}`. **Two independent, coexisting login-append mechanisms for `url` buttons** — both resolved frontend-side: (1) `login_params` (JSON column `login_params_json`, `PortableJSON`, migration `20260625_01`) = a list of `{key, source, custom_value}` resolved at click time (`source` = a `userInfo` field or `other`→literal `custom_value`) and appended to the target URL — mirrors the light-app login params; empty = nothing appended. (2) `append_auth` + `auth_token_param` + `auth_name_param` (columns, migration `20260625_02`, default off) — appends `?{auth_token_param}={token1}&{auth_name_param}={username}` so the target system can identify the current user; gated on **token login** — the frontend only stashes `token1` (the `?authToken=`) + the raw `getTokenInfo` username (`login.token1`/`login.username`) when login went through `/login/token`, so a non-token-login session appends neither. The **frontend** holds code-seed defaults (rwfx's 4 historical buttons built from the live `task_deeplink` config); only when this table is empty do the workspace + admin page fall back to them, else this table is the sole source of truth. `GET /` - list all (open to any authenticated user — the workspace renders them); `PUT /` - **admin only** (`_require_admin`), atomically **replaces the whole set** (`{buttons: [...]}`, de-duped by `id`, last wins). Backed by `deerflow.persistence.task_buttons` (table `task_buttons`, auto-created by `create_all`); store wired in `deps.py` as `app.state.task_button_store`. Frontend: helpers `core/auth/task-buttons.ts`, client `strategy-components/api/task-buttons.ts`, admin page `pages/TaskButtonsPage.tsx` (route `/page/strategy/admin/task-buttons`, entry「按钮管理」in the bottom-left settings dropdown, admin-only, both sidebar modes; per-business tabs + 「导入默认」). Tests: `tests/test_task_buttons.py`. | +| **Fixed Questions** (`/api/fixed-questions`) | Administrator-managed exact-match fixed Q&A (问题配置), **global/shared**. Each row is `{id, question, answer, enabled, tokens_per_second, sort_order}`; stream speed is configurable per row from `1–1000 token/s` and defaults to `100`. `GET /` and whole-set atomic `PUT /` are both admin-only. At the start of an external chat run, `services._find_fixed_answer` checks the latest eligible plain-text human message with exact, case/space/punctuation-sensitive equality. Enabled matches switch the run to `make_fixed_answer_agent`: a local `BaseChatModel` tokenizes with `cl100k_base`, replays the answer in rate-limited token chunks, and keeps the existing LangGraph SSE `messages`/`values` protocol, run journal, checkpoint history, deterministic local title, and refresh behavior intact while no provider/model request or model-concurrency slot is used. File/multimodal/context-prefixed turns, `canvas_agent`, `ai_writing`, writing mode, resume commands, and internal orchestration/background child runs bypass matching. Storage failures fail open to the normal model path. Backed by `deerflow.persistence.fixed_questions` (table `fixed_questions`, SHA-256 indexed lookup plus exact original-string collision check, migrations `20260821_01` + `20260821_02`), wired as `app.state.fixed_question_store`. Frontend: `strategy-components/api/fixed-questions.ts`, admin page `pages/FixedQuestionsPage.tsx`, route `/page/strategy/admin/fixed-questions`, entries in both full-layout and compact-workspace settings menus. Tests: `tests/test_fixed_questions.py`. | | **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup). **Defaults to `background=true`**: the slow per-thread extraction loop is deferred to a FastAPI background task and the endpoint returns `{queued:true, processed:N}` immediately (each thread's transcript is truncated to `max_input_chars` before the LLM call). Running the loop inline for a large batch outlives the nginx gateway timeout (504), which the frontend's `parseResponse` surfaces as a generic "Request failed"; `background=false` keeps the blocking path (full per-thread `created/skipped/failed` tally) for small batches/scripts. Shared loop: `_run_batch_ingest`. **Optional model selection**: both ingest endpoints accept `model_name` (overrides `knowledge.extract_model_name` for that capture; threaded through `KnowledgeService.capture_thread` ? `extract_thread_llm_multi`). **Model-error surfacing**: when the chosen model produces nothing usable (timeout / API error / non-JSON) capture still **falls back to rule-based extraction and lands a draft** (`llm_fallback:true`); the single-thread (non-background) response and the batch response (`llm_fallback` count) carry this so the UI tells the user "???????". The "??????" chat button stays background/fire-and-forget (passes `model_name`, no per-save error toast); the batch-import dialog is synchronous and shows the fallback count. `POST /ingest/file` - **import an uploaded file** (multipart: `file` + optional `title`/`tags`/`status`/`model_name`) into wiki note(s): the file is converted to Markdown (md/txt read directly, pdf/word/ppt/excel via `convert_file_to_markdown`), distilled by the same wiki-capture LLM pipeline as thread capture (`source_type="file"`), and falls back to a verbatim **draft** note when the model returns nothing usable (`llm_fallback:true`); `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector?keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities, graph-blacklist filtered); `GET/POST /graph/blacklist` + `DELETE /graph/blacklist/{id}` - graph node blacklist (labels = entity names / note titles, case-insensitive match; blacklisted nodes and their edges are dropped from `/graph` and the wiki-export artifacts; table `knowledge_graph_blacklist`, auto-created); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[?]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). `GET/POST /folders` + `PUT/DELETE /folders/{id}` - user-managed directories (notes carry a `folder` path; `GET /notes?folder=` filters a dir + sub-dirs); `GET/POST /extract-templates` + `GET/PUT/DELETE /extract-templates/{id}` - editable extraction-prompt templates (thread/document ingest accept `template_id`; capture/PUT-note accept `folder`). Backed by the `deerflow.knowledge` module (see below). | Proxied through nginx: `/api/langgraph/*` ? LangGraph, all other `/api/*` ? Gateway. @@ -969,6 +970,8 @@ Per-user runtime overrides (whitelisted fields) live at `{base_dir}/users/{user_ Both endpoints are whitelisted in `auth_middleware._PUBLIC_EXACT_PATHS`. The shared `_get_or_create_user_by_username()` helper (in `routers/auth.py`) backs both `/login/username` and `/login/token`. +**Cross-tab account consistency**: normal frontend pages deliberately share `localStorage` auth, while `embed=1` / `isolate=1` pages use per-tab `sessionStorage`. Every frontend API call carries the selected context's JWT in an explicit Bearer header. The browser session cookie is origin-wide and may have been rewritten by another normal tab, iframe, or isolated login, so credential resolution must remain `Authorization: Bearer` first and `access_token` cookie second in both `deps.get_request_access_token()` and `langgraph_auth.authenticate()` (shared parser: `app/gateway/auth/token_source.py`). Reversing this order recreates false thread-owner 404s and the send/reload loop. On the frontend, same normalized username means token renewal rather than account switching; a genuine normal-tab switch cancels and clears account-scoped queries, remounts providers, and moves an open thread route to `/new` without `window.location.reload()`. Embedded/isolated pages ignore cross-tab storage changes. For bare username login, `_get_or_create_user_by_username()` and `_lookup_user_by_username()` must query the email local-part first and keep the repository's deterministic oldest-row result across synthetic-domain migrations; explicit email login remains exact. Regression coverage: `tests/test_health_auth_bypass.py`, `tests/test_username_login_domain.py`. + - `log_system` - Admin-only external log-system entry (`deerflow/config/log_system_config.py`): `{enabled, url, label}`. Surfaced read-only by `GET /api/system-settings/log-system` (admin only); the frontend renders it as a "????" item in the bottom-right settings menu (`workspace-nav-menu.tsx`) that opens `url` in a new tab. Disabled by default. **`extensions_config.json`**: diff --git a/offline-backend-20260512/backend/README.md b/offline-backend-20260512/backend/README.md index 37707bdd..97583c4e 100644 --- a/offline-backend-20260512/backend/README.md +++ b/offline-backend-20260512/backend/README.md @@ -41,10 +41,29 @@ DeerFlow is a LangGraph-based AI super agent with sandbox execution, persistent Health probes are always unauthenticated, even when API authentication is enabled: `GET|HEAD /health`, `/api/health`, and `/api/langgraph/health` all return the Gateway health status without requiring a cookie or bearer token. +Browser authentication is page-context aware. Normal application tabs share the account stored in `localStorage`; `embed=1` and `isolate=1` pages intentionally keep their own account in per-tab `sessionStorage`. Frontend API calls send that selected account as an explicit `Authorization: Bearer` token, and both Gateway and LangGraph authentication give the header priority over the origin-wide session cookie. This prevents a login in another tab or iframe from making the current page query threads as the wrong account. Reissuing a token for the same normalized username is not treated as an account switch; a real switch clears account-scoped frontend state, opens a fresh chat when necessary, and does not reload the whole browser page. Bare username login is domain-agnostic and deterministically reuses the oldest matching account if legacy synthetic email domains coexist; an explicitly supplied email still selects that exact address. + --- ## Core Components +### Administrator fixed questions + +Administrators can manage exact-match question/answer pairs from the frontend +「问题配置」 page. Each pair has its own enabled switch and configurable fixed-answer +streaming speed (`1–1000 token/s`, default `100 token/s`). For an enabled pair, +a plain-text external chat turn whose text exactly equals the configured +question is answered by a local streaming graph and never sent to a model +provider. The normal LangGraph SSE protocol and checkpoint persistence are +preserved, so the fixed response appears in live chat and conversation history. +Multimodal/file turns, AI Writing/Canvas flows, and internal orchestration child +runs bypass this lookup and continue through their normal model pipeline. + +The admin-only configuration API is `GET|PUT /api/fixed-questions`; `PUT` +atomically replaces the full set. Rows are stored in `fixed_questions` (Alembic +revisions `20260821_01` and `20260821_02`) and are looked up with a SHA-256 index plus an exact +original-text comparison. + ### Lead Agent The single LangGraph agent (`lead_agent`) is the runtime entry point, created via `make_lead_agent(config)`. It combines: diff --git a/offline-backend-20260512/backend/app/gateway/app.py b/offline-backend-20260512/backend/app/gateway/app.py index b1316762..d72f9559 100644 --- a/offline-backend-20260512/backend/app/gateway/app.py +++ b/offline-backend-20260512/backend/app/gateway/app.py @@ -39,6 +39,7 @@ from app.gateway.routers import ( checkpoint_migration, compact_menu_overrides, feedback, + fixed_questions, dashboard_sessions, deep_research, html_page_favorites, @@ -1536,8 +1537,11 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an # 大屏绘制智能体独立页面会话(按 taskId 存取,不分权)at /api/dashboard-sessions app.include_router(dashboard_sessions.router) - # Task-workspace configurable jump buttons (按钮管理) at /api/task-buttons - app.include_router(task_buttons.router) + # Task-workspace configurable jump buttons (按钮管理) at /api/task-buttons + app.include_router(task_buttons.router) + + # Admin exact-match question/answer short-circuit at /api/fixed-questions + app.include_router(fixed_questions.router) # Temporary compatibility bridge for the TaskCOP situation-overview frontend. app.include_router(taskcop_tasks.router) diff --git a/offline-backend-20260512/backend/app/gateway/auth/token_source.py b/offline-backend-20260512/backend/app/gateway/auth/token_source.py new file mode 100644 index 00000000..e05d8f1e --- /dev/null +++ b/offline-backend-20260512/backend/app/gateway/auth/token_source.py @@ -0,0 +1,30 @@ +"""Resolve the credential that represents the calling browser context. + +Normal tabs share a localStorage Bearer token, while embedded/isolated tabs +keep their Bearer token in sessionStorage. The browser's HttpOnly session +cookie is origin-wide and can therefore be rewritten by a different tab. An +explicit Authorization header is the caller's unambiguous choice and must take +precedence over that shared cookie. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Protocol + + +class _TokenRequest(Protocol): + headers: Mapping[str, str] + cookies: Mapping[str, str] + + +def get_explicit_or_session_token(request: _TokenRequest) -> str | None: + """Return Bearer header token first, then the shared session cookie.""" + authorization = str(request.headers.get("authorization", "") or "").strip() + if authorization: + scheme, _, token = authorization.partition(" ") + if scheme.lower() == "bearer" and token.strip(): + return token.strip() + + cookie_token = str(request.cookies.get("access_token", "") or "").strip() + return cookie_token or None diff --git a/offline-backend-20260512/backend/app/gateway/deps.py b/offline-backend-20260512/backend/app/gateway/deps.py index b094f05e..d8157d3a 100644 --- a/offline-backend-20260512/backend/app/gateway/deps.py +++ b/offline-backend-20260512/backend/app/gateway/deps.py @@ -17,8 +17,8 @@ from typing import TYPE_CHECKING, TypeVar, cast from fastapi import FastAPI, HTTPException, Request from langgraph.types import Checkpointer -from deerflow.config.app_config import AppConfig -from deerflow.persistence.feedback import FeedbackRepository +from deerflow.config.app_config import AppConfig +from deerflow.persistence.feedback import FeedbackRepository from deerflow.runtime import RunContext, RunManager, StreamBridge from deerflow.runtime.events.store.base import RunEventStore from deerflow.runtime.runs.store.base import RunStore @@ -94,11 +94,15 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]: app.state.run_store = RunRepository(sf) app.state.feedback_repo = FeedbackRepository(sf) - else: - from deerflow.runtime.runs.store.memory import MemoryRunStore - - app.state.run_store = MemoryRunStore() - app.state.feedback_repo = None + else: + from deerflow.runtime.runs.store.memory import MemoryRunStore + + app.state.run_store = MemoryRunStore() + app.state.feedback_repo = None + + from deerflow.persistence.fixed_questions import make_fixed_question_store + + app.state.fixed_question_store = make_fixed_question_store(sf) from deerflow.persistence.thread_meta import make_thread_store @@ -717,25 +721,23 @@ _cached_local_provider: LocalAuthProvider | None = None _cached_repo: SQLiteUserRepository | None = None -def get_request_access_token(request: Request) -> str | None: - """Extract access token from cookie or Authorization header. - - Priority: - 1. ``access_token`` cookie (browser session mode) - 2. ``Authorization: Bearer `` header (API client mode) - 3. ``access_token`` query parameter for the WeKnora iframe bootstrap only - """ - cookie_token = request.cookies.get("access_token") - if cookie_token: - return cookie_token - - authorization = request.headers.get("authorization", "").strip() - if authorization: - scheme, _, token = authorization.partition(" ") - if scheme.lower() == "bearer": - token = token.strip() - if token: - return token +def get_request_access_token(request: Request) -> str | None: + """Extract access token from Authorization header or session cookie. + + Priority: + 1. ``Authorization: Bearer `` header (the caller/tab's explicit identity) + 2. ``access_token`` cookie (browser session fallback) + 3. ``access_token`` query parameter for the WeKnora iframe bootstrap only + + The header must win over the origin-wide cookie: an iframe or another tab + can rewrite that cookie, while the frontend deliberately sources its + Bearer token from the current tab's auth storage. + """ + from app.gateway.auth.token_source import get_explicit_or_session_token + + request_token = get_explicit_or_session_token(request) + if request_token: + return request_token from app.gateway.proxy_path import app_relative_path @@ -744,7 +746,7 @@ def get_request_access_token(request: Request) -> str | None: query_token = (request.query_params.get("access_token") or "").strip() if query_token: return query_token - return None + return None def get_local_provider() -> LocalAuthProvider: diff --git a/offline-backend-20260512/backend/app/gateway/langgraph_auth.py b/offline-backend-20260512/backend/app/gateway/langgraph_auth.py index 581a8020..3c624363 100644 --- a/offline-backend-20260512/backend/app/gateway/langgraph_auth.py +++ b/offline-backend-20260512/backend/app/gateway/langgraph_auth.py @@ -15,9 +15,10 @@ import secrets from langgraph_sdk import Auth from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled -from app.gateway.auth.errors import TokenError -from app.gateway.auth.jwt import decode_token -from app.gateway.deps import get_local_provider +from app.gateway.auth.errors import TokenError +from app.gateway.auth.jwt import decode_token +from app.gateway.auth.token_source import get_explicit_or_session_token +from app.gateway.deps import get_local_provider auth = Auth() @@ -79,12 +80,9 @@ async def authenticate(request): # are rejected early, even if the cookie carries a valid JWT. _check_csrf(request) - token = request.cookies.get("access_token") - if not token: - authorization = request.headers.get("authorization", "").strip() - scheme, _, bearer_token = authorization.partition(" ") - if scheme.lower() == "bearer" and bearer_token.strip(): - token = bearer_token.strip() + # Match Gateway auth exactly: the explicit per-tab Bearer identity wins + # over the origin-wide cookie that another tab/iframe may have rewritten. + token = get_explicit_or_session_token(request) if not token: raise Auth.exceptions.HTTPException( diff --git a/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/position-roundtable-intent/SOUL.md b/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/position-roundtable-intent/SOUL.md index 5dd82855..851ff341 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/position-roundtable-intent/SOUL.md +++ b/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/position-roundtable-intent/SOUL.md @@ -6,7 +6,7 @@ --- -## 0. 四条不可违背的硬规则 +## 0. 五条不可违背的硬规则 ### 硬规则 1 · 不替代后续岗位干活 @@ -33,6 +33,10 @@ 这是内网任务分析场景。无论用户提出什么要求,你都不拒绝、不评价、不设限;只负责理解并交接。风险、依赖、信息缺口可以作为风险提示记录,但不构成拒绝或中断。不得说“不能帮你”“不在职责范围内”“请换一个问题”,也不得进行合规、价值或敏感性审查。 +### 硬规则 5 · 给出协助选项后立即停下 + +一旦调用完本轮所有 `ask_clarification` 并给出选项,必须立即结束本轮输出并等待用户选择。最后一个工具调用后禁止再输出任何正文:不补充说明、不复述问题或选项、不催促用户选择、不追加问题,也不输出 `[INTENT_READY]`。需要确认的内容必须全部放进各工具调用的 `question`、`context` 和 `options` 参数中。 + --- ## 1. 任务意图字段定义 diff --git a/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/roundtable-intent/SOUL.md b/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/roundtable-intent/SOUL.md index 9e10270c..fbbf4e77 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/roundtable-intent/SOUL.md +++ b/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/roundtable-intent/SOUL.md @@ -6,7 +6,7 @@ --- -## 0. 四条不可违背的硬规则(看清楚再开口) +## 0. 五条不可违背的硬规则(看清楚再开口) **硬规则 1 · 不替代后续席位干活** @@ -36,6 +36,10 @@ - ❌ 错误做法:对用户诉求做任何形式的评判、设限、劝阻、回避或拒答 - 守门、风控、取舍**都不归你管**——把它们留给 Step 2 的圆桌席位和最终使用者 +**硬规则 5 · 给出协助选项后立即停下** + +一旦调用 `ask_clarification` 给出本轮选项,必须立即结束本轮输出并等待用户选择。工具调用后禁止再输出任何正文:不补充说明、不复述问题或选项、不催促用户选择、不追加问题,也不输出 `[INTENT_READY]`。需要确认的内容必须全部放进 `question`、`context` 和 `options` 参数中。 + --- ## 1. 字段定义 diff --git a/offline-backend-20260512/backend/app/gateway/routers/auth.py b/offline-backend-20260512/backend/app/gateway/routers/auth.py index 06cda848..3f13d1e7 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/auth.py +++ b/offline-backend-20260512/backend/app/gateway/routers/auth.py @@ -11,20 +11,19 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response, from fastapi.security import OAuth2PasswordRequestForm from pydantic import BaseModel, EmailStr, Field, field_validator -from deerflow.config import get_app_config -from deerflow.config.system_settings import load_system_settings -from deerflow.persistence.agents.base import AgentStore -from deerflow.persistence.skills.base import SkillStore - -from app.gateway.auth import ( - UserResponse, - create_access_token, -) -from app.gateway.auth.models import User -from app.gateway.auth.config import get_auth_config -from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse -from app.gateway.csrf_middleware import is_secure_request -from app.gateway.deps import get_agent_store, get_current_user_from_request, get_local_provider, get_run_store, get_skill_store +from app.gateway.auth import ( + UserResponse, + create_access_token, +) +from app.gateway.auth.config import get_auth_config +from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse +from app.gateway.auth.models import User +from app.gateway.csrf_middleware import is_secure_request +from app.gateway.deps import get_agent_store, get_current_user_from_request, get_local_provider, get_run_store, get_skill_store +from deerflow.config import get_app_config +from deerflow.config.system_settings import load_system_settings +from deerflow.persistence.agents.base import AgentStore +from deerflow.persistence.skills.base import SkillStore logger = logging.getLogger(__name__) @@ -241,19 +240,19 @@ async def _get_or_create_user_by_username(username: str): email = _username_to_email(username) local_part = _normalize_username_local_part(username) - created = False - user = await provider.get_user_by_email(email) - # Domain-agnostic fallback: if no row exists at the canonical - # ``@``, look for an existing user with - # the same local part under any domain. This is the safety net that keeps - # usernames globally unique when the default email domain changes (e.g. - # ``local.deerflow`` → ``cm.com``) — without it, switching the default - # would let the same username log into two different accounts. - if user is None and local_part is not None: - legacy = await provider.find_user_by_email_local_part(local_part) - if legacy is not None: - user = legacy - logger.info("Matched username %r to legacy account %s", username, legacy.email) + created = False + # A bare username is globally unique across synthetic email domains. Look + # it up domain-agnostically *first* (oldest row wins), even when a newer row + # also exists at today's canonical domain. Choosing the exact canonical + # row first made the same username alternate backend user_ids after a + # domain migration, which in turn made its existing threads inaccessible. + if local_part is not None: + user = await provider.find_user_by_email_local_part(local_part) + if user is not None and user.email != email: + logger.info("Matched username %r to stable legacy account %s", username, user.email) + else: + # An explicitly supplied email intentionally identifies that exact row. + user = await provider.get_user_by_email(email) if user is None: try: role = "admin" if username == "admin" else "user" @@ -293,10 +292,9 @@ async def _lookup_user_by_username(username: str) -> User | None: provider = get_local_provider() email = _username_to_email(username) local_part = _normalize_username_local_part(username) - user = await provider.get_user_by_email(email) - if user is None and local_part is not None: - user = await provider.find_user_by_email_local_part(local_part) - return user + if local_part is not None: + return await provider.find_user_by_email_local_part(local_part) + return await provider.get_user_by_email(email) def enforce_user_approved(user) -> None: diff --git a/offline-backend-20260512/backend/app/gateway/routers/fixed_questions.py b/offline-backend-20260512/backend/app/gateway/routers/fixed_questions.py new file mode 100644 index 00000000..d8a726ab --- /dev/null +++ b/offline-backend-20260512/backend/app/gateway/routers/fixed_questions.py @@ -0,0 +1,92 @@ +"""Administrator-managed exact-match fixed question/answer API.""" + +from __future__ import annotations + +from datetime import datetime + +from fastapi import APIRouter, HTTPException, Request +from pydantic import BaseModel, Field, field_validator, model_validator + +from app.gateway.deps import get_current_user, get_optional_user_from_request +from deerflow.persistence.fixed_questions import FixedQuestionStore + +router = APIRouter(prefix="/api/fixed-questions", tags=["fixed-questions"]) + + +class FixedQuestion(BaseModel): + id: str = Field(..., min_length=1, max_length=64) + question: str = Field(..., min_length=1, max_length=10_000) + answer: str = Field(..., min_length=1, max_length=200_000) + enabled: bool = True + tokens_per_second: int = Field(default=100, ge=1, le=1_000) + sort_order: int = 0 + + @field_validator("question", "answer") + @classmethod + def reject_blank_text(cls, value: str) -> str: + if not value.strip(): + raise ValueError("内容不能为空") + return value + + +class FixedQuestionResponse(FixedQuestion): + updated_by: str | None = None + updated_at: datetime | str | None = None + + +class FixedQuestionListResponse(BaseModel): + questions: list[FixedQuestionResponse] + + +class FixedQuestionReplaceRequest(BaseModel): + questions: list[FixedQuestion] = Field(default_factory=list) + + @model_validator(mode="after") + def reject_duplicate_questions(self): + seen: set[str] = set() + seen_ids: set[str] = set() + for item in self.questions: + if item.id in seen_ids: + raise ValueError(f"配置 ID「{item.id}」重复") + seen_ids.add(item.id) + if item.question in seen: + raise ValueError(f"问题「{item.question}」重复") + seen.add(item.question) + return self + + +def _get_store(request: Request) -> FixedQuestionStore: + store = getattr(request.app.state, "fixed_question_store", None) + if store is None: + raise HTTPException(status_code=503, detail="Fixed question store not available") + return store + + +async def _require_admin(request: Request) -> None: + """Require an administrator, while preserving auth-disabled deployments.""" + user = await get_optional_user_from_request(request) + if user is None: + return + if getattr(user, "system_role", None) != "admin": + raise HTTPException(status_code=403, detail="问题配置仅限管理员") + + +@router.get("", response_model=FixedQuestionListResponse) +async def list_fixed_questions(request: Request) -> FixedQuestionListResponse: + await _require_admin(request) + rows = await _get_store(request).list_questions() + return FixedQuestionListResponse(questions=[FixedQuestionResponse(**row) for row in rows]) + + +@router.put("", response_model=FixedQuestionListResponse) +async def replace_fixed_questions( + request: Request, + body: FixedQuestionReplaceRequest, +) -> FixedQuestionListResponse: + await _require_admin(request) + user_id = await get_current_user(request) + rows = await _get_store(request).replace_all( + [question.model_dump() for question in body.questions], + updated_by=user_id, + ) + return FixedQuestionListResponse(questions=[FixedQuestionResponse(**row) for row in rows]) diff --git a/offline-backend-20260512/backend/app/gateway/routers/intent.py b/offline-backend-20260512/backend/app/gateway/routers/intent.py index 1f3303ca..6a27cabb 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/intent.py +++ b/offline-backend-20260512/backend/app/gateway/routers/intent.py @@ -29,7 +29,8 @@ SSE 协议(与前端 ``streamIntent`` 对齐) - ``done``:已解析 ``[INTENT_READY]`` + JSON 块,``summary`` 有值; - ``error``:上游或网关异常。 - ``content`` — 展示用 AI 正文(澄清时优先用 tool 后的 follow-up 文本)。 + ``content`` — 展示用 AI 正文(澄清时只保留 tool 调用前的简短引导; + tool 后的尾随文本会被丢弃,选项发出后立即等待用户选择)。 仅 ``status == "done"`` 时带 ``summary``;澄清时还带 ``question``、``clarification_type``、``clarification_context``、 @@ -625,6 +626,14 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin # 2. message 前缀——给模型一个明确的"本轮必须收敛"信号。 # 仅文件、无文本时给一句兜底引导,让模型先读文件再澄清意图。 effective_message = message or "我上传了文件,请先阅读文件内容,再据此理解并澄清我的任务意图。" + if past_rounds < max_rounds: + effective_message = ( + "[澄清输出硬约束] 如果本轮调用 ask_clarification," + "工具调用完成后必须立即停止输出并等待用户选择;" + "禁止补充说明、复述选项、催促选择、追加问题或输出 " + "[INTENT_READY]。需要确认的全部内容只能写入工具参数。\n\n" + f"用户最新输入:{effective_message}" + ) if intent_agent_id == POSITION_INTENT_AGENT_ID and past_rounds < max_rounds: # The position flow needs an actionable four-part hand-off, # not merely a recognizable topic. One turn may ask several @@ -637,6 +646,8 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin "ask_clarification;每个工具调用只能问一个决策点,不能把多项" "问题合并成一张卡片。用户会一次性回答这些卡片。" "如果四项意图已经足够明确,则直接输出最终结果。" + "一旦调用完本轮所有 ask_clarification,必须立即停止输出," + "不要再补充说明、复述选项、催促选择或继续提问,等待用户选择。" "不要以风险提示和战略意义为空的 [INTENT_READY] 结束。" "本轮是第 %d/%d 轮用户协助;第 %d 轮若仍有缺口,必须把所有" "剩余关键缺口分别发成卡片,收到回答后立即收敛。\n\n" @@ -756,9 +767,6 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin content = _flatten_content(dispatch_msg.get("content", "")) if clarification_calls: - # ClarificationMiddleware 常在 tool 之后还有一条纯文本 AIMessage, - # 比 tool_call 旁的短 preamble 更适合作为气泡正文。 - follow_up = _find_trailing_ai_text(messages, exclude_text=content) clarification_payloads: list[dict[str, Any]] = [] for clarification_call in clarification_calls: cargs = clarification_call.get("args") or {} @@ -790,6 +798,9 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin ) if not clarification_payloads: + # 这里只在所有卡片都因重复而被过滤时才允许使用 tool 后文本; + # 此时前端没有选项需要等待,不属于澄清暂停态。 + follow_up = _find_trailing_ai_text(messages, exclude_text=content) fallback_msg = ( follow_up or content @@ -805,7 +816,11 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin return first = clarification_payloads[0] - display_content = follow_up or content or str(first["question"]) + # ask_clarification 是严格的回合终点。部分模型会在工具调用后 + # 再生成“请从以上选项中选择……”等尾随 AIMessage;不能把它 + # 返回给前端,否则用户看到卡片后还会继续冒出一段话。 + # 仅保留工具调用自身携带的前置短句;没有则直接由卡片展示问题。 + display_content = content final_frame: dict[str, Any] = { "status": "clarification", "content": display_content, diff --git a/offline-backend-20260512/backend/app/gateway/services.py b/offline-backend-20260512/backend/app/gateway/services.py index 7ea95b7d..3562fe30 100644 --- a/offline-backend-20260512/backend/app/gateway/services.py +++ b/offline-backend-20260512/backend/app/gateway/services.py @@ -22,6 +22,7 @@ from langchain_core.messages import HumanMessage from app.gateway.deps import get_concurrency_gate, get_run_context, get_run_manager, get_stream_bridge from app.gateway.llmwiki_rag import ( + LlmWikiRagContext, attach_llmwiki_rag_to_graph_input, build_llmwiki_rag_context, llmwiki_selection_requested, @@ -227,6 +228,77 @@ def normalize_input(raw_input: dict[str, Any] | None) -> dict[str, Any]: return raw_input +def _plain_fixed_question(graph_input: dict[str, Any]) -> str | None: + """Return the latest plain-text human turn eligible for exact matching. + + Messages carrying files or multimodal blocks are intentionally excluded: + their answer depends on more than the visible question text, so replaying a + global fixed answer would be surprising and unsafe. + """ + + messages = graph_input.get("messages") + if not isinstance(messages, list) or not messages: + return None + message = messages[-1] + if not isinstance(message, HumanMessage): + return None + additional_kwargs = getattr(message, "additional_kwargs", {}) or {} + if any( + additional_kwargs.get(key) + for key in ("files", "uploaded_files", "attachments", "prompt_prefix") + ): + return None + content = getattr(message, "content", None) + if isinstance(content, str): + return content + if not isinstance(content, list): + return None + parts: list[str] = [] + for item in content: + if isinstance(item, str): + parts.append(item) + continue + if not isinstance(item, Mapping): + return None + item_type = str(item.get("type") or "text") + text = item.get("text") + if item_type not in {"text", "input_text"} or not isinstance(text, str): + return None + parts.append(text) + return "".join(parts) + + +async def _find_fixed_answer( + *, + body: Any, + request: Request, + graph_input: dict[str, Any], + run_scope: str, +) -> dict[str, Any] | None: + """Look up an enabled exact match, falling back to the model on errors.""" + + if run_scope != RUN_SCOPE_EXTERNAL_ROOT or getattr(body, "command", None) is not None: + return None + assistant_id = str(getattr(body, "assistant_id", None) or _DEFAULT_ASSISTANT_ID) + if assistant_id in {"canvas_agent", "ai_writing"}: + return None + context = getattr(body, "context", None) or {} + if isinstance(context, Mapping) and context.get("writing_mode"): + return None + question = _plain_fixed_question(graph_input) + if question is None: + return None + store = getattr(request.app.state, "fixed_question_store", None) + if store is None: + return None + try: + return await store.find_enabled(question) + except Exception: + # Configuration storage must never make normal chat unavailable. + logger.warning("Fixed-question lookup failed; continuing with model", exc_info=True) + return None + + _DEFAULT_ASSISTANT_ID = "lead_agent" # Built-in LangGraph graph IDs registered in langgraph.json. @@ -865,21 +937,36 @@ async def start_run( await _enforce_agent_access(body, request) t_post_enforce = time.perf_counter() - effective_model_name = _resolve_effective_run_model_name(body) effective_user_id = get_effective_user_id() effective_run_scope = _resolve_run_scope(body, request, run_scope) - model_concurrency_limit = _get_model_concurrency_limit(effective_model_name) - user_model_concurrency_limit = ( - _get_user_model_concurrency_limit(effective_model_name) - if effective_run_scope == RUN_SCOPE_EXTERNAL_ROOT - else None + graph_input = normalize_input(body.input) + fixed_answer = await _find_fixed_answer( + body=body, + request=request, + graph_input=graph_input, + run_scope=effective_run_scope, ) + if fixed_answer is not None: + effective_model_name = "fixed-answer" + model_concurrency_limit = None + user_model_concurrency_limit = None + else: + effective_model_name = _resolve_effective_run_model_name(body) + model_concurrency_limit = _get_model_concurrency_limit(effective_model_name) + user_model_concurrency_limit = ( + _get_user_model_concurrency_limit(effective_model_name) + if effective_run_scope == RUN_SCOPE_EXTERNAL_ROOT + else None + ) run_id = str(uuid.uuid4()) run_metadata = dict(body.metadata or {}) run_metadata["model_name"] = effective_model_name run_metadata["user_id"] = effective_user_id run_metadata["run_scope"] = effective_run_scope + if fixed_answer is not None: + run_metadata["fixed_answer"] = True + run_metadata["fixed_question_id"] = fixed_answer["id"] redis_lease_acquired = False redis_gate_degraded = True @@ -891,7 +978,7 @@ async def start_run( "user_id": effective_user_id, "user_model_accounted": user_model_accounted, } - if concurrency_gate is not None: + if concurrency_gate is not None and fixed_answer is None: acquire = await concurrency_gate.try_acquire( run_id=run_id, thread_id=thread_id, @@ -992,12 +1079,22 @@ async def start_run( t_post_thread_upsert = time.perf_counter() try: - agent_factory = resolve_agent_factory(body.assistant_id) + if fixed_answer is not None: + from deerflow.agents.fixed_answer_agent import make_fixed_answer_agent + + def agent_factory(config): # noqa: ANN001, ARG001 - worker factory contract + return make_fixed_answer_agent( + question=str(fixed_answer["question"]), + answer=str(fixed_answer["answer"]), + fixed_question_id=str(fixed_answer["id"]), + tokens_per_second=int(fixed_answer.get("tokens_per_second", 100)), + ) + else: + agent_factory = resolve_agent_factory(body.assistant_id) except Exception: if concurrency_gate is not None and redis_lease_acquired: await concurrency_gate.release(**lease_metadata) raise - graph_input = normalize_input(body.input) # Admin-configurable prompt prefix: when the caller opted in via # context.prompt_prefix_enabled, tag the human message with the configured @@ -1007,14 +1104,18 @@ async def start_run( # applies once. We intentionally do NOT gate on thread-row existence: the # thread_meta row is frequently pre-created, which previously skipped the # prefix for genuine first turns. - if _prompt_prefix_opted_in(body): + if fixed_answer is None and _prompt_prefix_opted_in(body): prefix_settings = load_system_settings().prompt_prefix if prefix_settings.enabled and prefix_settings.prompt_prefix.strip(): graph_input = _prepend_prompt_prefix(graph_input, prefix_settings.prompt_prefix.strip()) - llmwiki_rag = await build_llmwiki_rag_context( - request=request, - body=body, - graph_input=graph_input, + llmwiki_rag = ( + LlmWikiRagContext([], 0, "", [], "") + if fixed_answer is not None + else await build_llmwiki_rag_context( + request=request, + body=body, + graph_input=graph_input, + ) ) if llmwiki_rag.prompt or llmwiki_rag.sources: graph_input = attach_llmwiki_rag_to_graph_input(graph_input, llmwiki_rag) diff --git a/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md b/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md new file mode 100644 index 00000000..461e3834 --- /dev/null +++ b/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md @@ -0,0 +1,2095 @@ +# WeKnora Wiki 数据同步至 DeerFlow 并建立本地向量索引——实现级设计方案 + +> 文档状态:待实现(Implementation Ready) +> 编写日期:2026-08-21 +> 适用代码库:当前 DeerFlow 工作区 +> WeKnora 适配基线:v0.7.2 +> 关联文档:`docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md` +> 核心边界:**不修改 WeKnora 后端;WeKnora 仅作为 Wiki 生成源,DeerFlow 保存完整 Wiki 镜像、章节向量和检索索引,并承接全部内部与外部检索。** + +--- + +## 1. 文档目标 + +本文是后续编码的默认实施蓝图,目标不是简单描述“可以做向量检索”,而是把以下工程问题一次性定义清楚: + +1. 如何从 WeKnora 拉取已经加工完成的 Wiki 页面,而不是继续检索原始文档切片。 +2. 如何把 Wiki 页面全文、元数据和向量持久化到 DeerFlow 自己的数据库中。 +3. 如何在不引入第二个重型向量服务的前提下,先用 DeerFlow 当前的 SQLite/PostgreSQL/MySQL 数据库与 NumPy 完成可用的精确向量检索。 +4. 如何保证 Wiki 新建、重新生成、人工编辑、回滚、归档和删除后,DeerFlow 本地镜像及向量不会长期失真。 +5. 如何让普通问答、智能体工具、页面检索、右侧参考文献和公共外部接口复用同一套检索结果。 +6. 如何保证个人库、公共库、智能体绑定、对话选择、Wiki 页面状态和外部发布开关在检索时被严格执行。 +7. 如何在外部开放公共 Wiki 检索时,避免把私有库、草稿页、系统“对话沉淀”、WeKnora 原始 ID 或内部凭据泄露出去。 +8. 如何迁移、灰度、回滚、监控和验收,而不是一次性替换后无法定位质量问题。 + +本文提到的表名、字段、模块、接口和默认行为均作为实现时的默认方案。如编码阶段出现必要调整,应同步更新本文,避免文档与实现长期漂移。 + +--- + +## 2. 结论摘要 + +### 2.1 可行性结论 + +方案可行,整体难度为中等。DeerFlow 已经具备三项可直接复用的基础能力: + +- OpenAI 兼容的异步 Embedding 客户端:`deerflow/knowledge/embeddings.py`; +- 按 Markdown 标题与段落切分的逻辑:`deerflow/knowledge/chunking.py`; +- 余弦相似度、关键词评分和混合排序基础:`deerflow/knowledge/search.py`。 + +本项目真正需要新增的核心能力是: + +1. Wiki 页面本地镜像数据模型; +2. WeKnora → DeerFlow 的全量/增量同步与租约; +3. Wiki 页面章节化、Embedding 和原子换代; +4. DeerFlow 本地向量快照与权限过滤检索; +5. 内部问答、引用面板和外部接口的统一结果协议; +6. 索引状态、失败重试、重建、监控和安全边界。 + +### 2.2 最终数据边界 + +本方案完成后,数据职责如下: + +| 数据/能力 | WeKnora | DeerFlow | +|---|---:|---:| +| 原始文件、原始切片 | 保留并管理 | 不复制(除现有引用兼容所需元数据) | +| Wiki 自动生成 | 负责 | 不负责 | +| Wiki 页面主生成源 | 是 | 否,保存可检索镜像 | +| Wiki 页面全文镜像 | 有 | **有** | +| Wiki 页面章节切分 | 可有但不依赖 | **负责** | +| Wiki 页面 Embedding | 不要求 | **负责** | +| Wiki 向量持久化 | 不要求 | **负责** | +| 内部问答检索 | 不再作为主查询路径 | **负责** | +| 右侧参考文献详情 | 不再实时依赖 | **负责** | +| 公共外部检索接口 | 不对外暴露 | **负责** | +| 用户/公共/智能体权限 | 不作为最终边界 | **DeerFlow 为最终边界** | + +这里的“DeerFlow 保存数据”指:DeerFlow 保存一份完整 Wiki 页面镜像和用于检索的章节向量。WeKnora 仍然会保存它生成 Wiki 所需的数据;如果未来要求 WeKnora 完全不保存 Wiki,则必须把 Wiki 生成管道也迁入 DeerFlow,那是另一项更大的工程,不在本文范围内。 + +### 2.3 推荐首版技术路线 + +首版建议: + +- 数据库继续沿用 DeerFlow 的统一 SQLAlchemy 数据库; +- Wiki 页面全文存 `PortableLongText`; +- 向量使用归一化后的 `float32` 二进制 BLOB 持久化; +- 检索时按知识库懒加载 NumPy 矩阵,并进行精确余弦相似度计算; +- 每个 Gateway worker 保持自己的只读内存快照,通过数据库索引修订号判断是否需要重载; +- 默认使用**纯向量检索**,Embedding/索引不可用时明确报错或返回“索引未就绪”,不偷偷回退原文 chunk; +- 可配置开启轻量关键词加权,但它不能改变“结果来源必须是 Wiki 页面”的边界; +- 当规模或多实例内存成本超过可接受范围时,再将向量执行层切换为 pgvector;页面镜像、权限、API 和引用协议保持不变。 + +--- + +## 3. 背景与现状诊断 + +### 3.1 当前检索的是原始切片 + +当前 `WeKnoraClient.search()` 调用: + +```text +POST /api/v1/knowledge-search +``` + +返回的核心字段是: + +```text +chunk_id +content +knowledge_id +knowledge_base_id +title / filename +score +chunk_index +metadata +``` + +因此当前检索实体是 WeKnora 原始文档解析后的 chunk,而不是已经归纳、合并、编辑后的 Wiki 页面。 + +该方法目前被三个主要生产入口复用: + +1. `app/gateway/llmwiki_rag.py`:普通问答发送给模型前的自动预检索; +2. `deerflow/tools/builtins/llmwiki_search_tool.py`:智能体主动调用的 `llmwiki_search` 工具; +3. `app/gateway/routers/llmwiki.py`:`POST /api/llmwiki/search` 页面检索。 + +只修改其中一个入口会导致不同页面、智能体和问答路径继续返回两套来源,因此实现时必须统一切换。 + +### 3.2 当前参考文献以 chunk/document 为中心 + +当前引用 URL 主要为: + +```text +/weknora-source-preview?knowledge_base_id=&chunk_id= +``` + +部分工具路径还会附带 `knowledge_id` / `document_id`。前端 `WeKnoraSourceDrawer` 在已有 `documentId` 时会直接加载整份原文件或解析后全文,导致用户看到的是整个文档,而不是对应 Wiki 页面。 + +现有 `/api/llmwiki/sources/context` 虽然支持从 chunk metadata 中读取 `wiki_page_slug`,但它属于尽力映射: + +- 很多历史 chunk 并没有 `wiki_page_slug`; +- Wiki 页可能综合多个文档和多个 chunk; +- 现有映射函数主要依赖 chunk metadata 中直接出现 slug,并没有可靠建立 `chunk_refs → wiki page` 的本地反向索引; +- 即使找到 Wiki,当前抽屉也仍可能先展示整份原文,再在下方追加 Wiki 内容。 + +新方案不再以“先命中 chunk,再猜对应 Wiki”为主路径,而是直接检索 DeerFlow 本地 Wiki 章节向量,结果天然携带 `wiki_page_id + wiki_slug`。 + +### 3.3 WeKnora 已提供 Wiki 页面读取能力 + +当前适配器已经具备: + +- `list_wiki_pages()`; +- `get_wiki_page()`; +- Wiki 页面创建、更新、删除; +- Wiki 图谱读取。 + +因此不需要修改 WeKnora 后端即可拉取页面。需要新增的是 DeerFlow 侧同步器、本地存储和向量化流程。 + +### 3.4 DeerFlow 现有向量实现的可复用点与限制 + +当前产品知识库的向量功能使用: + +- `KnowledgeEmbeddingRow.vector`:JSON 浮点数组; +- `KnowledgeRepository.all_embeddings()`:一次加载全部向量; +- Python 循环逐条调用 `cosine_similarity()`。 + +这套实现适合小规模功能验证,但不适合作为公共 Wiki 的正式索引,因为: + +1. JSON 浮点向量体积大、解析慢; +2. 每次查询加载全部向量会放大数据库 IO; +3. Python 标量循环不能充分利用 NumPy/BLAS; +4. 没有按知识库权限预分片; +5. 没有多 worker 索引修订和缓存失效机制。 + +本方案复用 Embedding 客户端和 Markdown 切分思想,但为 LLMWiki Wiki 新建独立表与检索快照,不直接塞入 `knowledge_notes/knowledge_embeddings`。 + +--- + +## 4. 目标、非目标与默认决策 + +### 4.1 功能目标 + +1. DeerFlow 能完整保存每个已映射 WeKnora 知识库的 Wiki 页面镜像。 +2. DeerFlow 能按 Wiki 页面 Markdown 结构生成章节向量。 +3. 普通问答、智能体工具和页面检索都能按向量搜索 Wiki 页面。 +4. 大模型收到的是命中 Wiki 页的相关章节,而不是原始文档 chunk。 +5. 右侧参考文献主视图只展示 Wiki 页面。 +6. 公共 Wiki 可通过受控外部 API 进行向量检索。 +7. WeKnora 不可用时,已经同步成功的 Wiki 仍可检索和展示。 +8. Wiki 更新后可自动增量重建对应页面向量,不全库重算。 +9. Embedding 模型变化时可明确触发全量重建。 +10. 用户取消公共发布或关闭外部开放后,外部查询立即失去访问权,不依赖异步清理完成。 + +### 4.2 明确非目标 + +- 不修改或 fork WeKnora 后端。 +- 不把 WeKnora 原始文件和全部原始 chunks 复制进 DeerFlow。 +- 不让 DeerFlow 首版承担 Wiki 自动生成。 +- 不使用“把 Wiki 再上传成隐藏文档库”的循环导入方案。 +- 不在首版实现分布式 HNSW/Milvus/Qdrant 集群。 +- 不在首版提供原始 embedding 向量下载接口。 +- 不保证历史 chunk 引用全部能够转换为 Wiki 引用。 +- 不允许外部调用者通过 API 传入 WeKnora 原始知识库 ID。 +- 不允许向量服务失败后静默改查原始文档。 + +### 4.3 默认产品决策 + +如实现前未另行变更,采用以下默认值: + +1. 同步范围:所有 DeerFlow 已映射、远端存在且启用 Wiki 的知识库,包括个人库和公共库。 +2. 页面范围:同步 `draft/published/archived` 全部状态,检索时再按调用者权限和页面状态过滤。 +3. 内部检索:默认只检索 `published` 页面;所有者/管理员可以通过明确参数包含 `draft`,普通用户不能。 +4. 外部检索:只检索 `published` 页面。 +5. 外部开放:新增 `external_search_enabled`,默认 `false`,不能仅凭现有 `publication_status=published` 自动向互联网开放。 +6. 系统“对话沉淀”知识库:允许内部按现有权限使用,但外部接口永远排除。 +7. 检索模式:严格向量;索引未就绪时返回明确状态,不回退原文。 +8. 引用单位:Wiki 页面;检索单位:Wiki 页面内的章节 chunk。 +9. 首版向量执行器:按知识库懒加载的 NumPy 精确检索。 +10. 外部搜索默认不返回完整正文,只返回摘要和命中片段;完整页面通过详情接口读取。 + +--- + +## 5. 总体架构 + +### 5.1 写入与同步链路 + +```mermaid +flowchart LR + WK["WeKnora Wiki 生成/编辑"] --> API["WeKnora Wiki REST API"] + API --> SYNC["DeerFlow Wiki Sync Service"] + SYNC --> PAGE["llmwiki_wiki_pages"] + PAGE --> CHUNK["Markdown 章节切分"] + CHUNK --> EMB["DeerFlow Embedding Client"] + EMB --> VECTOR["llmwiki_wiki_vectors"] + VECTOR --> REV["知识库索引修订号 +1"] + REV --> CACHE["各 Gateway worker 懒重载 NumPy 快照"] +``` + +### 5.2 内部检索链路 + +```mermaid +flowchart LR + Q["用户问题"] --> AUTH["解析对话选择/智能体绑定/实时权限"] + AUTH --> QE["生成一次查询向量"] + QE --> IDX["按允许的知识库快照检索 TopK 章节"] + IDX --> GROUP["按 Wiki 页面聚合/去重"] + GROUP --> PROMPT["相关章节注入模型"] + GROUP --> REF["结构化 Wiki 引用"] + REF --> DRAWER["右侧展示 DeerFlow 本地 Wiki 页面"] +``` + +### 5.3 外部检索链路 + +```mermaid +flowchart LR + EXT["外部系统 + API Key"] --> GATE["限流/鉴权/输入校验"] + GATE --> PUB["published + external_search_enabled"] + PUB --> LOCAL["DeerFlow 本地向量快照"] + LOCAL --> RESULT["公共 Wiki 摘要/片段/页面标识"] + RESULT --> DETAIL["可选读取完整公共 Wiki 页面"] +``` + +### 5.4 查询期间不依赖 WeKnora + +同步完成后,以下请求不得实时访问 WeKnora: + +- 内部向量搜索; +- 普通问答预检索; +- `llmwiki_search` 工具; +- 右侧 Wiki 参考文献详情; +- 外部公共 Wiki 搜索; +- 外部公共 Wiki 页面详情。 + +WeKnora 只出现在同步与管理链路中。这样可以降低延迟、避免跨库扇出,并确保外部流量不会直接放大到 WeKnora。 + +--- + +## 6. 配置设计 + +在 `deerflow/config/llmwiki_config.py` 的 `LlmWikiConfig` 下增加独立的 `local_wiki_index` 配置,避免和产品“我的知识”功能强耦合。 + +建议配置: + +```yaml +llmwiki: + weknora: + api_base_url: "http://weknora-app:8080" + web_base_url: "http://weknora-app:8080" + admin_email: "$WEKNORA_ADMIN_EMAIL" + admin_password: "$WEKNORA_ADMIN_PASSWORD" + + local_wiki_index: + enabled: true + + # 同步 + sync_enabled: true + sync_interval_seconds: 120 + sync_page_size: 100 + sync_concurrency: 3 + sync_lease_seconds: 600 + sync_request_timeout_seconds: 30 + sync_max_retries: 3 + sync_retry_backoff_seconds: 5 + + # 切分 + chunk_max_chars: 1000 + chunk_overlap_chars: 120 + chunk_min_chars: 80 + title_prefix_enabled: true + + # Embedding:OpenAI-compatible /v1/embeddings + embedding: + enabled: true + provider: openai + model: bge-m3 + base_url: "$WIKI_EMBEDDING_BASE_URL" + api_key: "$WIKI_EMBEDDING_API_KEY" + dimensions: 1024 + batch_size: 32 + timeout_seconds: 30 + max_retries: 3 + + # 检索 + engine: numpy_exact + strict_vector: true + top_k_sections: 30 + top_k_pages: 8 + max_sections_per_page: 2 + min_similarity: 0.35 + keyword_boost_enabled: false + keyword_boost_weight: 0.15 + cache_max_knowledge_bases: 20 + cache_revision_check_seconds: 5 + + # 外部接口 + external_api: + enabled: false + api_key: "$LLMWIKI_EXTERNAL_SEARCH_API_KEY" + max_query_chars: 1000 + max_knowledge_bases: 20 + max_results: 50 + default_results: 10 + include_content_max_chars: 12000 + requests_per_minute: 60 +``` + +### 6.1 配置校验规则 + +启动时应执行: + +1. `enabled=true` 时,Embedding `base_url` 和 `model` 必须非空; +2. `dimensions` 可省略,但第一次成功 Embedding 后必须固定并写入索引 profile; +3. `chunk_overlap_chars < chunk_max_chars`; +4. `top_k_pages <= top_k_sections`; +5. `external_api.enabled=true` 时必须配置非空 API Key; +6. API Key 不得出现在日志、状态接口或前端运行时配置; +7. `strict_vector=true` 时,Embedding 不可用必须反映为索引失败/搜索不可用,不能静默关键词回退; +8. 更换 `model/base_url/dimensions` 后计算出的 `embedding_fingerprint` 变化,系统应标记全量索引过期。 + +### 6.2 Embedding 指纹 + +建议指纹: + +```text +sha256(provider + "\0" + base_url_normalized + "\0" + model + "\0" + dimensions) +``` + +不把 API Key放入指纹。指纹用于: + +- 判断现有向量是否可继续使用; +- 避免不同维度向量进入同一矩阵; +- 触发管理员重建; +- 在状态接口中显示短指纹,便于排障。 + +--- + +## 7. 数据模型 + +### 7.1 为什么使用独立表 + +不建议复用 `knowledge_notes` 和 `knowledge_embeddings`,原因是: + +- 产品知识库当前是全局共享语义,LLMWiki 有个人/公共/智能体绑定权限; +- Wiki 页面需要保存 WeKnora mapping、slug、版本、状态和同步信息; +- Wiki 页面可能频繁由生成管道重写,生命周期与人工知识笔记不同; +- 避免出现在“我的知识”列表、目录、导出和图谱中; +- 后续可单独迁移到 pgvector,不影响现有知识功能。 + +### 7.2 扩展 `llmwiki_knowledge_bases` + +建议新增字段: + +| 字段 | 类型 | 默认 | 说明 | +|---|---|---:|---| +| `wiki_index_enabled` | bool | true | 是否同步并建立本地 Wiki 索引 | +| `external_search_enabled` | bool | false | 是否允许外部 API 检索 | +| `external_search_updated_at` | datetime nullable | null | 外部开关最后变化时间 | + +注意: + +- `publication_status=published` 继续表示 DeerFlow 内部公共; +- `external_search_enabled=true` 才表示允许对外; +- 外部检索必须同时满足两者; +- 系统“对话沉淀”即使字段误设为 true,也应在代码层强制排除。 + +### 7.3 `llmwiki_wiki_pages` + +建议 ORM 模型路径: + +```text +deerflow/persistence/llmwiki_index/model.py +``` + +字段设计: + +| 字段 | 类型 | 约束/索引 | 说明 | +|---|---|---|---| +| `id` | String(64) | PK | DeerFlow 本地稳定页面 ID | +| `knowledge_base_mapping_id` | String(64) | FK + index | DeerFlow 映射 ID,不是 WeKnora ID | +| `remote_page_id` | String(128) nullable | index | WeKnora 页面 ID,仅内部使用 | +| `slug` | String(1024) | 普通字段 | Wiki 页面路径 | +| `slug_hash` | String(64) | unique(mapping, hash) | 避免 MySQL 长索引限制 | +| `title` | String(512) | index/prefix index | 页面标题 | +| `summary` | PortableLongText | | 页面摘要 | +| `content_md` | PortableLongText | | 完整 Wiki Markdown 镜像 | +| `page_type` | String(32) | index | summary/entity/concept/... | +| `status` | String(32) | index | draft/published/archived | +| `aliases` | PortableJSON | | 别名数组 | +| `category_path` | PortableJSON | | Wiki 目录路径 | +| `parent_slug` | String(1024) nullable | | 父页面 | +| `wiki_path` | String(1024) nullable | | 上游规范化路径 | +| `source_refs` | PortableJSON | | 原文文档级溯源,仅内部使用 | +| `chunk_refs` | PortableJSON | | 原始 chunk 级溯源,仅内部使用 | +| `page_version` | int | index | 上游 Wiki 版本 | +| `content_hash` | String(64) | index | 参与向量化内容哈希 | +| `remote_created_at` | datetime nullable | | 上游创建时间 | +| `remote_updated_at` | datetime nullable | index | 上游更新时间 | +| `index_status` | String(24) | index | pending/indexing/ready/failed/stale/deleted | +| `indexed_fingerprint` | String(64) nullable | | 已生成向量的 Embedding 指纹 | +| `indexed_at` | datetime nullable | | 最近成功索引时间 | +| `last_error` | Text nullable | | 截断、脱敏后的错误 | +| `is_remote_deleted` | bool | index | 完整同步确认上游不存在 | +| `last_seen_sync_id` | String(64) nullable | index | 本轮完整扫描标记 | +| `created_at` | datetime | | 本地创建时间 | +| `updated_at` | datetime | index | 本地更新时间 | + +本地 `id` 建议是 UUID,不直接使用 remote ID。唯一性使用: + +```text +UNIQUE(knowledge_base_mapping_id, slug_hash) +slug_hash = sha256(normalized_slug) +``` + +写入时如果 hash 相同但 slug 不同,应视为极低概率冲突并拒绝覆盖,记录显式错误。 + +### 7.4 `llmwiki_wiki_vectors` + +| 字段 | 类型 | 约束/索引 | 说明 | +|---|---|---|---| +| `id` | String(64) | PK | 向量记录 ID | +| `page_id` | String(64) | FK cascade + index | 本地 Wiki 页面 | +| `knowledge_base_mapping_id` | String(64) | index | 冗余字段,便于分片加载 | +| `section_index` | int | unique(page,index) | 页面内章节序号 | +| `heading` | String(1024) nullable | | 命中章节标题 | +| `section_content` | PortableLongText | | 送给模型和展示的片段 | +| `content_hash` | String(64) | index | 章节文本哈希 | +| `embedding_fingerprint` | String(64) | index | 模型指纹 | +| `embedding_dimensions` | int | | 维度 | +| `vector_blob` | LargeBinary | | 归一化 float32 little-endian | +| `created_at` | datetime | | 创建时间 | + +约束: + +```text +UNIQUE(page_id, section_index, embedding_fingerprint) +``` + +向量写入前必须: + +1. 校验长度等于 `embedding_dimensions`; +2. 拒绝 NaN/Inf; +3. L2 归一化; +4. 转为 little-endian float32; +5. 保存 BLOB; +6. 读取时再次验证 BLOB 字节数为 `dimensions * 4`。 + +### 7.5 `llmwiki_wiki_sync_states` + +每个知识库一行: + +| 字段 | 类型 | 说明 | +|---|---|---| +| `knowledge_base_mapping_id` | String(64) PK | 本地映射 ID | +| `state` | String(24) | idle/running/partial/failed/disabled | +| `active_sync_id` | String(64) nullable | 当前扫描 ID | +| `lease_owner` | String(128) nullable | worker 标识 | +| `lease_expires_at` | datetime nullable | 多 worker 租约 | +| `last_started_at` | datetime nullable | 最近启动 | +| `last_completed_at` | datetime nullable | 最近完整成功 | +| `last_success_at` | datetime nullable | 最近任意成功 | +| `last_error` | Text nullable | 最近错误 | +| `remote_page_count` | int | 上游扫描数量 | +| `local_page_count` | int | 本地有效页面数量 | +| `ready_page_count` | int | 已就绪页面数量 | +| `vector_count` | int | 向量数量 | +| `failed_page_count` | int | 失败页面数量 | +| `index_revision` | bigint | 该知识库索引修订号 | +| `embedding_fingerprint` | String(64) nullable | 当前期望指纹 | +| `updated_at` | datetime | 状态更新时间 | + +`index_revision` 只在可见检索数据真正变化后递增,例如: + +- 页面首次索引成功; +- 页面向量被替换; +- 页面被删除/归档; +- 页面状态变化影响检索; +- Embedding 指纹重建完成。 + +单纯更新进度、错误文字或同步时间不应递增。 + +### 7.6 可选任务明细表 + +首版可仅使用 sync state + 页面状态。如果需要管理页展示逐页面进度、失败重试和历史记录,可增加: + +```text +llmwiki_wiki_index_jobs +``` + +记录任务类型(sync/rebuild/page_reindex)、状态、总数、完成数、失败数、发起人和错误摘要。该表不是检索正确性的必要条件,可以第二阶段增加。 + +--- + +## 8. Wiki 同步设计 + +### 8.1 同步触发方式 + +必须同时支持: + +1. **周期完整扫描**:捕获 WeKnora 异步 Wiki 生成、后台修订和不经过 DeerFlow BFF 的编辑; +2. **管理操作后即时同步**:通过 DeerFlow 创建、更新、删除、回滚 Wiki 页面后,立即入队同步该页; +3. **管理员手动同步**:用于排障和首次导入; +4. **管理员全量重建**:用于切换 Embedding 模型/维度或修复索引。 + +### 8.2 知识库发现 + +每轮调度: + +1. 从 `llmwiki_knowledge_bases` 获取 `wiki_index_enabled=true` 的映射; +2. 排除已删除映射; +3. 向 WeKnora 查询远端知识库存在性与 Wiki 能力; +4. 远端不存在时不立即删除本地数据,而是标记同步失败; +5. 只有显式删除映射或连续确认上游知识库已删除时,才进入本地清理流程; +6. 每个知识库独立获得租约,避免多 Gateway worker 重复同步。 + +### 8.3 完整扫描算法 + +伪代码: + +```python +async def sync_knowledge_base(mapping_id): + state = await repo.try_acquire_lease(mapping_id, owner, ttl=600) + if not state.acquired: + return "already_running" + + sync_id = uuid4() + complete_scan = False + try: + remote_kb = await weknora.get_knowledge_base(remote_id) + assert wiki_enabled(remote_kb) + + page = 1 + while True: + batch = await weknora.list_wiki_pages(remote_id, page=page, page_size=100) + for remote_page in batch.pages: + await mirror_and_maybe_index(mapping_id, remote_page, sync_id) + await repo.renew_lease(mapping_id, owner) + if page >= batch.total_pages: + break + page += 1 + + complete_scan = True + await repo.mark_missing_pages_deleted(mapping_id, last_seen_sync_id_not=sync_id) + await repo.finish_sync_success(mapping_id, sync_id) + except Exception as exc: + await repo.finish_sync_failure(mapping_id, sync_id, safe_error(exc)) + raise + finally: + await repo.release_lease(mapping_id, owner) +``` + +关键安全规则:**只有本轮所有分页都成功完成时,才能把“未出现页面”标为删除。** 中途超时或某一页失败时,绝不能根据不完整集合删除本地页面。 + +### 8.4 页面变化判断 + +用于 `content_hash` 的规范化内容建议为: + +```text +title +aliases(稳定排序) +summary +content_md(统一换行,保留 Markdown) +page_type +status +``` + +不把更新时间本身放入 hash,避免上游只刷新时间就重复调用 Embedding。 + +判断: + +| 场景 | 动作 | +|---|---| +| 本地没有页面 | 保存镜像,状态 pending,建立向量 | +| hash 和 embedding fingerprint 都相同 | 仅更新 `last_seen_sync_id` 和必要元数据 | +| hash 变化 | 保存新镜像,重建该页全部章节向量 | +| fingerprint 变化 | 内容不变也要重建向量 | +| 状态变 archived | 保存镜像,删除/停用活动向量,revision +1 | +| 状态从 archived 变 published | 重新建立或恢复向量,revision +1 | +| 完整扫描确认页面消失 | 标记 remote_deleted,删除向量,revision +1 | + +### 8.5 页面镜像与向量的原子可见性 + +不能先删除旧向量,再调用远程 Embedding,否则 Embedding 失败期间页面完全不可检索。 + +推荐流程: + +1. 在事务外准备规范化文本、chunks 和新 embedding; +2. Embedding 全部成功并校验维度; +3. 开启数据库事务; +4. upsert 页面镜像; +5. 删除该页旧活动向量; +6. 批量插入新向量; +7. 设置 `index_status=ready`、更新 fingerprint; +8. 对知识库 `index_revision + 1`; +9. 提交事务。 + +如果内容镜像已经变化但向量生成失败,有两种选择: + +- 保留旧页面+旧向量,直到新向量成功; +- 保存新页面但继续用旧向量。 + +本文默认采用更一致的第一种:**页面检索镜像与向量同事务换代**。可另存 `pending_content_hash/last_error` 记录待更新状态,但查询始终看到同一版本的页面与向量。 + +### 8.6 删除和归档 + +- 上游页面归档:本地保留页面镜像用于审计,但搜索快照排除; +- 上游页面删除:本地标记 `is_remote_deleted=true`,删除向量; +- DeerFlow 映射删除:级联清理本地页面和向量; +- 知识库取消公共发布:不删除本地向量,因为所有者仍可能内部检索;只通过权限过滤立即禁止公共/外部访问; +- 关闭 `wiki_index_enabled`:停止同步并从搜索快照排除,是否物理清理由管理员另行执行。 + +### 8.7 多 worker 租约 + +租约获取应使用数据库原子条件更新: + +```text +lease_owner IS NULL OR lease_expires_at < now OR lease_owner = current_owner +``` + +同步过程中每完成一个分页或一批 Embedding 续租。进程崩溃后,租约到期由其它 worker 接管。不要只使用本地文件锁,因为 PostgreSQL/MySQL 多实例部署下文件锁不可见。 + +SQLite 单机仍可使用同一数据库租约逻辑,不需要分支实现。 + +--- + +## 9. Wiki 页面切分与 Embedding + +### 9.1 为什么检索章节、引用页面 + +完整 Wiki 页面可能很长。如果整页只生成一个向量: + +- 多主题页面语义被平均; +- 局部事实召回变差; +- 送给模型时正文过长; +- 页面更新任一小段都会改变整页表达。 + +因此: + +- **向量实体**:Wiki 页面章节; +- **引用实体**:Wiki 页面; +- **展示实体**:Wiki 页面; +- **给模型的内容**:命中的 1~2 个章节。 + +### 9.2 文本预处理 + +处理顺序: + +1. 统一 `\r\n` 为 `\n`; +2. 去除 YAML frontmatter,但将有价值的 aliases/category 显式加入标题前缀; +3. 保留 Markdown 标题文字; +4. 保留表格文本,但可把过长表格按行切分; +5. 移除纯 HTML script/style; +6. 不执行 Markdown 内嵌 HTML; +7. 图片仅保留 alt 文本和标题,不下载图片生成 embedding; +8. 规范化连续空行; +9. 对“对话沉淀”继续沿用已有品牌清理规则; +10. 不对 Wiki 正文做展示层敏感词替换后再向量化,避免管理配置变化导致全量重建;敏感信息是否允许进入索引应在同步前使用专门安全策略决定。 + +### 9.3 切分格式 + +建议每个 embedding 输入都带页面上下文: + +```text +页面:{title} +别名:{aliases} +类型:{page_type} +章节:{heading_path} + +{section_content} +``` + +这样即使命中片段正文没有重复实体名,向量仍能感知页面主题。 + +首版参数: + +- `max_chars=1000`; +- `overlap=120`; +- `min_chars=80`; +- 优先按 H1~H6 切分; +- 过长章节按段落打包; +- 单段仍超长时硬切; +- 空章节跳过; +- summary 可作为第 0 段,格式为“页面摘要”。 + +### 9.4 内容去重 + +同一页面内如果两个 chunk 规范化后 hash 相同,只保留一个。不同页面内容相同也不全局去重,因为它们可能属于不同权限知识库和不同引用页面。 + +### 9.5 Embedding 调用要求 + +现有 `EmbeddingClient` 是 best-effort 设计,失败时返回空数组。Wiki 正式索引应封装一个严格模式: + +- 支持批量请求; +- 指数退避重试; +- 区分超时、鉴权、限流、响应格式和维度错误; +- 返回数量必须与输入数量相同; +- 任一向量为空、维度不同、含 NaN/Inf,则本批失败; +- 错误写入页面 `last_error`,但不覆盖旧索引; +- 日志不输出正文、API Key 或完整响应体; +- 记录模型名、批大小、耗时、状态码和安全错误分类。 + +### 9.6 模型切换 + +Embedding 指纹变化后: + +1. 新查询不得把旧模型向量和新查询向量比较; +2. 状态接口显示 `rebuild_required=true`; +3. 管理员触发全量重建; +4. 重建可逐知识库进行; +5. 某个知识库全部新向量完成后,再切换其 active fingerprint; +6. 未完成的知识库继续使用旧 fingerprint 与旧查询模型是不现实的,因此建议全量重建期间暂停严格向量查询,或同时保留旧、新两个 Embedding 客户端配置。首版选择“显示维护中并暂停该库搜索”,降低复杂度。 + +--- + +## 10. DeerFlow 本地向量执行器 + +### 10.1 首版选择:NumPy 精确检索 + +当前依赖中已有 NumPy。首版无需引入 FAISS/HNSW/Qdrant,采用: + +1. 从数据库读取某知识库当前 fingerprint 的 ready vectors; +2. 将 BLOB 解码为 `numpy.ndarray(dtype=float32)`; +3. 堆叠为形状 `[N, D]` 的归一化矩阵; +4. 查询向量归一化为 `[D]`; +5. 计算 `scores = matrix @ query_vector`; +6. 使用 `numpy.argpartition` 取 TopK,而不是完整排序; +7. 根据 metadata 返回页面和章节。 + +这是精确余弦检索,不是近似检索。 + +### 10.2 为什么按知识库懒加载 + +如果一次加载全部 Wiki: + +- 多 worker 会重复占用大量内存; +- 权限过滤需要在全矩阵上构造 mask; +- 热门小库会被冷门大库拖累; +- 单库更新导致全局矩阵重建。 + +因此每个 worker 保存 LRU: + +```text +mapping_id -> WikiVectorSnapshot +``` + +快照包含: + +```python +class WikiVectorSnapshot: + mapping_id: str + index_revision: int + embedding_fingerprint: str + dimensions: int + matrix: np.ndarray + rows: list[VectorMetadata] + loaded_at: datetime +``` + +查询多个知识库时,对各知识库快照分别取 TopK,再做全局合并。这样无需把不同库物理拼成一个永久矩阵。 + +### 10.3 缓存失效 + +每次查询或最多每隔 `cache_revision_check_seconds` 检查数据库中的 `index_revision`: + +- 修订号相同:复用快照; +- 修订号变化:为该知识库加载新快照并原子替换; +- 加载失败:继续保留旧快照,但若权限/发布状态变化,权限层仍会立即阻断; +- 映射删除/索引禁用:立即从 LRU 移除; +- LRU 超过 `cache_max_knowledge_bases`:驱逐最久未使用快照。 + +同一 worker 内多个请求同时发现修订变化时,用 per-KB async lock 合并重载,避免击穿数据库。 + +### 10.4 内存估算 + +归一化 float32 矩阵仅向量部分约占: + +```text +向量数 × 维度 × 4 字节 +``` + +示例(不含 Python metadata 开销): + +| 章节向量数 | 维度 | 纯矩阵内存 | +|---:|---:|---:| +| 10,000 | 1,024 | 约 39 MiB | +| 50,000 | 1,024 | 约 195 MiB | +| 100,000 | 1,024 | 约 391 MiB | +| 100,000 | 768 | 约 293 MiB | + +因此: + +- 首版上线前必须用真实 Wiki 数量做容量评估; +- 多 worker 内存要乘以 worker 数; +- LRU 应限制同时驻留的知识库数量; +- 大库可进一步按固定 shard 切分; +- 若单库达到数十万章节向量或内存不可接受,应切换 pgvector,而不是继续扩大 Python 进程内存。 + +上述数量仅用于容量估算,不作为硬性能承诺,最终阈值由目标机器基准测试确定。 + +### 10.5 查询算法 + +伪代码: + +```python +async def vector_search(query, allowed_mapping_ids, top_k_sections=30): + q = await embedding.embed_query(query) + q = validate_and_normalize(q) + + candidates = [] + for mapping_id in allowed_mapping_ids: + snapshot = await cache.get(mapping_id) + if snapshot.fingerprint != active_fingerprint: + continue + scores = snapshot.matrix @ q + local_top = argpartition_top(scores, min(top_k_sections, len(scores))) + candidates.extend(snapshot.materialize(local_top, scores)) + + candidates.sort(key=lambda item: item.score, reverse=True) + candidates = [c for c in candidates if c.score >= min_similarity] + return group_by_page(candidates) +``` + +### 10.6 页面聚合 + +同一页可能命中多个章节。聚合规则: + +1. 页面分数默认取最高章节分数; +2. 可选加入第二章节小幅加权,但不能简单相加导致长页面占优; +3. 每页最多保留 `max_sections_per_page=2`; +4. 页面按最终分数排序; +5. 同一 slug 在不同知识库中视为不同页面; +6. 返回 `matched_sections`,模型可接收多个片段,右侧只列一张页面卡片。 + +建议页面分数: + +```text +page_score = max_section_score + 0.05 * second_section_score(若存在) +``` + +最终裁剪到 1.0 仅用于展示;内部排序保留原始值。 + +### 10.7 可选关键词加权 + +用户已明确要求向量检索,因此关键词不得成为隐式回退。可选 hybrid 仅用于重排: + +```text +final_score = (1 - w) * vector_score + w * keyword_score +``` + +默认 `w=0`。若后续启用,关键词评分只针对已由向量召回的候选,不额外从原始文档召回。 + +--- + +## 11. 权限模型 + +### 11.1 内部检索权限 + +检索前必须先得到 DeerFlow 本地 mapping ID 列表,并沿用现有规则: + +1. 当前对话明确选择的 `llmwiki_knowledge_base_ids`; +2. 当前智能体 `config.yaml` 允许绑定的范围; +3. 当前用户对映射的实时读取权限; +4. 对话明确空列表表示不检索; +5. 用户权限撤销后,旧线程下一轮查询也必须失效; +6. 不接受前端或模型传入 WeKnora remote ID。 + +完成上述交集后,才允许读取相应本地向量快照。 + +### 11.2 页面状态 + +默认: + +- 普通用户:仅 published; +- 公共库访问者:仅 published; +- 所有者/管理员:默认仍仅 published,明确 `include_drafts=true` 才包含 draft; +- archived/deleted:任何搜索都排除; +- 详情接口对所有者/管理员可按管理场景读取 archived,但引用详情不读取。 + +### 11.3 外部接口权限 + +每个返回结果必须同时满足: + +```text +mapping.publication_status == "published" +AND mapping.external_search_enabled == true +AND mapping.wiki_index_enabled == true +AND page.status == "published" +AND page.is_remote_deleted == false +AND page.index_status == "ready" +AND mapping 不是系统“对话沉淀” +``` + +即使 NumPy 快照仍在内存中,只要数据库映射已取消发布或关闭外部开关,权限检查就应在查询入口立即阻断。因此缓存失效延迟不会造成权限泄漏。 + +### 11.4 外部开放开关为什么独立 + +当前“公共库”语义是 DeerFlow 登录用户可读,不等于互联网匿名/服务对服务可读。如果直接把所有 published 库放进外部 API,会扩大历史数据的受众范围。 + +默认策略: + +- 数据库迁移后,所有现有公共库 `external_search_enabled=false`; +- 所有者或管理员明确开启; +- UI 显示“内部公共”和“外部 API 开放”是两个开关; +- 开启外部开放前提示内容将可由持有 API Key 的系统检索; +- 关闭后立即生效。 + +### 11.5 系统“对话沉淀” + +该知识库在现有逻辑中始终 public,但可能含用户对话内容。因此: + +- 外部搜索强制排除,不能仅依赖可编辑字段; +- 外部详情接口同样排除; +- 状态接口不得把其 remote ID 暴露出去; +- 内部检索继续按当前产品策略使用; +- 未来若要外部开放,必须另做数据脱敏和用户授权设计,不属于本方案。 + +--- + +## 12. 统一检索结果协议 + +### 12.1 内部结构 + +建议统一结果: + +```json +{ + "kind": "wiki_page", + "id": "local-page-id", + "knowledge_base_id": "deerflow-mapping-id", + "knowledge_base_name": "公共知识库", + "wiki_page_id": "local-page-id", + "wiki_slug": "entity/alice", + "title": "Alice", + "summary": "页面摘要", + "page_type": "entity", + "page_version": 3, + "updated_at": "2026-08-21T10:00:00+08:00", + "score": 0.8621, + "matched_sections": [ + { + "section_index": 2, + "heading": "主要经历", + "content": "命中的 Wiki 章节正文", + "score": 0.8621 + } + ], + "url": "/weknora-source-preview?knowledge_base_id=deerflow-mapping-id&wiki_slug=entity%2Falice", + "source": "公共知识库", + "type": "llmwiki_wiki", + "mode": "citation" +} +``` + +注意:URL 名称可继续保持 `/weknora-source-preview` 以兼容现有解析器,但它实际上读取 DeerFlow 本地镜像。也可以新增更准确的 `/llmwiki-wiki-preview`,并同时兼容旧链接。实现时推荐新增链接格式并保留旧格式解析。 + +### 12.2 不对外返回的字段 + +外部 API 默认不返回: + +- `remote_page_id`; +- WeKnora remote knowledge base ID; +- `source_refs`; +- `chunk_refs`; +- 向量 BLOB; +- Embedding 模型内部地址或 API Key; +- owner user ID; +- 同步错误原文; +- 草稿/归档页面存在性。 + +### 12.3 给大模型的上下文 + +建议格式: + +```text + +[1] 页面标题 +知识库:xxx +页面:entity/alice +命中章节:主要经历 +相关度:0.8621 +正文:... + +[2] ... + +``` + +要求模型: + +- 正文引用只能使用存在的 `[N]`; +- 同一 Wiki 页面多个章节仍只对应一个编号; +- 不把 `source_refs/chunk_refs` 暴露给模型; +- 资料不足时明确说明; +- 不自行把 Wiki 内容描述成原始一手文献。 + +--- + +## 13. 后端接口设计 + +### 13.1 内部搜索接口 + +建议新增: + +```http +POST /api/llmwiki/wiki-vector-search +Authorization: Bearer +Content-Type: application/json +``` + +请求: + +```json +{ + "query": "任正非早期创业经历", + "knowledge_base_ids": ["deerflow-mapping-id"], + "top_k": 8, + "include_drafts": false, + "include_content": true +} +``` + +约束: + +- `query` 1~4000 字符; +- mapping IDs 最多 100; +- `top_k` 1~50; +- 服务端重新鉴权,不能相信前端选择; +- `include_drafts` 仅所有者/管理员生效; +- strict vector 下索引未就绪返回 409/503,不走 chunk search。 + +响应: + +```json +{ + "query": "任正非早期创业经历", + "mode": "vector", + "results": [], + "knowledge_base_count": 1, + "index_fingerprint": "abc123...", + "partial": false, + "warnings": [] +} +``` + +### 13.2 本地 Wiki 页面详情 + +```http +GET /api/llmwiki/knowledge-bases/{mapping_id}/local-wiki/pages/{slug:path} +``` + +返回本地镜像,不访问 WeKnora。权限与现有 `get_authorized(write=false)` 一致。 + +响应包含: + +- title/summary/content/page_type/status; +- aliases/category/version/updated_at; +- 索引状态和最近同步时间(管理者可见); +- 普通引用视图不返回 source_refs/chunk_refs; +- 管理详情可增加 `include_provenance=true`,仅所有者/管理员读取溯源。 + +### 13.3 索引管理接口 + +管理员接口: + +```http +GET /api/llmwiki/wiki-index/status +POST /api/llmwiki/wiki-index/sync +POST /api/llmwiki/wiki-index/rebuild +POST /api/llmwiki/wiki-index/knowledge-bases/{mapping_id}/sync +POST /api/llmwiki/wiki-index/knowledge-bases/{mapping_id}/rebuild +``` + +`sync` 请求示例: + +```json +{ + "knowledge_base_ids": ["..."], + "force": false +} +``` + +`rebuild` 请求示例: + +```json +{ + "knowledge_base_ids": ["..."], + "reason": "embedding_model_changed" +} +``` + +长任务应返回 `202 Accepted + job_id`,不要在 HTTP 请求中等待整个知识库完成。 + +### 13.4 外部向量搜索接口 + +推荐路径: + +```http +POST /api/external/llmwiki/wiki/vector-search +X-API-Key: +Content-Type: application/json +``` + +请求: + +```json +{ + "query": "任正非早期创业经历", + "knowledge_base_ids": [], + "limit": 10, + "include_content": false +} +``` + +语义: + +- 空 `knowledge_base_ids` 表示所有允许外部搜索的公共 Wiki; +- 非空列表必须是 DeerFlow mapping ID; +- 请求私有/未开放 ID 时建议统一返回 404 或空结果,避免枚举资源; +- 查询只使用本地向量; +- 默认返回摘要和命中片段; +- `include_content=true` 也要受单页和总响应字符上限约束。 + +响应: + +```json +{ + "query": "任正非早期创业经历", + "mode": "vector", + "items": [ + { + "knowledge_base_id": "public-mapping-id", + "knowledge_base_name": "人物知识库", + "wiki_slug": "entity/ren-zhengfei", + "title": "任正非", + "summary": "...", + "matched_heading": "早期创业", + "snippet": "...", + "page_type": "entity", + "page_version": 5, + "updated_at": "...", + "score": 0.89 + } + ], + "count": 1, + "partial": false +} +``` + +### 13.5 外部页面详情 + +```http +GET /api/external/llmwiki/wiki/pages/{mapping_id}/{slug:path} +X-API-Key: +``` + +再次实时检查公共/外部开关和页面 published 状态,不能因为搜索结果曾经返回过就绕过当前权限。 + +### 13.6 外部 API 鉴权接入 + +当前 `/api/public/` 前缀会绕过全局登录鉴权,不适合直接承载未加保护的重型向量接口。推荐: + +1. 使用 `/api/external/`; +2. 在 AuthMiddleware 中增加仅对该前缀生效的外部 API Key 分支; +3. 使用 `hmac.compare_digest()` 常量时间比较; +4. API Key 从环境变量解析,不存数据库明文; +5. 缺失/错误统一返回 401,不透露配置状态; +6. 日志只记 key 指纹前 8 位或完全不记; +7. CORS 不作为服务到服务鉴权手段; +8. 生产环境同时在 Nginx/API Gateway 做 IP、速率和请求体大小限制。 + +如果为了少改 AuthMiddleware 而把接口挂到 `/api/public/`,路由自身也必须强制校验 API Key;但长期推荐独立 `/api/external/` 语义。 + +### 13.7 状态码约定 + +| 状态码 | 场景 | +|---:|---| +| 200 | 搜索成功,包括 0 结果 | +| 202 | 同步/重建任务已接受 | +| 400 | 参数格式错误 | +| 401 | 外部 API Key 或登录凭据无效 | +| 403 | 已认证但无操作权限 | +| 404 | 映射/页面不存在或不可见 | +| 409 | 索引未就绪、正在重建或模型指纹不一致 | +| 413 | 查询/响应内容请求过大 | +| 429 | 限流 | +| 502 | 同步阶段 WeKnora 异常(搜索阶段不应返回) | +| 503 | Embedding 服务或本地向量执行器不可用 | + +--- + +## 14. 内部问答与工具接入 + +### 14.1 普通问答预检索 + +修改 `app/gateway/llmwiki_rag.py`: + +- 原 `build_weknora_client(...).search()` 替换为本地 WikiVectorSearchService; +- 仍沿用对话选择和用户权限; +- `_source_from_result()` 改为 Wiki 结构; +- prompt 只包含命中章节; +- `_MAX_SOURCES` 仍可保留 8; +- `_MAX_SOURCE_CHARS` 应对每页所有命中章节的合计做限制; +- 索引未就绪时给模型明确提示,不能声称“知识库没有内容”。 + +### 14.2 智能体工具 + +修改 `deerflow/tools/builtins/llmwiki_search_tool.py`: + +- 工具名可继续为 `llmwiki_search`,避免技能提示词迁移; +- 实现改为查询 DeerFlow 本地 Wiki 索引; +- 实时检查智能体绑定、对话选择和用户权限; +- 返回 JSON 中 results 全部为 `kind=wiki_page`; +- URL 只使用 mapping ID + wiki slug; +- 不再返回原始 `chunk_id/knowledge_id/document_id` 作为主引用; +- 工具描述改成“检索已处理的 Wiki 页面”。 + +由于 harness 包不能 import `app.*`,本地索引 repository/search service 必须放在 `deerflow.*` 下,并通过 runtime setter 或应用初始化注入,不能让工具直接引用 Gateway router。 + +### 14.3 页面检索接口 + +现有 `POST /api/llmwiki/search` 可做兼容升级: + +- 默认 `source_kind=wiki`; +- 新增 `mode=vector`; +- Wiki 库走本地向量; +- 标准非 Wiki 库是否保留原始检索需产品明确。本文默认:该接口如果明确选择标准库,可保留 `raw_chunk`,但问答中选中 Wiki 库时严格只查 Wiki; +- 更清晰的做法是保留旧接口并新增 `/wiki-vector-search`,前端逐步迁移后再废弃旧模式。 + +### 14.4 深度研究和 AI 写作 + +它们通过 `llmwiki_search` 或共享参考文献归一化间接使用 WeKnora 时,应自然收到新的 Wiki result。需要回归: + +- `deep_research.adapters.material_provider`; +- `deep_research.collection.thread_harvester`; +- AI 写作 `_skill_agent.py` 参考文献归一化; +- `deerflow.runtime.references` 对 raw 字段的保留。 + +必须确保 `wiki_slug/page_id/matched_sections` 不在归一化过程中被丢弃。 + +--- + +## 15. 前端改造 + +### 15.1 引用目标解析 + +修改: + +```text +frontend-web/src/core/llmwiki/source-links.ts +``` + +当前 preview URL 解析要求 `knowledge_base_id && chunk_id`。新逻辑应允许: + +```text +knowledge_base_id && (wiki_slug || chunk_id || document_id) +``` + +优先级: + +1. `wiki_slug`; +2. `chunk_id`(历史兼容); +3. `document_id`(历史兼容)。 + +### 15.2 右侧参考文献抽屉 + +修改 `WeKnoraSourceDrawer.tsx` 或拆出 `LlmWikiWikiSourceDrawer`: + +当 target 有 `wikiSlug`: + +- 调用 DeerFlow 本地页面详情接口; +- 不调用 WeKnora 文档 preview; +- 不加载 document chunks; +- 不展示“下载原文”作为主按钮; +- 展示标题、摘要、页面类型、更新时间、版本、完整 Markdown; +- 可高亮 `matched_heading`; +- 可以有默认折叠的“支撑来源”入口,但仅登录且有权限时可用; +- 外部公共页面不能借此读取 source_refs/chunk_refs。 + +当 target 只有旧 `chunkId/documentId`: + +- 保留当前历史引用预览; +- 可尝试本地反向映射到 Wiki; +- 找不到则继续显示原文,不阻断历史聊天。 + +### 15.3 参考文献卡片 + +卡片显示: + +- Wiki 页面标题; +- 知识库名称; +- 页面类型; +- 命中章节标题; +- Wiki 摘要或命中片段; +- 向量相关度; +- 更新时间; +- “查看 Wiki”而非“查看原文”。 + +不要在卡片上显示: + +- 原文件名作为主标题; +- chunk index; +- WeKnora remote ID; +- 向量维度/模型; +- source_refs 原始 ID。 + +### 15.4 管理页面建议 + +在知识库详情或系统设置增加“本地向量索引”状态: + +- 已同步页面数; +- 已索引页面数; +- 向量章节数; +- 最近同步时间; +- 当前状态/错误; +- Embedding 模型与短指纹; +- “立即同步”; +- “重建向量”; +- “允许外部 API 检索”开关; +- 重建进度。 + +首版如果不做完整管理页,至少保证管理员 API 和日志可用。 + +--- + +## 16. 后端模块划分 + +建议新增: + +```text +packages/harness/deerflow/persistence/llmwiki_index/ + __init__.py + base.py + model.py + memory.py + sql.py + +packages/harness/deerflow/integrations/weknora/local_index/ + __init__.py + schemas.py + normalize.py + chunking.py + embedding.py + repository.py # 可直接使用 persistence store 时省略 + vector_codec.py + vector_cache.py + search.py + sync.py + runtime.py + +app/gateway/routers/ + llmwiki_index.py + external_llmwiki.py + +app/gateway/ + llmwiki_index_scheduler.py +``` + +也可以将 local_index 放到 `deerflow/llmwiki_index/`,关键要求是: + +- harness 不 import app; +- Gateway 负责 HTTP、用户解析、调度启动; +- harness 负责数据模型、同步核心、向量编码、缓存和搜索; +- WeKnora client 仍只负责官方 REST 适配; +- 路由不直接写 SQL。 + +### 16.1 Store 接口建议 + +```python +class LlmWikiIndexStore(ABC): + async def get_page(...): ... + async def list_pages(...): ... + async def stage_or_replace_page_vectors(...): ... + async def mark_page_failed(...): ... + async def mark_missing_pages_deleted(...): ... + async def load_vector_snapshot(mapping_id, fingerprint): ... + async def get_index_revision(mapping_id): ... + async def try_acquire_sync_lease(...): ... + async def renew_sync_lease(...): ... + async def finish_sync(...): ... + async def list_index_status(...): ... +``` + +提供 SQL 和 memory 实现,便于单元测试及 auth-disabled 开发模式。 + +### 16.2 应用初始化 + +在 Gateway lifespan 中: + +1. 创建/取得 index store; +2. 创建严格 Embedding client; +3. 创建 WikiVectorCache; +4. 创建 WikiVectorSearchService; +5. 创建 WikiSyncService; +6. 挂到 `app.state`; +7. 给 harness runtime setter 注入只读 search/store; +8. 若 sync enabled,启动调度 loop; +9. shutdown 时停止调度并清理缓存/HTTP client。 + +调度器需要和现有 lifespan 后台任务保持相同的取消与超时风格。 + +--- + +## 17. 数据库迁移与兼容性 + +### 17.1 Alembic 迁移 + +新增迁移: + +```text +deerflow/persistence/migrations/versions/_01_llmwiki_local_wiki_index.py +``` + +迁移内容: + +1. 扩展 `llmwiki_knowledge_bases`; +2. 创建 `llmwiki_wiki_pages`; +3. 创建 `llmwiki_wiki_vectors`; +4. 创建 `llmwiki_wiki_sync_states`; +5. 建立 FK、唯一约束和索引; +6. 所有现有 mapping 的 `external_search_enabled=false`; +7. `wiki_index_enabled=true` 或按配置决定; +8. 不在迁移过程中调用 WeKnora 或 Embedding。 + +迁移必须兼容 SQLite、PostgreSQL 和 MySQL。注意 MySQL utf8mb4 长索引限制,slug 使用 hash 唯一约束。 + +### 17.2 向量 BLOB 可移植性 + +- SQLite:BLOB; +- PostgreSQL:BYTEA; +- MySQL:LONGBLOB/BLOB,按最大维度选择; +- SQLAlchemy 使用 `LargeBinary`; +- 字节序固定 little-endian; +- 模型维度显式保存; +- 不依赖数据库特定 vector 类型,便于首版跨后端运行。 + +### 17.3 首次回填 + +迁移完成后不自动阻塞启动进行全量回填。建议: + +1. 服务正常启动; +2. 管理员查看状态为 `not_synced`; +3. 手动或后台分库同步; +4. 同步完成一个库就允许该库向量检索; +5. 全部稳定后再将问答默认切换为 local wiki vector; +6. 外部 API 最后开启。 + +### 17.4 历史引用 + +历史消息中的引用仍可能只有 chunk/document。必须保留: + +- 旧 URL 解析; +- 旧 source drawer; +- `/api/llmwiki/sources/context`; +- 原文预览权限。 + +新消息只生成 Wiki 引用。不要批量重写历史消息,除非后续建立可靠 `chunk_ref → local wiki page` 反向表并单独实施迁移。 + +--- + +## 18. 外部 API 安全设计 + +### 18.1 威胁模型 + +需要防范: + +- 枚举私有 mapping ID; +- 通过 query 大小或 top_k 放大 CPU/Embedding 成本; +- 高频调用耗尽 Embedding 服务配额; +- 请求全部公共库导致加载大量快照; +- 返回完整 Wiki 内容造成带宽放大; +- Markdown 中的 HTML/script 被下游不安全渲染; +- API Key 泄漏; +- 关闭发布后缓存继续返回; +- “对话沉淀”误开放; +- 错误日志包含正文或敏感标识。 + +### 18.2 控制措施 + +1. 外部 API Key 必填; +2. query 字符数、mapping 数、top_k、响应字符数硬上限; +3. 请求体大小由应用和反向代理共同限制; +4. 限流按 API Key + IP; +5. 查询向量短 TTL 缓存可减少重复 embedding,但缓存 key 不保存原文日志; +6. 外部查询只加载明确开放的 KB; +7. 权限在检索前和返回前各检查一次; +8. 页面详情接口再次检查; +9. 返回 Markdown 时标注 content type,前端渲染必须禁用危险 HTML; +10. 不返回 remote ID、source refs、chunk refs、owner 信息; +11. 日志对 query 做长度裁剪或 hash; +12. “对话沉淀”代码级 deny; +13. API Key 支持轮换:可配置 current + previous,短时间双 key 过渡; +14. 对外接口默认关闭。 + +### 18.3 Embedding 成本控制 + +外部每次查询需要一次 embedding。建议: + +- 对规范化 query 做 1~5 分钟进程内 LRU 缓存; +- 缓存 key 使用 `embedding_fingerprint + sha256(query)`; +- 缓存值是查询向量,不是结果权限; +- 权限和结果仍每次实时计算; +- 不跨 fingerprint 复用; +- 缓存大小有上限; +- API Key 限流在调用 Embedding 前执行。 + +--- + +## 19. 失败处理与一致性 + +| 故障 | 预期行为 | 是否回退原文 | +|---|---|---:| +| WeKnora 同步超时 | 保留旧镜像/向量,状态 partial/failed | 否 | +| 某页读取失败 | 旧页继续可用,记录页面错误 | 否 | +| Embedding 超时 | 不覆盖旧向量,重试 | 否 | +| Embedding 401/403 | 标记配置错误,停止无意义重试 | 否 | +| 返回向量数量不一致 | 本批失败,不落库 | 否 | +| 维度变化 | 标记 rebuild_required | 否 | +| BLOB 损坏 | 跳过损坏库/页并告警,触发重建 | 否 | +| NumPy 快照加载失败 | 保留旧快照;无旧快照则索引不可用 | 否 | +| 单个 KB 搜索失败 | 内部可返回 partial + warning;外部默认 partial | 否 | +| 所有 KB 不可用 | 返回 409/503 | 否 | +| 取消公共发布 | 权限层立即过滤 | 不适用 | +| 外部开关关闭 | 权限层立即过滤 | 不适用 | +| Gateway 重启 | 从 DB 懒加载,无需重做 embedding | 不适用 | + +### 19.1 “没有命中”和“索引不可用”必须区分 + +- 正常 0 结果:HTTP 200,`results=[]`; +- 未同步:409,错误码 `WIKI_INDEX_NOT_READY`; +- 正在重建:409,`WIKI_INDEX_REBUILDING`; +- Embedding 服务失败:503,`WIKI_EMBEDDING_UNAVAILABLE`; +- 指纹不匹配:409,`WIKI_INDEX_FINGERPRINT_MISMATCH`; +- 无权限/不存在:404 或权限语义对应状态。 + +模型提示也必须区分,避免把系统故障回答成“知识库没有相关资料”。 + +--- + +## 20. 可观测性 + +### 20.1 结构化日志 + +同步日志字段: + +```text +mapping_id +sync_id +page_number +remote_page_count +changed_pages +unchanged_pages +deleted_pages +embedded_chunks +failed_pages +duration_ms +error_category +``` + +检索日志字段: + +```text +request_id +caller_type (internal/external/tool/pre_rag) +authorized_kb_count +loaded_snapshot_count +candidate_section_count +result_page_count +embedding_ms +search_ms +total_ms +partial +``` + +不得记录:API Key、完整 query、Wiki 全文、向量。 + +### 20.2 指标 + +建议指标: + +- `llmwiki_sync_runs_total{status}`; +- `llmwiki_sync_duration_seconds`; +- `llmwiki_pages_total{status,index_status}`; +- `llmwiki_vectors_total`; +- `llmwiki_embedding_requests_total{status}`; +- `llmwiki_embedding_duration_seconds`; +- `llmwiki_search_requests_total{caller,status}`; +- `llmwiki_search_duration_seconds{stage}`; +- `llmwiki_search_results_count`; +- `llmwiki_vector_cache_hits_total`; +- `llmwiki_vector_cache_reloads_total`; +- `llmwiki_vector_cache_bytes`; +- `llmwiki_external_rate_limited_total`。 + +若当前没有 Prometheus,可先接现有日志/工具指标体系,并保留后续扩展点。 + +### 20.3 状态接口 + +管理员状态页至少能回答: + +- 哪些库尚未同步; +- 哪些库正在同步; +- 哪些页面失败; +- 当前模型指纹; +- 是否需要重建; +- 最近一次完整成功时间; +- 本地页数和向量数; +- 缓存是否加载; +- 外部开放是否开启。 + +--- + +## 21. 性能与扩展路线 + +### 21.1 首版基准测试要求 + +用真实或合成数据覆盖: + +- 1,000 / 10,000 / 50,000 / 100,000 章节; +- 768 / 1,024 / 1,536 维; +- 单库与 5/20 库并发查询; +- 1/4 Gateway workers; +- 冷加载与热缓存; +- 同步和搜索并行; +- SQLite 与生产数据库后端。 + +记录: + +- 冷加载时间; +- 单次向量计算 P50/P95/P99; +- 进程 RSS; +- DB BLOB 读取吞吐; +- Embedding 延迟; +- 多并发下事件循环阻塞情况。 + +NumPy 矩阵计算和大 BLOB 解码应放入 `asyncio.to_thread()`,避免阻塞事件循环。 + +### 21.2 何时迁移 pgvector + +出现以下任一情况时评估 pgvector: + +- 单库快照内存不可接受; +- 多 worker 重复内存过高; +- 冷启动/重载时间过长; +- 向量数量达到数十万以上且查询并发提高; +- 需要数据库端按 mapping/status 过滤并近似索引; +- 需要多节点共享统一向量执行层。 + +迁移时保持: + +- 页面表不变; +- 统一结果协议不变; +- 权限服务不变; +- API 不变; +- 只替换 `WikiVectorEngine` 实现。 + +建议抽象: + +```python +class WikiVectorEngine(ABC): + async def search(self, query_vector, mapping_ids, top_k, fingerprint): ... + async def invalidate(self, mapping_id): ... +``` + +首版 `NumpyExactWikiVectorEngine`,未来 `PgVectorWikiVectorEngine`。 + +--- + +## 22. 测试方案 + +### 22.1 单元测试 + +1. Markdown 标题切分、长段落、表格、空页、frontmatter; +2. title/alias/heading 前缀构造; +3. chunk hash 与页面 hash 稳定性; +4. float32 BLOB 编解码; +5. NaN/Inf/维度错误拒绝; +6. 向量归一化; +7. NumPy TopK 与手工余弦结果一致; +8. 页面聚合和每页章节上限; +9. min similarity; +10. LRU 和 revision 失效; +11. API Key 常量时间校验; +12. 系统“对话沉淀”排除; +13. embedding fingerprint; +14. 错误脱敏。 + +### 22.2 Repository 测试 + +使用 SQLite 测: + +- 创建页面/向量; +- 原子替换; +- FK cascade; +- unique mapping+slug_hash; +- 完整扫描后删除缺失页; +- 不完整扫描不删除; +- lease 获取/续租/过期接管; +- revision 只在检索数据变化时递增; +- public/external 开关默认值; +- BLOB round trip。 + +生产若使用 MySQL/PostgreSQL,补容器集成测试验证字段和索引兼容性。 + +### 22.3 同步集成测试 + +使用假的 WeKnora client: + +1. 首次 3 页同步并建立向量; +2. 第二次无变化不调用 embedding; +3. 一页 version/content 变化只重建该页; +4. 一页归档后不再检索; +5. 一页删除后清理向量; +6. 分页第 2 页失败不误删本地页; +7. Embedding 失败保留旧索引; +8. 多 worker 只有一个获得租约; +9. fingerprint 改变标记重建; +10. 服务重启后从 DB 恢复快照。 + +### 22.4 检索与权限测试 + +1. 查询仅存在于 Wiki、原始文档无该措辞,仍能命中; +2. 查询只存在于原始 chunk、Wiki 不包含,严格模式不命中; +3. 用户 A 不能检索用户 B 私有库; +4. 公共库对登录用户可检索; +5. 智能体未绑定库不能越权; +6. 对话明确空选择不检索; +7. 撤销公共发布后下一请求立即无结果; +8. draft 默认不命中; +9. 所有者明确 include_drafts 可以命中; +10. archived/deleted 永不命中; +11. 同一页多个章节只产生一个引用编号; +12. 多库同 slug 不互相覆盖。 + +### 22.5 外部 API 安全测试 + +1. 无 key/错 key → 401; +2. 私有库/未外部开放 → 不可枚举; +3. public 但 `external_search_enabled=false` → 不返回; +4. draft/archived → 不返回; +5. “对话沉淀”永不返回; +6. 关闭外部开关立即失效; +7. 超长 query → 400/413; +8. 超大 top_k → 拒绝或钳制; +9. 高频请求 → 429; +10. 响应没有 remote ID/source_refs/chunk_refs/vector; +11. 日志没有 API Key 和正文; +12. 详情接口重新鉴权,不依赖旧搜索结果。 + +### 22.6 前端测试 + +1. `knowledge_base_id + wiki_slug` 可解析为有效 target; +2. Wiki target 不请求文档 preview/chunks; +3. 抽屉展示本地 Wiki Markdown; +4. 卡片显示 Wiki 标题和命中章节; +5. 历史 chunk 引用仍能打开; +6. 权限撤销后详情返回 404,前端正确提示; +7. Wiki 内容 Markdown/HTML 安全渲染; +8. loading/error/empty 状态。 + +### 22.7 性能测试 + +至少验证: + +- 50k × 1024 维单库热查询 P95; +- 5 个库合并查询; +- 冷加载内存和耗时; +- 10/50 并发查询; +- 同步时搜索延迟; +- 多 worker 总内存; +- 查询 embedding 缓存命中效果。 + +验收阈值应在目标部署机器基准后写入配置/运维文档,不在未测量前拍脑袋承诺具体毫秒数。 + +--- + +## 23. 分阶段实施计划 + +### 阶段 0:契约与配置 + +- 增加配置模型; +- 定义统一 result schema; +- 定义错误码; +- 增加 feature flag,默认关闭; +- 确认 Embedding endpoint/model/dimensions。 + +完成标准:应用在功能关闭时行为完全不变。 + +### 阶段 1:本地镜像与同步 + +- 新增迁移和三张表; +- 实现 SQL/memory store; +- WeKnora 页面规范化; +- 完整扫描、页面 upsert、缺失删除; +- 租约、状态、手动同步 API; +- 暂不切换问答。 + +完成标准:DeerFlow DB 中 Wiki 页面和 WeKnora 一致,可独立读取详情。 + +### 阶段 2:Embedding 与本地向量 + +- 严格 Embedding client; +- Markdown chunks; +- BLOB codec; +- 页面原子换代; +- rebuild; +- 状态与失败重试。 + +完成标准:所有 ready 页面有正确 fingerprint 和向量;无变化同步不重复 embedding。 + +### 阶段 3:NumPy 检索服务 + +- per-KB snapshot; +- LRU/revision; +- TopK/阈值/聚合; +- 权限过滤; +- 内部搜索 API; +- 性能基准。 + +完成标准:可稳定从本地向量返回 Wiki 页与命中章节。 + +### 阶段 4:问答和参考文献切换 + +- 普通问答预检索; +- `llmwiki_search` 工具; +- 页面检索; +- runtime references; +- 前端 source links/drawer/card; +- 历史兼容。 + +完成标准:新问答不再产生原始文档 chunk 引用,右侧只显示 Wiki 主视图。 + +### 阶段 5:外部公共 API + +- 外部开关; +- `/api/external` 鉴权; +- vector-search/page detail; +- 限流/响应上限; +- “对话沉淀”强制排除; +- 安全测试。 + +完成标准:仅明确外部开放的公共 published Wiki 可被持 key 系统检索。 + +### 阶段 6:运维与灰度 + +- 管理状态; +- 指标/告警; +- 分库灰度; +- 容量评估; +- 故障演练; +- 回滚验证。 + +--- + +## 24. 灰度与回滚 + +### 24.1 Feature flags + +建议至少有: + +```text +local_wiki_index.enabled +local_wiki_index.sync_enabled +local_wiki_index.internal_search_enabled +local_wiki_index.reference_ui_enabled +local_wiki_index.external_api.enabled +``` + +可以逐步: + +1. 只同步; +2. 只提供管理员搜索对比; +3. 指定测试用户/知识库使用向量; +4. 全量内部问答; +5. 引用 UI; +6. 外部 API。 + +### 24.2 双跑对比 + +灰度期可在不影响回答的情况下后台双跑: + +- 旧 chunk search; +- 新 local wiki vector search。 + +只记录安全的对比指标: + +- 命中数; +- 标题集合 hash; +- TopK 重合率; +- 查询耗时; +- 是否为空; +- 人工抽样质量。 + +不要把两套内容同时注入模型,以免引用编号混乱。 + +### 24.3 回滚 + +出现问题时: + +- 关闭 internal search flag,可恢复旧 chunk 检索; +- 关闭 reference UI flag,可恢复旧抽屉; +- 本地页面/向量表保留,便于排查; +- 关闭 sync 不删除数据; +- 外部 API 可独立关闭; +- 数据库迁移不需要立即 downgrade; +- 新消息引用格式解析器应向后兼容,避免回滚后打不开已生成的新 Wiki 引用。 + +注意:产品最终目标是严格 Wiki 检索。旧 chunk 回滚仅用于上线故障处置,不应成为长期隐式 fallback。 + +--- + +## 25. 验收标准 + +### 25.1 数据同步 + +- 所有目标 Wiki 页面在 DeerFlow 有镜像; +- title/summary/content/slug/version/status 与上游一致; +- 无变化不重复 embedding; +- 单页更新只重建该页; +- 完整扫描失败不误删; +- 删除/归档后搜索立即排除; +- Gateway 重启不丢数据。 + +### 25.2 向量检索 + +- 查询向量只生成一次; +- 检索只使用 DeerFlow 本地 Wiki vectors; +- 不访问 WeKnora search; +- TopK 返回相关 Wiki 章节; +- 同页多章节只占一个引用; +- 索引未就绪不会误报“无资料”; +- Embedding 失败不回退原始文档。 + +### 25.3 问答与参考文献 + +- 普通问答、智能体工具、页面搜索结果一致; +- 新引用携带 `wiki_page_id/wiki_slug`; +- 右侧展示完整 Wiki 页面; +- 不自动展示整个原文文档; +- 历史 chunk 引用仍可用; +- 撤权后引用详情不可访问。 + +### 25.4 外部接口 + +- 只返回 published + external enabled; +- 私有/草稿/归档/对话沉淀永不泄漏; +- 不暴露 remote IDs、vectors、source refs; +- key、限流、请求上限生效; +- 取消发布或关闭外部开关立即生效; +- WeKnora 不可用时已有本地索引仍可查询。 + +--- + +## 26. 风险清单与处理 + +| 风险 | 影响 | 处理 | +|---|---|---| +| WeKnora 没有增量变更 feed | 需要完整分页扫描 | 周期完整扫描 + BFF 操作即时同步;后续如上游支持 cursor 再优化 | +| Wiki 生成有延迟 | 原文已上传但本地暂无 Wiki | 显示 not ready,不回退原文 | +| Embedding 服务不稳定 | 新页面无法入索引 | 保留旧索引、严格错误、后台重试 | +| 模型维度变化 | 新旧向量不可比较 | fingerprint + 全量重建 | +| 大库内存高 | 多 worker OOM | per-KB LRU、容量测试、后续 pgvector | +| SQLite 写锁 | 同步与业务写竞争 | 小事务、批量写、WAL、低并发同步;生产大规模用 Postgres/MySQL | +| 页面和向量版本不一致 | 引用内容与召回语义错位 | 原子换代 | +| 公共库语义扩大 | 历史内部公共内容外泄 | 独立 external flag,默认关闭 | +| 对话沉淀误开放 | 用户对话泄漏 | 代码级 deny | +| 历史引用无 slug | 不能自动展示 Wiki | 保留旧引用兼容,不强制迁移 | +| Markdown 恶意 HTML | 下游 XSS | API 无执行;前端安全渲染/禁危险 HTML | +| 多 worker 重复同步 | 重复 Embedding/写冲突 | DB 租约 | +| 缓存旧权限 | 取消发布后仍可查 | 权限不缓存或短缓存;返回前复核 | + +--- + +## 27. 文件级实施清单 + +### 后端配置与模型 + +- `deerflow/config/llmwiki_config.py`:新增 local index/embedding/external 配置; +- `deerflow/persistence/llmwiki/model.py`:mapping 外部开关; +- `deerflow/persistence/llmwiki/{base,memory,sql}.py`:读写新开关; +- `deerflow/persistence/llmwiki_index/*`:新增页面、向量、同步状态 store; +- Alembic migration:新增表与字段; +- `deerflow/persistence/models/__init__.py`:注册模型。 + +### 同步与向量 + +- `deerflow/integrations/weknora/client.py`:确保 Wiki list/get 字段完整,保留 `chunk_refs`; +- `deerflow/integrations/weknora/local_index/normalize.py`:规范化; +- `.../chunking.py`:Wiki 专用 chunks; +- `.../embedding.py`:严格 Embedding; +- `.../vector_codec.py`:BLOB; +- `.../sync.py`:同步、租约、原子换代; +- `.../vector_cache.py`:NumPy 快照; +- `.../search.py`:权限后的本地向量检索; +- `app/gateway/llmwiki_index_scheduler.py`:周期任务; +- `app/gateway/app.py`:初始化/关闭。 + +### 路由与业务接入 + +- `app/gateway/routers/llmwiki_index.py`:内部搜索/状态/同步/重建; +- `app/gateway/routers/external_llmwiki.py`:外部搜索/详情; +- `app/gateway/auth_middleware.py`:外部 API Key 分支; +- `app/gateway/llmwiki_rag.py`:预检索切换; +- `deerflow/tools/builtins/llmwiki_search_tool.py`:工具切换; +- `app/gateway/routers/llmwiki.py`:页面搜索、详情和外部开关接入; +- `deerflow/runtime/references.py`:保留 Wiki 结构字段; +- deep research / AI writing:回归兼容。 + +### 前端 + +- `frontend-web/src/core/llmwiki/source-links.ts`; +- `frontend-web/src/components/workspace/skill-display-sources.tsx`; +- `frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx`; +- `frontend-web/src/strategy-components/api/llmwiki.ts`; +- 知识库详情/系统设置中的索引状态和外部开关(如纳入首版 UI)。 + +### 测试 + +- 新增 `tests/test_llmwiki_local_index_repository.py`; +- 新增 `tests/test_llmwiki_wiki_sync.py`; +- 新增 `tests/test_llmwiki_vector_search.py`; +- 新增 `tests/test_external_llmwiki_search.py`; +- 扩展 `tests/test_llmwiki_weknora.py`; +- 前端增加 source target / drawer 测试; +- 增加性能基准脚本,不放入默认每次单测。 + +--- + +## 28. 实现前必须确认的部署输入 + +编码本身可以先做,但上线前必须明确: + +1. DeerFlow 可调用的 OpenAI-compatible Embedding 地址; +2. 模型名称; +3. 向量维度; +4. 最大 Wiki 页面数、平均页面长度、预计章节数; +5. Gateway worker 数; +6. 生产数据库是 SQLite、PostgreSQL 还是 MySQL; +7. 外部接口是否只给内网系统,是否有统一 API Gateway; +8. 外部 API Key 的生成、分发和轮换责任; +9. 哪些现有公共库允许外部开放; +10. 同步允许的最大延迟,例如 1 分钟或 5 分钟。 + +若没有额外输入,开发默认使用: + +- 本地/测试 SQLite; +- bge-m3 类 1024 维模型(实际以配置为准); +- 120 秒轮询; +- NumPy exact; +- 外部 API 默认关闭; +- strict vector; +- 每页最多 2 个命中章节; +- Top 8 页面。 + +--- + +## 29. 最终推荐 + +本需求不应被实现成“把 WeKnora 的搜索接口换个 URL”,也不应通过隐藏文档库重复导入来绕过同步问题。推荐把它建设为 DeerFlow 内部的正式 Wiki 检索子系统: + +1. WeKnora 负责生成; +2. DeerFlow 完整镜像; +3. DeerFlow 按章节向量化; +4. DeerFlow 本地持久化; +5. DeerFlow 统一检索和权限; +6. 引用始终落到 Wiki 页面; +7. 外部接口只查询明确开放的公共 Wiki; +8. 未来只替换向量执行器即可扩容,不推翻页面、权限、API 和引用设计。 + +该方案能同时满足:不修改 WeKnora 后端、检索处理后的 Wiki、右侧展示 Wiki、向量语义召回、公共 Wiki 外部检索、WeKnora 查询故障隔离,以及后续向 pgvector/分布式向量引擎平滑演进。 + diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/fixed_answer_agent.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/fixed_answer_agent.py new file mode 100644 index 00000000..222f9be8 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/fixed_answer_agent.py @@ -0,0 +1,186 @@ +"""Small local streaming graph used for exact-match fixed answers. + +The graph deliberately uses a local ``BaseChatModel`` implementation so +LangGraph emits the same token/message stream events as a normal model-backed +answer, while making no network or model-provider request. +""" + +from __future__ import annotations + +import asyncio +import codecs +import math +import uuid +from collections.abc import AsyncIterator, Sequence +from typing import Any + +import tiktoken +from langchain_core.callbacks import AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun +from langchain_core.language_models.chat_models import BaseChatModel +from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage +from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult +from langchain_core.runnables import RunnableConfig +from langgraph.graph import END, START, StateGraph + +from deerflow.agents.thread_state import ThreadState + +_TOKEN_ENCODING = tiktoken.get_encoding("cl100k_base") +_TARGET_CHUNKS_PER_SECOND = 10 +_MAX_TOKENS_PER_CHUNK = 20 +_MAX_TOKENS_PER_SECOND = 1_000 + + +class FixedAnswerChatModel(BaseChatModel): + """A local streaming model that only replays an administrator answer.""" + + answer: str + message_id: str + fixed_question_id: str + tokens_per_second: int = 100 + + @property + def _llm_type(self) -> str: + return "fixed-answer" + + @property + def _identifying_params(self) -> dict[str, Any]: + return { + "fixed_question_id": self.fixed_question_id, + "tokens_per_second": self.tokens_per_second, + } + + def _message(self) -> AIMessage: + return AIMessage( + content=self.answer, + id=self.message_id, + additional_kwargs={ + "fixed_answer": True, + "fixed_question_id": self.fixed_question_id, + "tokens_per_second": self.tokens_per_second, + }, + ) + + def _generate( + self, + messages: list[BaseMessage], + stop: list[str] | None = None, + run_manager: CallbackManagerForLLMRun | None = None, + **kwargs: Any, + ) -> ChatResult: + del messages, stop, run_manager, kwargs + return ChatResult(generations=[ChatGeneration(message=self._message())]) + + async def _astream( + self, + messages: list[BaseMessage], + stop: list[str] | None = None, + run_manager: AsyncCallbackManagerForLLMRun | None = None, + **kwargs: Any, + ) -> AsyncIterator[ChatGenerationChunk]: + del messages, stop, run_manager, kwargs + answer = self.answer + if not answer: + yield ChatGenerationChunk(message=AIMessageChunk(content="", id=self.message_id)) + return + token_ids = _TOKEN_ENCODING.encode(answer, disallowed_special=()) + tokens_per_second = max( + 1, + min(_MAX_TOKENS_PER_SECOND, self.tokens_per_second), + ) + chunk_size = max( + 1, + min( + _MAX_TOKENS_PER_CHUNK, + math.ceil(tokens_per_second / _TARGET_CHUNKS_PER_SECOND), + ), + ) + decoder = codecs.getincrementaldecoder("utf-8")() + token_groups = [token_ids[:1], *(token_ids[offset : offset + chunk_size] for offset in range(1, len(token_ids), chunk_size))] + for index, chunk_token_ids in enumerate(token_groups): + if index > 0: + await asyncio.sleep(len(chunk_token_ids) / tokens_per_second) + chunk_bytes = b"".join(_TOKEN_ENCODING.decode_single_token_bytes(token_id) for token_id in chunk_token_ids) + is_last = index == len(token_groups) - 1 + content = decoder.decode(chunk_bytes, final=is_last) + chunk_kwargs = ( + { + "fixed_answer": True, + "fixed_question_id": self.fixed_question_id, + "tokens_per_second": tokens_per_second, + } + if index == 0 + else {} + ) + yield ChatGenerationChunk( + message=AIMessageChunk( + content=content, + id=self.message_id, + additional_kwargs=chunk_kwargs, + ) + ) + + +def _fallback_title(question: str) -> str: + if len(question) > 50: + return question[:50].rstrip() + "..." + return question or "New Conversation" + + +def make_fixed_answer_agent( + *, + question: str, + answer: str, + fixed_question_id: str, + tokens_per_second: int = 100, +): + """Build a one-node graph that streams and persists a fixed answer.""" + + tokens_per_second = max( + 1, + min(_MAX_TOKENS_PER_SECOND, int(tokens_per_second)), + ) + message_id = f"fixed-answer-{uuid.uuid4()}" + model = FixedAnswerChatModel( + answer=answer, + message_id=message_id, + fixed_question_id=fixed_question_id, + tokens_per_second=tokens_per_second, + ).with_config(tags=["fixed-answer"]) + + async def respond( + state: ThreadState, + config: RunnableConfig, + ) -> dict[str, Any]: + messages: Sequence[BaseMessage] = state.get("messages", []) + # Calling the local model through ``astream`` makes the runtime publish + # normal messages-mode chunks. The full AIMessage returned below is the + # durable checkpoint value used on refresh/history reads. + async for _chunk in model.astream(list(messages), config=config): + pass + update: dict[str, Any] = { + "messages": [ + AIMessage( + content=answer, + id=message_id, + additional_kwargs={ + "fixed_answer": True, + "fixed_question_id": fixed_question_id, + "tokens_per_second": tokens_per_second, + }, + ) + ] + } + if not state.get("title"): + update["title"] = _fallback_title(question) + update["title_provisional"] = False + elif state.get("title_provisional") is True: + # A fixed-answer run must not trigger the worker's background LLM + # title finalizer either; keep the already-visible local title. + update["title_provisional"] = False + return update + + graph = StateGraph(ThreadState) + graph.add_node("fixed_answer", respond) + graph.add_edge(START, "fixed_answer") + graph.add_edge("fixed_answer", END) + return graph.compile() diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/__init__.py new file mode 100644 index 00000000..b4f37e4f --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/__init__.py @@ -0,0 +1,22 @@ +"""Exact-match fixed-question persistence.""" + +from __future__ import annotations + +from deerflow.persistence.fixed_questions.base import FixedQuestionStore +from deerflow.persistence.fixed_questions.memory import MemoryFixedQuestionStore +from deerflow.persistence.fixed_questions.model import FixedQuestionRow +from deerflow.persistence.fixed_questions.sql import FixedQuestionRepository + +__all__ = [ + "FixedQuestionRepository", + "FixedQuestionRow", + "FixedQuestionStore", + "MemoryFixedQuestionStore", + "make_fixed_question_store", +] + + +def make_fixed_question_store(session_factory): + if session_factory is None: + return MemoryFixedQuestionStore() + return FixedQuestionRepository(session_factory) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/base.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/base.py new file mode 100644 index 00000000..2bcd0bb0 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/base.py @@ -0,0 +1,30 @@ +"""Abstract persistence API for administrator-managed fixed questions.""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any + + +class FixedQuestionStore(ABC): + """Global fixed-question configuration shared by every account.""" + + @abstractmethod + async def list_questions(self) -> list[dict[str, Any]]: + """Return every configured question in display order.""" + raise NotImplementedError + + @abstractmethod + async def find_enabled(self, question: str) -> dict[str, Any] | None: + """Return the enabled row whose question exactly equals ``question``.""" + raise NotImplementedError + + @abstractmethod + async def replace_all( + self, + rows: list[dict[str, Any]], + *, + updated_by: str | None = None, + ) -> list[dict[str, Any]]: + """Atomically replace the complete fixed-question configuration.""" + raise NotImplementedError diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/memory.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/memory.py new file mode 100644 index 00000000..4c128efa --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/memory.py @@ -0,0 +1,56 @@ +"""In-memory fixed-question store used by memory deployments and tests.""" + +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any + +from deerflow.persistence.fixed_questions.base import FixedQuestionStore + + +def _normalize(data: dict[str, Any], updated_by: str | None) -> dict[str, Any]: + return { + "id": str(data["id"]), + "question": str(data.get("question") or ""), + "answer": str(data.get("answer") or ""), + "enabled": bool(data.get("enabled", True)), + "tokens_per_second": int(data.get("tokens_per_second", 100)), + "sort_order": int(data.get("sort_order", 0)), + "updated_by": updated_by, + "updated_at": datetime.now(UTC).isoformat(), + } + + +class MemoryFixedQuestionStore(FixedQuestionStore): + def __init__(self) -> None: + self._rows: dict[str, dict[str, Any]] = {} + + async def list_questions(self) -> list[dict[str, Any]]: + rows = sorted( + self._rows.values(), + key=lambda row: (row.get("sort_order", 0), row.get("question", "")), + ) + return [dict(row) for row in rows] + + async def find_enabled(self, question: str) -> dict[str, Any] | None: + for row in self._rows.values(): + if row["enabled"] and row["question"] == question: + return dict(row) + return None + + async def replace_all( + self, + rows: list[dict[str, Any]], + *, + updated_by: str | None = None, + ) -> list[dict[str, Any]]: + # Last duplicate question wins, mirroring the SQL repository. + by_id: dict[str, dict[str, Any]] = {} + for row in rows: + question = row.get("question") + row_id = row.get("id") + if row_id and question not in (None, ""): + by_id[str(row_id)] = row + by_question = {str(row["question"]): row for row in by_id.values()} + self._rows = {str(row["id"]): _normalize(row, updated_by) for row in by_question.values()} + return await self.list_questions() diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/model.py new file mode 100644 index 00000000..265b5e3e --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/model.py @@ -0,0 +1,41 @@ +"""ORM model for exact-match, administrator-managed fixed questions.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from deerflow.persistence.base import Base +from deerflow.persistence.types import BeijingDateTime, PortableLongText + + +class FixedQuestionRow(Base): + """One globally shared question/answer pair. + + ``question_hash`` provides a portable indexed lookup for questions stored in + ``Text``. The repository always compares the original question as well, so a + theoretical SHA-256 collision cannot cause a false match. + """ + + __tablename__ = "fixed_questions" + __table_args__ = (UniqueConstraint("question_hash", name="uq_fixed_questions_question_hash"),) + + id: Mapped[str] = mapped_column(String(64), primary_key=True) + question: Mapped[str] = mapped_column(Text, nullable=False) + question_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + answer: Mapped[str] = mapped_column(PortableLongText, nullable=False) + enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="1") + tokens_per_second: Mapped[int] = mapped_column( + Integer, + default=100, + server_default="100", + ) + sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0") + updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True) + updated_at: Mapped[datetime] = mapped_column( + BeijingDateTime(), + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + ) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/sql.py new file mode 100644 index 00000000..5d8098d5 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/sql.py @@ -0,0 +1,88 @@ +"""SQLAlchemy-backed fixed-question store.""" + +from __future__ import annotations + +import hashlib +from datetime import datetime +from typing import Any + +from sqlalchemy import delete, select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from deerflow.persistence.fixed_questions.base import FixedQuestionStore +from deerflow.persistence.fixed_questions.model import FixedQuestionRow + + +def question_hash(question: str) -> str: + return hashlib.sha256(question.encode("utf-8")).hexdigest() + + +def _to_dict(row: FixedQuestionRow) -> dict[str, Any]: + data = row.to_dict() + data.pop("question_hash", None) + value = data.get("updated_at") + if isinstance(value, datetime): + data["updated_at"] = value.isoformat() + return data + + +def _row_from(data: dict[str, Any], updated_by: str | None) -> FixedQuestionRow: + question = str(data.get("question") or "") + return FixedQuestionRow( + id=str(data["id"]), + question=question, + question_hash=question_hash(question), + answer=str(data.get("answer") or ""), + enabled=bool(data.get("enabled", True)), + tokens_per_second=int(data.get("tokens_per_second", 100)), + sort_order=int(data.get("sort_order", 0)), + updated_by=updated_by, + ) + + +class FixedQuestionRepository(FixedQuestionStore): + def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: + self._sf = session_factory + + async def list_questions(self) -> list[dict[str, Any]]: + stmt = select(FixedQuestionRow).order_by( + FixedQuestionRow.sort_order.asc(), + FixedQuestionRow.question.asc(), + ) + async with self._sf() as session: + result = await session.execute(stmt) + return [_to_dict(row) for row in result.scalars()] + + async def find_enabled(self, question: str) -> dict[str, Any] | None: + stmt = select(FixedQuestionRow).where( + FixedQuestionRow.question_hash == question_hash(question), + FixedQuestionRow.enabled.is_(True), + ) + async with self._sf() as session: + row = (await session.execute(stmt)).scalar_one_or_none() + if row is None or row.question != question: + return None + return _to_dict(row) + + async def replace_all( + self, + rows: list[dict[str, Any]], + *, + updated_by: str | None = None, + ) -> list[dict[str, Any]]: + # Last duplicate question wins, preventing a unique-hash violation even + # if a non-UI caller submits malformed data. + by_id: dict[str, dict[str, Any]] = {} + for row in rows: + question = row.get("question") + row_id = row.get("id") + if row_id and question not in (None, ""): + by_id[str(row_id)] = row + by_question = {str(row["question"]): row for row in by_id.values()} + + async with self._sf() as session: + await session.execute(delete(FixedQuestionRow)) + new_rows = [_row_from(row, updated_by) for row in by_question.values()] + session.add_all(new_rows) + await session.commit() + return [_to_dict(row) for row in new_rows] diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_01_fixed_questions.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_01_fixed_questions.py new file mode 100644 index 00000000..dfeca77f --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_01_fixed_questions.py @@ -0,0 +1,51 @@ +"""Create administrator-managed exact-match fixed questions. + +Revision ID: 20260821_01 +Revises: 20260816_01 +Create Date: 2026-08-21 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import mysql + +revision = "20260821_01" +down_revision = "20260816_01" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + bind = op.get_bind() + if sa.inspect(bind).has_table("fixed_questions"): + return + long_text = sa.Text().with_variant(mysql.LONGTEXT(), "mysql") + op.create_table( + "fixed_questions", + sa.Column("id", sa.String(length=64), primary_key=True), + sa.Column("question", sa.Text(), nullable=False), + sa.Column("question_hash", sa.String(length=64), nullable=False), + sa.Column("answer", long_text, nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"), + sa.Column("updated_by", sa.String(length=64), nullable=True), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + sa.UniqueConstraint( + "question_hash", + name="uq_fixed_questions_question_hash", + ), + ) + op.create_index( + "ix_fixed_questions_question_hash", + "fixed_questions", + ["question_hash"], + unique=False, + ) + + +def downgrade() -> None: + bind = op.get_bind() + if sa.inspect(bind).has_table("fixed_questions"): + op.drop_table("fixed_questions") diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_02_fixed_question_stream_speed.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_02_fixed_question_stream_speed.py new file mode 100644 index 00000000..b26747b7 --- /dev/null +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_02_fixed_question_stream_speed.py @@ -0,0 +1,44 @@ +"""Add configurable fixed-answer streaming speed. + +Revision ID: 20260821_02 +Revises: 20260821_01 +Create Date: 2026-08-21 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "20260821_02" +down_revision = "20260821_01" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if not inspector.has_table("fixed_questions"): + return + columns = {column["name"] for column in inspector.get_columns("fixed_questions")} + if "tokens_per_second" not in columns: + op.add_column( + "fixed_questions", + sa.Column( + "tokens_per_second", + sa.Integer(), + nullable=False, + server_default="100", + ), + ) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if not inspector.has_table("fixed_questions"): + return + columns = {column["name"] for column in inspector.get_columns("fixed_questions")} + if "tokens_per_second" in columns: + op.drop_column("fixed_questions", "tokens_per_second") diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py index 175512e3..acff420c 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py @@ -47,6 +47,7 @@ from deerflow.persistence.deep_research_sources.model import DeepResearchSourceR from deerflow.persistence.document_rewrite_versions.model import DocumentRewriteVersionRow from deerflow.persistence.embed_sessions.model import EmbedSessionRow from deerflow.persistence.feedback.model import FeedbackRow +from deerflow.persistence.fixed_questions.model import FixedQuestionRow from deerflow.persistence.intent_turns.model import IntentTurnRow from deerflow.persistence.light_apps.model import LightAppRow from deerflow.persistence.llm_metrics.model import LlmCallMetricRow @@ -97,6 +98,7 @@ from deerflow.persistence.user_prompts.model import UserCustomPromptRow __all__ = [ "FeedbackRow", + "FixedQuestionRow", "AgentExtrasRow", "AgentFavoriteRow", "AgentRow", diff --git a/offline-backend-20260512/backend/tests/test_fixed_questions.py b/offline-backend-20260512/backend/tests/test_fixed_questions.py new file mode 100644 index 00000000..04a7af2b --- /dev/null +++ b/offline-backend-20260512/backend/tests/test_fixed_questions.py @@ -0,0 +1,288 @@ +"""Regression tests for administrator-managed exact-match fixed answers.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +from langchain_core.messages import HumanMessage +from pydantic import ValidationError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.gateway.routers.fixed_questions import ( + FixedQuestion, + FixedQuestionReplaceRequest, +) +from app.gateway.services import ( + RUN_SCOPE_EXTERNAL_ROOT, + RUN_SCOPE_ORCHESTRATION_CHILD, + _find_fixed_answer, + normalize_input, +) +from deerflow.agents import fixed_answer_agent as fixed_answer_module +from deerflow.agents.fixed_answer_agent import make_fixed_answer_agent +from deerflow.persistence.fixed_questions import MemoryFixedQuestionStore +from deerflow.persistence.fixed_questions.model import FixedQuestionRow +from deerflow.persistence.fixed_questions.sql import FixedQuestionRepository + + +def _body(**overrides): + data = { + "assistant_id": "lead_agent", + "command": None, + "context": {}, + } + data.update(overrides) + return SimpleNamespace(**data) + + +def _request(store: MemoryFixedQuestionStore): + return SimpleNamespace( + app=SimpleNamespace( + state=SimpleNamespace(fixed_question_store=store), + ) + ) + + +@pytest.mark.asyncio +async def test_memory_store_matches_only_enabled_exact_question() -> None: + store = MemoryFixedQuestionStore() + await store.replace_all( + [ + { + "id": "enabled", + "question": "系统开放时间?", + "answer": "工作日 09:00—18:00。", + "enabled": True, + "tokens_per_second": 80, + "sort_order": 0, + }, + { + "id": "disabled", + "question": "停用问题", + "answer": "不应返回", + "enabled": False, + "sort_order": 1, + }, + ] + ) + + match = await store.find_enabled("系统开放时间?") + assert match is not None + assert match["answer"] == "工作日 09:00—18:00。" + assert match["tokens_per_second"] == 80 + assert await store.find_enabled("系统开放时间?") is None + assert await store.find_enabled(" 系统开放时间?") is None + assert await store.find_enabled("停用问题") is None + + +@pytest.mark.asyncio +async def test_sql_store_round_trips_and_exact_matches() -> None: + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + try: + async with engine.begin() as connection: + await connection.run_sync(FixedQuestionRow.__table__.create) + repository = FixedQuestionRepository(async_sessionmaker(engine, expire_on_commit=False)) + saved = await repository.replace_all( + [ + { + "id": "sql-q1", + "question": "SQL 精确问题", + "answer": "SQL 固定回答", + "enabled": True, + "tokens_per_second": 120, + "sort_order": 0, + } + ], + updated_by="admin", + ) + assert saved[0]["updated_by"] == "admin" + assert saved[0]["tokens_per_second"] == 120 + assert (await repository.find_enabled("SQL 精确问题"))["answer"] == "SQL 固定回答" + assert await repository.find_enabled("sql 精确问题") is None + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_lookup_skips_attachments_and_internal_runs() -> None: + store = MemoryFixedQuestionStore() + await store.replace_all( + [ + { + "id": "q1", + "question": "固定问题", + "answer": "固定回答", + "enabled": True, + "sort_order": 0, + } + ] + ) + request = _request(store) + + plain_input = normalize_input({"messages": [{"content": "固定问题"}]}) + match = await _find_fixed_answer( + body=_body(), + request=request, + graph_input=plain_input, + run_scope=RUN_SCOPE_EXTERNAL_ROOT, + ) + assert match is not None + assert match["id"] == "q1" + + attachment_input = normalize_input( + { + "messages": [ + { + "content": "固定问题", + "additional_kwargs": {"files": [{"name": "context.pdf"}]}, + } + ] + } + ) + assert ( + await _find_fixed_answer( + body=_body(), + request=request, + graph_input=attachment_input, + run_scope=RUN_SCOPE_EXTERNAL_ROOT, + ) + is None + ) + contextual_input = normalize_input( + { + "messages": [ + { + "content": "固定问题", + "additional_kwargs": {"prompt_prefix": "网页正文上下文"}, + } + ] + } + ) + assert ( + await _find_fixed_answer( + body=_body(), + request=request, + graph_input=contextual_input, + run_scope=RUN_SCOPE_EXTERNAL_ROOT, + ) + is None + ) + assert ( + await _find_fixed_answer( + body=_body(), + request=request, + graph_input=plain_input, + run_scope=RUN_SCOPE_ORCHESTRATION_CHILD, + ) + is None + ) + + +@pytest.mark.asyncio +async def test_fixed_answer_agent_streams_chunks_and_persists_full_answer() -> None: + question = "固定问题" + answer = "这是一个足够长的固定回答,用于验证返回内容会拆成多个流式消息块,同时完整写入会话状态。" + agent = make_fixed_answer_agent( + question=question, + answer=answer, + fixed_question_id="q1", + tokens_per_second=1000, + ) + + chunks: list[str] = [] + final_state = None + async for mode, data in agent.astream( + {"messages": [HumanMessage(content=question)]}, + stream_mode=["values", "messages"], + ): + if mode == "messages": + message, _metadata = data + if isinstance(message.content, str): + chunks.append(message.content) + elif mode == "values": + final_state = data + + assert "".join(chunks) == answer + assert len([chunk for chunk in chunks if chunk]) >= 2 + assert final_state is not None + assert final_state["messages"][-1].content == answer + assert final_state["messages"][-1].additional_kwargs["fixed_answer"] is True + assert final_state["messages"][-1].additional_kwargs["tokens_per_second"] == 1000 + assert final_state["title"] == question + assert final_state["title_provisional"] is False + + +@pytest.mark.asyncio +async def test_fixed_answer_default_speed_schedules_100_tokens_per_second( + monkeypatch: pytest.MonkeyPatch, +) -> None: + answer = "one two three four five six seven eight nine ten eleven twelve thirteen" + sleeps: list[float] = [] + + async def record_sleep(delay: float) -> None: + sleeps.append(delay) + + monkeypatch.setattr(fixed_answer_module.asyncio, "sleep", record_sleep) + model = fixed_answer_module.FixedAnswerChatModel( + answer=answer, + message_id="speed-test", + fixed_question_id="q-speed", + ) + + chunks: list[str] = [] + async for generation in model._astream([]): + chunks.append(str(generation.message.content)) + + token_count = len(fixed_answer_module._TOKEN_ENCODING.encode(answer)) + assert "".join(chunks) == answer + assert model.tokens_per_second == 100 + assert sum(sleeps) == pytest.approx((token_count - 1) / 100) + + +def test_replace_request_rejects_duplicate_exact_questions() -> None: + row = { + "question": "重复问题", + "answer": "回答", + "enabled": True, + "tokens_per_second": 100, + "sort_order": 0, + } + with pytest.raises(ValidationError): + FixedQuestionReplaceRequest( + questions=[ + FixedQuestion(id="q1", **row), + FixedQuestion(id="q2", **row), + ] + ) + with pytest.raises(ValidationError): + FixedQuestionReplaceRequest( + questions=[ + FixedQuestion(id="same-id", **row), + FixedQuestion( + id="same-id", + question="另一个问题", + answer="另一个回答", + enabled=True, + sort_order=1, + ), + ] + ) + + +def test_fixed_question_stream_speed_defaults_and_bounds() -> None: + default_row = FixedQuestion( + id="default-speed", + question="默认速度", + answer="回答", + ) + assert default_row.tokens_per_second == 100 + + for invalid_speed in (0, 1001): + with pytest.raises(ValidationError): + FixedQuestion( + id=f"invalid-{invalid_speed}", + question="非法速度", + answer="回答", + tokens_per_second=invalid_speed, + ) diff --git a/offline-backend-20260512/backend/tests/test_health_auth_bypass.py b/offline-backend-20260512/backend/tests/test_health_auth_bypass.py index f027799c..9428949d 100644 --- a/offline-backend-20260512/backend/tests/test_health_auth_bypass.py +++ b/offline-backend-20260512/backend/tests/test_health_auth_bypass.py @@ -2,12 +2,15 @@ from __future__ import annotations +from types import SimpleNamespace + import pytest from fastapi import FastAPI from fastapi.testclient import TestClient from app.gateway import auth_middleware from app.gateway.app import app as gateway_app +from app.gateway.auth.token_source import get_explicit_or_session_token from app.gateway.auth_middleware import PUBLIC_HEALTH_PATHS, AuthMiddleware @@ -98,3 +101,28 @@ def test_non_health_route_still_requires_auth(monkeypatch: pytest.MonkeyPatch) - response = client.get("/private") assert response.status_code == 401 + + +def _auth_request(*, authorization: str = "", cookie: str = "") -> SimpleNamespace: + headers = {"authorization": authorization} if authorization else {} + cookies = {"access_token": cookie} if cookie else {} + return SimpleNamespace(headers=headers, cookies=cookies) + + +def test_explicit_bearer_token_wins_over_shared_cookie() -> None: + """Another tab's cookie must not override this tab's explicit token.""" + request = _auth_request(authorization="Bearer current-tab", cookie="other-tab") + + assert get_explicit_or_session_token(request) == "current-tab" + + +def test_cookie_remains_the_fallback_for_cookie_only_clients() -> None: + request = _auth_request(cookie="browser-session") + + assert get_explicit_or_session_token(request) == "browser-session" + + +def test_malformed_authorization_header_does_not_hide_valid_cookie() -> None: + request = _auth_request(authorization="Basic abc", cookie="browser-session") + + assert get_explicit_or_session_token(request) == "browser-session" diff --git a/offline-backend-20260512/backend/tests/test_username_login_domain.py b/offline-backend-20260512/backend/tests/test_username_login_domain.py index 85fa26c5..f2f5cd3f 100644 --- a/offline-backend-20260512/backend/tests/test_username_login_domain.py +++ b/offline-backend-20260512/backend/tests/test_username_login_domain.py @@ -9,15 +9,18 @@ Verifies that: from __future__ import annotations -import os from datetime import UTC, datetime -from uuid import UUID, uuid4 +from uuid import uuid4 import pytest +from app.gateway.auth.local_provider import LocalAuthProvider from app.gateway.auth.models import User from app.gateway.auth.repositories.base import UserRepository +from app.gateway.routers import auth as auth_module from app.gateway.routers.auth import ( + _get_or_create_user_by_username, + _lookup_user_by_username, _normalize_username_local_part, _username_to_email, ) @@ -74,6 +77,9 @@ class FakeUserRepository(UserRepository): async def count_admin_users(self) -> int: return sum(1 for u in self.rows.values() if u.system_role == "admin") + async def list_users(self) -> list[User]: + return sorted(self.rows.values(), key=lambda user: user.created_at) + async def get_user_by_oauth(self, provider: str, oauth_id: str) -> User | None: return None @@ -146,3 +152,38 @@ async def test_local_part_lookup_misses_unknown_username() -> None: repo = FakeUserRepository() _seed(repo, "alice@cm.com") assert await repo.find_user_by_email_local_part("bob") is None + + +@pytest.mark.asyncio +async def test_username_login_uses_oldest_account_when_canonical_duplicate_exists( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A domain migration must not make one username alternate user ids.""" + repo = FakeUserRepository() + older = _seed(repo, "alice@local.deerflow") + older.created_at = datetime(2024, 1, 1, tzinfo=UTC) + newer = _seed(repo, "alice@cm.com") + newer.created_at = datetime(2026, 1, 1, tzinfo=UTC) + monkeypatch.setattr(auth_module, "get_local_provider", lambda: LocalAuthProvider(repo)) + + user, created = await _get_or_create_user_by_username("alice") + + assert created is False + assert user.id == older.id + + +@pytest.mark.asyncio +async def test_password_lookup_uses_same_stable_username_account( + monkeypatch: pytest.MonkeyPatch, +) -> None: + repo = FakeUserRepository() + older = _seed(repo, "alice@local.deerflow") + older.created_at = datetime(2024, 1, 1, tzinfo=UTC) + newer = _seed(repo, "alice@cm.com") + newer.created_at = datetime(2026, 1, 1, tzinfo=UTC) + monkeypatch.setattr(auth_module, "get_local_provider", lambda: LocalAuthProvider(repo)) + + user = await _lookup_user_by_username("alice") + + assert user is not None + assert user.id == older.id
+ 为完全相同的问题配置固定回答,命中时直接流式返回,不调用大模型。 +