diff --git a/.gitignore b/.gitignore
index 7ae1b28c..1c2eca88 100644
--- a/.gitignore
+++ b/.gitignore
@@ -87,24 +87,35 @@ offline-backend-20260512/backend/skill_evolution.json
skills/
hermes-agent-main/
.gitignore
-offline-backend-20260512/backend/tests/*
-!offline-backend-20260512/backend/tests/test_llmwiki_weknora.py
-!offline-backend-20260512/backend/tests/test_health_auth_bypass.py
-# Keep the model configuration regression coverage versioned even though
-# this deployment repository otherwise ignores newly added backend tests.
-!offline-backend-20260512/backend/tests/test_model_management.py
-# Deep-research reliability regression coverage.
-!offline-backend-20260512/backend/tests/test_deep_research_persistence.py
-!offline-backend-20260512/backend/tests/test_deep_research_event_sink.py
+offline-backend-20260512/backend/tests/*
+!offline-backend-20260512/backend/tests/test_llmwiki_weknora.py
+!offline-backend-20260512/backend/tests/test_health_auth_bypass.py
+# Keep the model configuration regression coverage versioned even though
+# this deployment repository otherwise ignores newly added backend tests.
+!offline-backend-20260512/backend/tests/test_model_management.py
+# Keep fixed-question exact-match and streaming regression coverage versioned.
+!offline-backend-20260512/backend/tests/test_fixed_questions.py
+# Keep configurable es_query system-prompt routing regression coverage versioned.
+!offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py
+# Keep the local LLMWiki vector-index and external API regression coverage versioned.
+!offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py
+!offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py
+!offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py
+!offline-backend-20260512/backend/tests/test_external_llmwiki_search.py
+!offline-backend-20260512/backend/tests/test_llmwiki_wiki_retrieval.py
+!offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py
+# Deep-research reliability regression coverage.
+!offline-backend-20260512/backend/tests/test_deep_research_persistence.py
+!offline-backend-20260512/backend/tests/test_deep_research_event_sink.py
!offline-backend-20260512/backend/tests/test_deep_research_executor_reliability.py
!offline-backend-20260512/backend/tests/test_deep_research_start_job.py
!offline-backend-20260512/backend/tests/test_summarization_human_message.py
!offline-backend-20260512/backend/tests/test_compact_menu_overrides.py
# Independent enterprise-research persistence, HTTP and race regressions.
!offline-backend-20260512/backend/tests/test_enterprise_research_tasks.py
-# Position-roundtable per-turn write/present cap (weak-model duplicate artifacts).
-!offline-backend-20260512/backend/tests/test_position_artifact_cap.py
-deploy
+# Position-roundtable per-turn write/present cap (weak-model duplicate artifacts).
+!offline-backend-20260512/backend/tests/test_position_artifact_cap.py
+deploy
.history
# Cloned third-party repos (local only)
diff --git a/frontend-web/src/App.tsx b/frontend-web/src/App.tsx
index 2833dea3..7cdb7fa7 100644
--- a/frontend-web/src/App.tsx
+++ b/frontend-web/src/App.tsx
@@ -38,6 +38,7 @@ import EmbedChatPage from "./pages/EmbedChatPage";
import EmbedRoundtablePage from "./pages/EmbedRoundtablePage";
import DashboardStandalonePage from "@/roundtable-planning/pages/DashboardStandalonePage";
import PositionRoundtablePage from "@/position-roundtable/pages/PositionRoundtablePage";
+import WeKnoraWikiPageView from "@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView";
import {
isPositionRoundtableStandaloneMode,
POSITION_ROUNDTABLE_ROUTE,
@@ -45,7 +46,12 @@ import {
import { ChatRuntime } from "./pages/WorkspaceRoutes";
import { NotFoundPage } from "./pages/NotFoundPage";
-const PUBLIC_EMBED_PREFIXES = ["/share/", "/public/", "/roundtable-share/"];
+const PUBLIC_EMBED_PREFIXES = [
+ "/share/",
+ "/public/",
+ "/roundtable-share/",
+ "/embed/knowledge/",
+];
function AppProviders({ children }: { children: React.ReactNode }) {
const locale = useMemo(() => detectLocale(), []);
@@ -102,16 +108,17 @@ function AppProviders({ children }: { children: React.ReactNode }) {
forcedTheme={forcedTheme}
>
-
-
-
- {!positionStandaloneMode && }
- {!positionStandaloneMode && }
-
- {children}
-
-
-
+
+
+
+
+ {!positionStandaloneMode && }
+
+ {children}
+
+
+
+
);
@@ -188,6 +195,10 @@ export function App() {
}
/>
} />
+ }
+ />
}
diff --git a/frontend-web/src/components/page-sidebar.tsx b/frontend-web/src/components/page-sidebar.tsx
index 5119a2ba..dd85aeeb 100644
--- a/frontend-web/src/components/page-sidebar.tsx
+++ b/frontend-web/src/components/page-sidebar.tsx
@@ -10,6 +10,7 @@ import {
FolderOpen,
Gauge,
ListTree,
+ MessageSquareText,
MousePointerClick,
BriefcaseBusiness,
Settings,
@@ -237,6 +238,14 @@ function SecondaryNavSidebar({ headerHidden }: { headerHidden: boolean }) {
按钮管理
)}
+ {isAdmin && (
+ navigate("/page/strategy/admin/fixed-questions")}
+ >
+
+ 问题配置
+
+ )}
{isAdmin && (
navigate("/page/strategy/admin/position-roles")}
@@ -558,6 +567,12 @@ function LegacyAccordionSidebar({
按钮管理
)}
+ {isAdmin && (
+ navigate("/page/strategy/admin/fixed-questions")}>
+
+ 问题配置
+
+ )}
{isAdmin && (
navigate("/page/strategy/admin/position-roles")}>
diff --git a/frontend-web/src/components/workspace/artifacts/artifact-directory.tsx b/frontend-web/src/components/workspace/artifacts/artifact-directory.tsx
new file mode 100644
index 00000000..8956d653
--- /dev/null
+++ b/frontend-web/src/components/workspace/artifacts/artifact-directory.tsx
@@ -0,0 +1,226 @@
+import {
+ ChevronDownIcon,
+ ChevronRightIcon,
+ FolderIcon,
+ FolderOpenIcon,
+ RefreshCwIcon,
+} from "lucide-react";
+import { useMemo, useState } from "react";
+
+import { Button } from "@/components/ui/button";
+import { artifactPathname, artifactRelativePath } from "@/core/artifacts/listing";
+import { getFileIcon, getFileName } from "@/core/utils/files";
+import { cn } from "@/lib/utils";
+
+type DirectoryNode = {
+ name: string;
+ path: string;
+ directories: DirectoryNode[];
+ files: Array<{ artifact: string; name: string; relativePath: string }>;
+};
+
+function buildDirectoryTree(artifacts: string[]): DirectoryNode {
+ const root: DirectoryNode = {
+ name: "生成文件",
+ path: "",
+ directories: [],
+ files: [],
+ };
+
+ for (const artifact of artifacts) {
+ const relativePath = artifactRelativePath(artifact);
+ const parts = relativePath.split("/").filter(Boolean);
+ if (parts.length === 0) continue;
+
+ let current = root;
+ for (const directoryName of parts.slice(0, -1)) {
+ let child = current.directories.find((node) => node.name === directoryName);
+ if (!child) {
+ child = {
+ name: directoryName,
+ path: current.path ? `${current.path}/${directoryName}` : directoryName,
+ directories: [],
+ files: [],
+ };
+ current.directories.push(child);
+ }
+ current = child;
+ }
+
+ current.files.push({
+ artifact,
+ name: parts.at(-1) ?? getFileName(artifact),
+ relativePath,
+ });
+ }
+
+ const sort = (node: DirectoryNode) => {
+ node.directories.sort((left, right) =>
+ left.name.localeCompare(right.name, "zh-CN"),
+ );
+ node.files.sort((left, right) =>
+ left.name.localeCompare(right.name, "zh-CN"),
+ );
+ node.directories.forEach(sort);
+ };
+ sort(root);
+ return root;
+}
+
+function DirectoryBranch({
+ node,
+ depth,
+ collapsedDirectories,
+ onToggleDirectory,
+ selectedArtifact,
+ onSelect,
+}: {
+ node: DirectoryNode;
+ depth: number;
+ collapsedDirectories: Set;
+ onToggleDirectory: (path: string) => void;
+ selectedArtifact: string | null;
+ onSelect: (artifact: string) => void;
+}) {
+ return (
+ <>
+ {node.directories.map((directory) => {
+ const collapsed = collapsedDirectories.has(directory.path);
+ return (
+
+
+ {!collapsed && (
+
+ )}
+
+ );
+ })}
+ {node.files.map((file) => {
+ const selected =
+ !!selectedArtifact &&
+ artifactPathname(selectedArtifact) === artifactPathname(file.artifact);
+ return (
+
+ );
+ })}
+ >
+ );
+}
+
+export function ArtifactDirectory({
+ className,
+ artifacts,
+ selectedArtifact,
+ onSelect,
+ onRefresh,
+ refreshing = false,
+}: {
+ className?: string;
+ artifacts: string[];
+ selectedArtifact: string | null;
+ onSelect: (artifact: string) => void;
+ onRefresh?: () => void;
+ refreshing?: boolean;
+}) {
+ const tree = useMemo(() => buildDirectoryTree(artifacts), [artifacts]);
+ const [collapsedDirectories, setCollapsedDirectories] = useState>(
+ () => new Set(),
+ );
+
+ const toggleDirectory = (path: string) => {
+ setCollapsedDirectories((current) => {
+ const next = new Set(current);
+ if (next.has(path)) next.delete(path);
+ else next.add(path);
+ return next;
+ });
+ };
+
+ return (
+
+ );
+}
diff --git a/frontend-web/src/components/workspace/artifacts/index.ts b/frontend-web/src/components/workspace/artifacts/index.ts
index 06c6bc71..270d9e68 100644
--- a/frontend-web/src/components/workspace/artifacts/index.ts
+++ b/frontend-web/src/components/workspace/artifacts/index.ts
@@ -1,4 +1,5 @@
-export * from "./artifact-file-detail";
-export * from "./artifact-file-list";
+export * from "./artifact-file-detail";
+export * from "./artifact-directory";
+export * from "./artifact-file-list";
export * from "./artifact-trigger";
export * from "./context";
diff --git a/frontend-web/src/components/workspace/chats/chat-box.tsx b/frontend-web/src/components/workspace/chats/chat-box.tsx
index fa812476..e2a6c51a 100644
--- a/frontend-web/src/components/workspace/chats/chat-box.tsx
+++ b/frontend-web/src/components/workspace/chats/chat-box.tsx
@@ -1,4 +1,5 @@
import type { Message } from "@langchain/langgraph-sdk";
+import { useQuery } from "@tanstack/react-query";
import { FilesIcon, XIcon } from "lucide-react";
import { usePathname } from "next/navigation";
import { useEffect, useMemo, useRef, useState } from "react";
@@ -13,6 +14,10 @@ import {
} from "@/components/ui/resizable";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { apiFetch } from "@/core/api";
+import {
+ listThreadArtifacts,
+ mergeArtifactPaths,
+} from "@/core/artifacts/listing";
import { getBackendBaseURL } from "@/core/config";
import { isThreadRagModeLocked, useLocalSettings } from "@/core/settings";
import { isSingleRoundRagMessage } from "@/core/threads/use-single-round-rag";
@@ -32,6 +37,7 @@ import { cn } from "@/lib/utils";
import {
ArtifactFileDetail,
+ ArtifactDirectory,
ArtifactFileList,
useArtifacts,
} from "../artifacts";
@@ -40,6 +46,7 @@ import { SkillDisplaySources } from "../skill-display-sources";
const CLOSE_MODE = { chat: 100, artifacts: 0 };
const OPEN_MODE = { chat: 60, artifacts: 40 };
+const DIRECTORY_OPEN_MODE = { chat: 48, artifacts: 52 };
const SHOW_REFERENCE_EVENT = "workspace:show-reference";
type BackendReferenceSource = Parameters[0]["sources"][number];
@@ -78,7 +85,7 @@ const ChatBox: React.FC<{
threadId,
disableArtifacts = false,
}) => {
- const { thread } = useThread();
+ const { isMock, thread } = useThread();
const pathname = usePathname();
const threadIdRef = useRef(threadId);
const artifactsCountRef = useRef(0);
@@ -92,6 +99,8 @@ const ChatBox: React.FC<{
const [pinnedReferenceId, setPinnedReferenceId] = useState(null);
const [backendReferenceBatches, setBackendReferenceBatches] = useState([]);
const [settings] = useLocalSettings();
+ const artifactDirectoryModeEnabled =
+ settings.display.artifactDirectoryModeEnabled ?? false;
const ragModeEnabled =
settings.display.ragModeEnabled || isThreadRagModeLocked(threadId);
// Frontend RAG modes write their own hidden source tool messages. They use
@@ -110,6 +119,52 @@ const ChatBox: React.FC<{
referencesEnabled && !hasFrontendRag;
const { skills } = useSkills();
+ const canScanArtifactDirectory =
+ artifactDirectoryModeEnabled &&
+ !disableArtifacts &&
+ !isMock &&
+ !!threadId &&
+ threadId !== "new";
+ const artifactListingQuery = useQuery({
+ queryKey: ["artifact-directory", threadId],
+ queryFn: ({ signal }) => listThreadArtifacts(threadId, signal),
+ enabled: canScanArtifactDirectory,
+ refetchInterval: thread.isLoading ? 1_500 : false,
+ retry: false,
+ });
+ const checkpointArtifacts = useMemo(
+ () => (disableArtifacts ? [] : (thread.values.artifacts ?? [])),
+ [disableArtifacts, thread.values.artifacts],
+ );
+ const checkpointArtifactSignature = checkpointArtifacts.join("\n");
+ const availableArtifacts = useMemo(
+ () =>
+ artifactDirectoryModeEnabled
+ ? mergeArtifactPaths(
+ checkpointArtifacts,
+ (artifactListingQuery.data?.files ?? []).map((file) => file.path),
+ )
+ : checkpointArtifacts,
+ [
+ artifactDirectoryModeEnabled,
+ artifactListingQuery.data?.files,
+ checkpointArtifacts,
+ ],
+ );
+
+ useEffect(() => {
+ if (!canScanArtifactDirectory || thread.isLoading) return;
+ // Run once after streaming settles as well as when checkpoint artifacts
+ // change, so files written without present_files are not missed by the
+ // last polling tick.
+ void artifactListingQuery.refetch();
+ }, [
+ artifactListingQuery.refetch,
+ canScanArtifactDirectory,
+ checkpointArtifactSignature,
+ thread.isLoading,
+ ]);
+
const {
artifacts,
open: artifactsOpen,
@@ -302,7 +357,8 @@ const ChatBox: React.FC<{
};
}, []);
useEffect(() => {
- if (threadIdRef.current !== threadId) {
+ const threadChanged = threadIdRef.current !== threadId;
+ if (threadChanged) {
threadIdRef.current = threadId;
artifactsCountRef.current = 0;
deselect();
@@ -316,15 +372,21 @@ const ChatBox: React.FC<{
}
// Update artifacts from the current thread
- const nextArtifacts = disableArtifacts ? [] : (thread.values.artifacts ?? []);
+ const nextArtifacts = availableArtifacts;
const hadArtifacts = artifactsCountRef.current > 0;
const hasNewArtifact = nextArtifacts.length > artifactsCountRef.current;
artifactsCountRef.current = nextArtifacts.length;
- setArtifacts(nextArtifacts);
+ const artifactsChanged =
+ artifacts.length !== nextArtifacts.length ||
+ artifacts.some((artifact, index) => artifact !== nextArtifacts[index]);
+ if (artifactsChanged) {
+ setArtifacts(nextArtifacts);
+ }
if (
!disableArtifacts &&
- (hasNewArtifact || (!hadArtifacts && nextArtifacts.length > 0)) &&
- !selectedArtifact
+ nextArtifacts.length > 0 &&
+ ((artifactDirectoryModeEnabled && threadChanged) ||
+ ((hasNewArtifact || !hadArtifacts) && !selectedArtifact))
) {
selectArtifact(nextArtifacts[nextArtifacts.length - 1]!);
setArtifactsOpen(true);
@@ -351,15 +413,41 @@ const ChatBox: React.FC<{
}
}, [
threadId,
+ artifactDirectoryModeEnabled,
autoSelectFirstArtifact,
deselect,
selectArtifact,
selectedArtifact,
+ artifacts,
setArtifacts,
- thread.values.artifacts,
+ availableArtifacts,
disableArtifacts,
]);
+ const directoryModeWasEnabledRef = useRef(artifactDirectoryModeEnabled);
+ useEffect(() => {
+ const wasEnabled = directoryModeWasEnabledRef.current;
+ directoryModeWasEnabledRef.current = artifactDirectoryModeEnabled;
+ if (
+ wasEnabled ||
+ !artifactDirectoryModeEnabled ||
+ availableArtifacts.length === 0
+ ) {
+ return;
+ }
+ if (!selectedArtifact) {
+ selectArtifact(availableArtifacts[availableArtifacts.length - 1]!);
+ }
+ setArtifactsOpen(true);
+ setSideTab("document");
+ }, [
+ artifactDirectoryModeEnabled,
+ availableArtifacts,
+ selectArtifact,
+ selectedArtifact,
+ setArtifactsOpen,
+ ]);
+
const artifactPanelOpen = useMemo(() => {
if (disableArtifacts) {
return false;
@@ -370,7 +458,10 @@ const ChatBox: React.FC<{
return artifactsOpen;
}, [artifactsOpen, artifacts, disableArtifacts]);
const sidePanelOpen = artifactPanelOpen || (hasCitationReferences && referencesOpen);
- const showTabs = !!selectedArtifact && hasCitationReferences;
+ const showTabs =
+ hasCitationReferences &&
+ (!!selectedArtifact ||
+ (artifactDirectoryModeEnabled && artifacts.length > 0));
useEffect(() => {
if (selectedArtifact) {
@@ -390,12 +481,103 @@ const ChatBox: React.FC<{
useEffect(() => {
if (layoutRef.current) {
if (sidePanelOpen) {
- layoutRef.current.setLayout(OPEN_MODE);
+ layoutRef.current.setLayout(
+ artifactDirectoryModeEnabled && artifactPanelOpen
+ ? DIRECTORY_OPEN_MODE
+ : OPEN_MODE,
+ );
} else {
layoutRef.current.setLayout(CLOSE_MODE);
}
}
- }, [sidePanelOpen]);
+ }, [artifactDirectoryModeEnabled, artifactPanelOpen, sidePanelOpen]);
+
+ const artifactDocumentPanel = artifactDirectoryModeEnabled ? (
+
+
{
+ selectArtifact(artifact);
+ setArtifactsOpen(true);
+ }}
+ onRefresh={
+ canScanArtifactDirectory
+ ? () => void artifactListingQuery.refetch()
+ : undefined
+ }
+ refreshing={artifactListingQuery.isFetching}
+ />
+
+ {selectedArtifact ? (
+
+ ) : (
+ <>
+
+
+
+
}
+ title="请选择文件"
+ description="从左侧目录中选择一个文件进行查看"
+ />
+ >
+ )}
+
+
+ ) : selectedArtifact ? (
+
+ ) : (
+
+
+
+
+ {artifacts.length === 0 ? (
+
}
+ title="No artifact selected"
+ description="Select an artifact to view its details"
+ />
+ ) : (
+
+ )}
+
+ );
if (disableArtifacts) {
return (
@@ -455,13 +637,7 @@ const ChatBox: React.FC<{
-
+ {artifactDocumentPanel}
- ) : selectedArtifact ? (
-
+ ) : artifactPanelOpen ? (
+ artifactDocumentPanel
) : hasCitationReferences ? (
@@ -497,41 +667,7 @@ const ChatBox: React.FC<{
sources={referenceSources}
/>
- ) : (
-
-
-
-
- {thread.values.artifacts?.length === 0 ? (
-
}
- title="No artifact selected"
- description="Select an artifact to view its details"
- />
- ) : (
-
- )}
-
- )}
+ ) : null}
diff --git a/frontend-web/src/components/workspace/input-box.tsx b/frontend-web/src/components/workspace/input-box.tsx
index 73a18a89..19cf6cb4 100644
--- a/frontend-web/src/components/workspace/input-box.tsx
+++ b/frontend-web/src/components/workspace/input-box.tsx
@@ -2,13 +2,14 @@
import type { ChatStatus } from "ai";
import {
- BarChart3Icon,
- BrainCircuitIcon,
- BookMarkedIcon,
- CheckIcon,
- ClipboardCheckIcon,
- FileTextIcon,
- GraduationCapIcon,
+ BarChart3Icon,
+ BrainCircuitIcon,
+ BookMarkedIcon,
+ CheckIcon,
+ ClipboardCheckIcon,
+ FileTextIcon,
+ FolderTreeIcon,
+ GraduationCapIcon,
LightbulbIcon,
MessageCircleQuestionIcon,
PaperclipIcon,
@@ -446,12 +447,20 @@ export function InputBox({
[onContextChange, context],
);
- const handleWritingModeToggle = useCallback(() => {
+ const handleWritingModeToggle = useCallback(() => {
onContextChange?.({
...context,
writing_mode: !context.writing_mode,
});
- }, [context, onContextChange]);
+ }, [context, onContextChange]);
+
+ const artifactDirectoryModeEnabled =
+ localSettings.display.artifactDirectoryModeEnabled ?? false;
+ const handleArtifactDirectoryModeToggle = useCallback(() => {
+ setLocalSettings("display", {
+ artifactDirectoryModeEnabled: !artifactDirectoryModeEnabled,
+ });
+ }, [artifactDirectoryModeEnabled, setLocalSettings]);
const memoryInjectionEnabled = context.memory_injection_enabled ?? true;
const handleMemoryInjectionToggle = useCallback(() => {
@@ -854,7 +863,7 @@ export function InputBox({
)}
/>
)}
- {!frontendRagActive &&
写作
-
- }
- {!frontendRagActive &&
+ }
+ {!frontendRagActive &&
+
+ {artifactDirectoryModeEnabled && (
+
+ )}
+
+ 文件
+
+ }
+ {!frontendRagActive && new Set(value), [value]);
const filtered = useMemo(() => {
@@ -97,6 +99,7 @@ function SelectionContent({
}, [rows, search]);
const groups = useMemo(() => groupRows(filtered), [filtered]);
const toggle = (id: string) => {
+ if (readOnly) return;
onChange(selected.has(id) ? value.filter((item) => item !== id) : [...value, id]);
};
@@ -168,6 +171,7 @@ function SelectionContent({
toggle(item.id)}
onSelect={(event) => event.preventDefault()}
className={cn(
@@ -183,6 +187,7 @@ function SelectionContent({
type="button"
role="checkbox"
aria-checked={selected.has(item.id)}
+ disabled={readOnly}
onClick={() => toggle(item.id)}
className={cn(
"my-0.5 flex min-h-11 w-full items-center gap-2 rounded-md px-2 py-1.5 text-left outline-none transition-colors hover:bg-muted/60 focus-visible:ring-2 focus-visible:ring-primary",
@@ -205,15 +210,21 @@ function SelectionContent({
-
已选择 {value.length} 个
-
-
-
-
+
+ {readOnly ? `本对话已选 ${value.length} 个` : `已选择 ${value.length} 个`}
+
+ {readOnly ? (
+
对话开始后不可更改
+ ) : (
+
+
+
+
+ )}
>
);
@@ -226,6 +237,7 @@ export function LlmWikiKnowledgeSelector({
agentId,
autoSelectAll = false,
disabled = false,
+ readOnly = false,
className,
label = "知识空间",
variant = "popover",
@@ -236,6 +248,8 @@ export function LlmWikiKnowledgeSelector({
agentId?: string | null;
autoSelectAll?: boolean;
disabled?: boolean;
+ /** Keep the selected knowledge spaces visible without allowing the thread scope to change. */
+ readOnly?: boolean;
className?: string;
label?: string;
variant?: SelectorVariant;
@@ -293,6 +307,7 @@ export function LlmWikiKnowledgeSelector({
error={error}
source={source}
variant={variant}
+ readOnly={readOnly}
/>
);
@@ -315,9 +330,14 @@ export function LlmWikiKnowledgeSelector({
);
}
- const selectedName = value.length === 1
- ? rows.find((item) => item.id === value[0])?.name
- : null;
+ const selectedNames = value
+ .map((id) => rows.find((item) => item.id === id)?.name)
+ .filter((name): name is string => !!name);
+ const selectedLabel = selectedNames.length > 0
+ ? selectedNames.join("、")
+ : value.length > 0
+ ? `已选 ${value.length} 个知识空间`
+ : label;
return (
@@ -332,9 +352,10 @@ export function LlmWikiKnowledgeSelector({
className,
)}
disabled={disabled || loading}
+ title={value.length > 0 ? selectedLabel : undefined}
>
{loading ? : }
- {selectedName || label}
+ {selectedLabel}
{value.length > 0 && (
{value.length}
diff --git a/frontend-web/src/components/workspace/messages/message-group.tsx b/frontend-web/src/components/workspace/messages/message-group.tsx
index dc4e1759..564384f8 100644
--- a/frontend-web/src/components/workspace/messages/message-group.tsx
+++ b/frontend-web/src/components/workspace/messages/message-group.tsx
@@ -341,9 +341,9 @@ function ToolCall({
ragModeEnabled?: boolean;
}) {
const { t } = useI18n();
- const displayMatch =
- result !== undefined && isDisplaySamplePayload(result)
- ? null
+ const displayMatch =
+ result !== undefined && isDisplaySamplePayload(result)
+ ? null
: (findDisplayConfigForTool(name, skillDisplayMap, args) ??
null);
const displayItems =
@@ -352,18 +352,24 @@ function ToolCall({
...item,
index: index + 1,
skillName: displayMatch.skillName,
- mode: item.displayMode ?? displayMatch.display.mode,
- }))
- : [];
- if (displayItems.length > 0) {
- return (
- 0) {
+ return (
+
@@ -387,7 +393,7 @@ function ToolCall({
mode: "citation" as const,
}));
return (
-
+
);
diff --git a/frontend-web/src/components/workspace/skill-display-sources.tsx b/frontend-web/src/components/workspace/skill-display-sources.tsx
index 27b012cb..f52c114d 100644
--- a/frontend-web/src/components/workspace/skill-display-sources.tsx
+++ b/frontend-web/src/components/workspace/skill-display-sources.tsx
@@ -1,4 +1,4 @@
-import { ExternalLinkIcon, FileTextIcon } from "lucide-react";
+import { BookOpenTextIcon, ChevronRightIcon, ExternalLinkIcon, FileTextIcon } from "lucide-react";
import { Button } from "@/components/ui/button";
import {
@@ -15,9 +15,11 @@ import {
} from "@/core/llmwiki/source-links";
import { useLocalSettings } from "@/core/settings";
import { useCitationDisplaySettings } from "@/core/system-settings/hooks";
+import { reasoningPlugins } from "@/core/streamdown";
import { cn } from "@/lib/utils";
import { useState } from "react";
import { WeKnoraSourceDrawer } from "@/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer";
+import { MarkdownContent } from "@/components/workspace/messages/markdown-content";
/** Whether a source URL is an internal WeKnora preview pointer. */
export function isWeKnoraSourcePreviewUrl(url?: string | null): boolean {
@@ -71,8 +73,20 @@ export function SourceCard({
const cleanupWords =
citationDisplay?.cleanup_words ?? settings.display.citationCleanupWords ?? [];
const content = cleanReferenceText(source.content || source.snippet, cleanupWords);
- const snippet = compactReferenceText(content).slice(0, 280);
- const paragraphs = segmentReferenceText(content);
+ const isWikiSource = Boolean(previewTarget?.wikiSlug) || source.type === "llmwiki_wiki";
+ const matchedHeading = isWikiSource
+ ? String(
+ (source.raw?.matched_heading as string | undefined) ||
+ (source.raw?.matched_sections as Array<{ heading?: string }> | undefined)?.find(
+ (section) => section.heading,
+ )?.heading ||
+ "",
+ ).trim()
+ : "";
+ const summary = isWikiSource && typeof source.raw?.summary === "string"
+ ? source.raw.summary
+ : "";
+ const snippet = plainTextReferencePreview(summary || content).slice(0, 280);
const score =
typeof source.score === "number"
? source.score <= 1
@@ -81,13 +95,22 @@ export function SourceCard({
: null;
const meta = [
source.source,
+ isWikiSource ? "检索对象:Wiki 文档" : null,
+ matchedHeading ? `命中章节:${matchedHeading}` : null,
source.author ? `作者 ${source.author}` : null,
- source.type && source.type !== "result" ? source.type : null,
+ !isWikiSource && source.type && source.type !== "result" ? source.type : null,
source.time,
score ? `相关度 ${score}` : null,
].filter(Boolean);
const isCitation = source.mode === "citation";
const showReferenceNumber = isCitation;
+ const openSource = () => {
+ if (previewTarget) {
+ setDetailOpen(true);
+ } else {
+ setRelatedOpen(true);
+ }
+ };
return (
<>
@@ -102,11 +125,11 @@ export function SourceCard({
"border-amber-300 bg-amber-50/80 shadow-[0_0_0_2px_rgba(251,191,36,0.14)]",
"cursor-pointer",
)}
- onClick={() => setRelatedOpen(true)}
+ onClick={openSource}
onKeyDown={(event) => {
if (event.key === "Enter" || event.key === " ") {
event.preventDefault();
- setRelatedOpen(true);
+ openSource();
}
}}
>
@@ -136,11 +159,16 @@ export function SourceCard({
)}
- {source.url && (
+ {(source.url || isWikiSource) && (
previewTarget ? (
-
-
- 查看原文
+
+ {isWikiSource ? : }
+ {isWikiSource ? "Wiki 文档" : "查看原文"}
) : (
@@ -148,19 +176,9 @@ export function SourceCard({
)}
{previewTarget && (
-
-
+
+ {isWikiSource ? "点击卡片查看完整 Wiki" : "点击卡片查看详情"}
+
)}
@@ -173,7 +191,7 @@ export function SourceCard({
onOpenChange={setDetailOpen}
/>
)}
- }
>
);
}
@@ -254,44 +282,18 @@ function compactReferenceText(text: string): string {
return text.replace(/\s+/g, " ").trim();
}
-function segmentReferenceText(text: string): string[] {
- const explicit = text
- .split(/\n{2,}/)
- .map((item) => item.trim())
- .filter(Boolean);
- const base = explicit.length > 1 ? explicit : splitLongParagraph(text);
- return base.slice(0, 80);
-}
-
-function splitLongParagraph(text: string): string[] {
- const normalized = text
- .split(/\n+/)
- .map((item) => item.trim())
- .filter(Boolean)
- .join("\n");
- if (normalized.length <= 360) {
- return normalized ? [normalized] : [];
- }
-
- const sentences = normalized
- .split(/(?<=[。!?!?;;])\s*/)
- .map((item) => item.trim())
- .filter(Boolean);
- if (sentences.length <= 1) {
- return normalized.match(/[\s\S]{1,360}/g) ?? [];
- }
-
- const paragraphs: string[] = [];
- let current = "";
- for (const sentence of sentences) {
- if ((current + sentence).length > 360 && current) {
- paragraphs.push(current.trim());
- current = "";
- }
- current += sentence;
- }
- if (current.trim()) {
- paragraphs.push(current.trim());
- }
- return paragraphs;
+function plainTextReferencePreview(text: string): string {
+ return compactReferenceText(
+ text
+ .replace(/^---[\s\S]*?---\s*/m, "")
+ .replace(/```[\s\S]*?```/g, " ")
+ .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")
+ .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
+ .replace(/\[\[[^\]\s|]+[|\s]+([^\]]+)\]\]/g, "$1")
+ .replace(/\[\[([^\]]+)\]\]/g, "$1")
+ .replace(/^#{1,6}\s+/gm, "")
+ .replace(/^\s*[-*+]\s+/gm, "")
+ .replace(/^\s*\d+[.)]\s+/gm, "")
+ .replace(/[>*_`~|]/g, " "),
+ );
}
diff --git a/frontend-web/src/components/workspace/workspace-nav-menu.tsx b/frontend-web/src/components/workspace/workspace-nav-menu.tsx
index 8668c4c8..05e15990 100644
--- a/frontend-web/src/components/workspace/workspace-nav-menu.tsx
+++ b/frontend-web/src/components/workspace/workspace-nav-menu.tsx
@@ -13,6 +13,7 @@ import {
HistoryIcon,
ListTreeIcon,
MessageCircleQuestionIcon,
+ MessageSquareTextIcon,
ScrollTextIcon,
Settings2Icon,
SettingsIcon,
@@ -288,6 +289,16 @@ export function WorkspaceNavMenu() {
推荐问题配置
)}
+ {isAdmin && (
+
+ navigate("/page/strategy/admin/fixed-questions")
+ }
+ >
+
+ 问题配置
+
+ )}
{isAdmin && logSystem?.enabled && logSystem.url && (
diff --git a/frontend-web/src/core/artifacts/listing.ts b/frontend-web/src/core/artifacts/listing.ts
new file mode 100644
index 00000000..978d0a5f
--- /dev/null
+++ b/frontend-web/src/core/artifacts/listing.ts
@@ -0,0 +1,113 @@
+import { apiFetch } from "@/core/api";
+import { getBackendBaseURL } from "@/core/config";
+
+export interface ArtifactListingFile {
+ path: string;
+ name: string;
+ sizeBytes: number;
+ mimeType: string | null;
+ modifiedAt: string;
+}
+
+export interface ArtifactListing {
+ threadId: string;
+ files: ArtifactListingFile[];
+}
+
+/**
+ * Scan the thread's outputs directory instead of relying only on artifacts
+ * explicitly surfaced by `present_files`. This is used by file mode so files
+ * written by any tool are still discoverable from the right-hand directory.
+ */
+export async function listThreadArtifacts(
+ threadId: string,
+ signal?: AbortSignal,
+): Promise {
+ const response = await apiFetch(
+ `${getBackendBaseURL()}/api/threads/${encodeURIComponent(threadId)}/artifacts`,
+ { method: "GET", signal },
+ );
+ if (!response.ok) {
+ const detail = await response.text().catch(() => "");
+ throw new Error(
+ `Failed to list artifacts (${response.status})${detail ? `: ${detail}` : ""}`,
+ );
+ }
+
+ const data = (await response.json()) as {
+ thread_id?: string;
+ files?: Array<{
+ path: string;
+ name: string;
+ size_bytes: number;
+ mime_type: string | null;
+ modified_at: string;
+ }>;
+ };
+
+ return {
+ threadId: data.thread_id ?? threadId,
+ files: (data.files ?? []).map((file) => ({
+ path: file.path,
+ name: file.name,
+ sizeBytes: file.size_bytes,
+ mimeType: file.mime_type,
+ modifiedAt: file.modified_at,
+ })),
+ };
+}
+
+/** Return the generated file path represented by a real or streaming artifact. */
+export function artifactPathname(artifact: string): string {
+ const trimmed = artifact.trim();
+ if (!trimmed.startsWith("write-file:")) {
+ return trimmed.replace(/^\/+/, "");
+ }
+ try {
+ return decodeURIComponent(new URL(trimmed).pathname).replace(/^\/+/, "");
+ } catch {
+ return trimmed;
+ }
+}
+
+export function artifactRelativePath(artifact: string): string {
+ const pathname = artifactPathname(artifact);
+ return pathname.replace(/^mnt\/user-data\/outputs\/?/i, "") || pathname;
+}
+
+/**
+ * Keep the checkpoint order, append files found only on disk from old to new,
+ * and replace a completed streaming URL with its durable path.
+ */
+export function mergeArtifactPaths(
+ checkpointArtifacts: string[],
+ discoveredArtifacts: string[],
+): string[] {
+ const merged: string[] = [];
+ const indexByPath = new Map();
+
+ for (const artifact of checkpointArtifacts) {
+ const key = artifactPathname(artifact);
+ if (!key || indexByPath.has(key)) continue;
+ indexByPath.set(key, merged.length);
+ merged.push(artifact);
+ }
+
+ // The listing endpoint returns newest first. Append in chronological order
+ // so the last item remains the newest artifact for the existing auto-open.
+ for (const artifact of [...discoveredArtifacts].reverse()) {
+ const key = artifactPathname(artifact);
+ if (!key) continue;
+ const existingIndex = indexByPath.get(key);
+ if (existingIndex !== undefined) {
+ if (merged[existingIndex]?.startsWith("write-file:")) {
+ merged[existingIndex] = artifact;
+ }
+ continue;
+ }
+ indexByPath.set(key, merged.length);
+ merged.push(artifact);
+ }
+
+ return merged;
+}
diff --git a/frontend-web/src/core/auth/index.ts b/frontend-web/src/core/auth/index.ts
index 36eb5542..64ad9bad 100644
--- a/frontend-web/src/core/auth/index.ts
+++ b/frontend-web/src/core/auth/index.ts
@@ -49,6 +49,17 @@ function normalizeIdentityPart(value: unknown): string {
return typeof value === "string" ? value.trim().toLowerCase() : "";
}
+function identityFromLoginName(value: unknown): string | null {
+ const normalized = normalizeIdentityPart(value);
+ if (!normalized) return null;
+ // Explicit email login is domain-specific. Bare username login is the
+ // stable business principal even if the backend's synthetic email domain or
+ // internal UUID changes.
+ return normalized.includes("@")
+ ? `email:${normalized}`
+ : `account:${normalized}`;
+}
+
function readUserInfoIdentity(): string | null {
let info: Record | null = null;
try {
@@ -81,20 +92,24 @@ function readUserInfoIdentity(): string | null {
/**
* A stable cross-tab identity key for "did the account really change?" checks.
*
- * Some external login paths can re-issue a backend ``user_id`` while resolving
- * to the same business account. Prefer business-facing identifiers
- * (email/username/userInfo) and keep ``user_id`` only as the final fallback.
+ * Some external login paths can re-issue a backend ``user_id`` or use a
+ * different synthetic email domain while resolving to the same business
+ * account. Prefer the raw login username returned by the backend, then the
+ * email local-part, and keep ``user_id`` only as the final fallback.
*/
export function getAccountIdentityKey(
auth: LoginByUsernameResponse | null = getStoredAuth(),
): string | null {
if (!auth) return null;
- const email = normalizeIdentityPart(auth.email);
- if (email) return `email:${email}`;
+ const loginIdentity = identityFromLoginName(auth.username);
+ if (loginIdentity) return loginIdentity;
- const username = normalizeIdentityPart(auth.username);
- if (username) return `username:${username}`;
+ const email = normalizeIdentityPart(auth.email);
+ if (email) {
+ const localPart = email.split("@", 1)[0];
+ return localPart ? `account:${localPart}` : `email:${email}`;
+ }
const userInfoIdentity = readUserInfoIdentity();
if (userInfoIdentity) return `userInfo:${userInfoIdentity}`;
diff --git a/frontend-web/src/core/auth/sync.tsx b/frontend-web/src/core/auth/sync.tsx
index 65321bd3..b4fc5c95 100644
--- a/frontend-web/src/core/auth/sync.tsx
+++ b/frontend-web/src/core/auth/sync.tsx
@@ -1,9 +1,15 @@
"use client";
-import { useEffect, useRef } from "react";
+import { Fragment, useEffect, useRef, useState, type ReactNode } from "react";
+import { useLocation, useNavigate } from "react-router-dom";
+import { useQueryClient } from "@tanstack/react-query";
import { toast } from "sonner";
-import { getAccountIdentityKey } from "@/core/auth";
+import {
+ AUTH_STATE_CHANGED_EVENT,
+ getAccountDisplayName,
+ getAccountIdentityKey,
+} from "@/core/auth";
import { isEmbedActive, isIsolatedSession } from "@/core/embed/embed-session";
const AUTH_STORAGE_KEY = "deerflow.auth";
@@ -14,14 +20,37 @@ function readIdentityKey(): string | null {
return getAccountIdentityKey();
}
+function freshRouteForAccount(pathname: string): string {
+ // A thread URL belongs to the old account. Keep its workspace/agent context
+ // but move to a valid new conversation before the user can submit into the
+ // stale thread. Other pages can stay in place and are remounted below.
+ const chatMatch = pathname.match(/^(.*\/chats)\/[^/]+\/?$/);
+ return chatMatch ? `${chatMatch[1]}/new` : pathname;
+}
+
/**
- * Mount once at the app root. Watches localStorage auth changes from other
- * tabs and lets the user decide when to refresh on account switches.
+ * Cross-tab auth boundary for normal pages.
+ *
+ * Same-account token re-issuance is ignored. A genuine account change clears
+ * account-scoped query state, remounts the page, and moves an open chat to its
+ * fresh-chat route without a full browser reload. Embedded/isolated tabs keep
+ * their sessionStorage identity and never participate in this synchronization.
*/
-export function AuthSyncWatcher() {
+export function AuthSyncWatcher({
+ children,
+ disabled = false,
+}: {
+ children?: ReactNode;
+ disabled?: boolean;
+}) {
const lastIdentityRef = useRef(readIdentityKey());
+ const [revision, setRevision] = useState(0);
+ const queryClient = useQueryClient();
+ const navigate = useNavigate();
+ const location = useLocation();
useEffect(() => {
+ if (disabled) return;
// 嵌入(iframe)态 / 独立会话态(?isolate=1):鉴权走按标签页的 sessionStorage,本就不与其它标签页
// 共享,且不应被其它标签页的 localStorage 账号变更打扰 —— 直接不挂跨标签监听。
if (isEmbedActive() || isIsolatedSession()) return;
@@ -36,18 +65,27 @@ export function AuthSyncWatcher() {
lastIdentityRef.current = nextIdentity;
+ void queryClient.cancelQueries();
+ queryClient.clear();
+ window.dispatchEvent(new Event(AUTH_STATE_CHANGED_EVENT));
+ setRevision((value) => value + 1);
+
if (nextIdentity === null) {
+ toast.info("账号已在其它页面退出", {
+ id: IDENTITY_TOAST_ID,
+ description: "当前页面已同步退出状态。",
+ });
+ navigate("/login", { replace: true });
return;
}
- toast.warning("检测到账号在其它标签页发生变化", {
+ const freshRoute = freshRouteForAccount(location.pathname);
+ if (freshRoute !== location.pathname) {
+ navigate(freshRoute, { replace: true });
+ }
+ toast.info("账号已在页面间自动同步", {
id: IDENTITY_TOAST_ID,
- description: "当前标签页仍是原会话。如需与新账号同步,请刷新页面。",
- duration: Infinity,
- action: {
- label: "刷新同步",
- onClick: () => window.location.reload(),
- },
+ description: `当前账号:${getAccountDisplayName()}。无需刷新页面。`,
});
}
@@ -68,14 +106,24 @@ export function AuthSyncWatcher() {
settleTimer = setTimeout(evaluateSettledIdentity, SETTLE_DELAY_MS);
}
+ function handleSameTabAuthChange() {
+ // `storage` never fires in the tab that called setStoredAuth/clearStoredAuth.
+ // Keep the comparison baseline current so a later same-account token
+ // renewal from another normal page is not compared with the pre-login
+ // `null` identity and falsely classified as an account switch.
+ lastIdentityRef.current = readIdentityKey();
+ }
+
window.addEventListener("storage", handleStorage);
+ window.addEventListener(AUTH_STATE_CHANGED_EVENT, handleSameTabAuthChange);
return () => {
window.removeEventListener("storage", handleStorage);
+ window.removeEventListener(AUTH_STATE_CHANGED_EVENT, handleSameTabAuthChange);
if (settleTimer) {
clearTimeout(settleTimer);
}
};
- }, []);
+ }, [disabled, location.pathname, navigate, queryClient]);
- return null;
+ return children ? {children} : null;
}
diff --git a/frontend-web/src/core/auth/use-live-account.ts b/frontend-web/src/core/auth/use-live-account.ts
index 8cfedd8c..b855973f 100644
--- a/frontend-web/src/core/auth/use-live-account.ts
+++ b/frontend-web/src/core/auth/use-live-account.ts
@@ -8,7 +8,7 @@ import {
getAccountIdentityKey,
getStoredAuth,
} from "@/core/auth";
-import { isEmbedActive } from "@/core/embed/embed-session";
+import { isEmbedActive, isIsolatedSession } from "@/core/embed/embed-session";
const AUTH_STORAGE_KEY = "deerflow.auth";
@@ -65,7 +65,9 @@ export function useLiveAccount(): LiveAccount {
refresh();
}
}
- if (!isEmbedActive()) {
+ // Normal pages share localStorage identity across tabs. Embedded and
+ // isolated pages deliberately keep their per-tab sessionStorage identity.
+ if (!isEmbedActive() && !isIsolatedSession()) {
window.addEventListener("storage", onStorage);
}
window.addEventListener(AUTH_STATE_CHANGED_EVENT, refresh);
diff --git a/frontend-web/src/core/i18n/locales/en-US.ts b/frontend-web/src/core/i18n/locales/en-US.ts
index de6a7d60..e8a7ad64 100644
--- a/frontend-web/src/core/i18n/locales/en-US.ts
+++ b/frontend-web/src/core/i18n/locales/en-US.ts
@@ -456,7 +456,7 @@ export const enUS: Translations = {
lightDescription: "Bright palette with higher contrast for daytime.",
darkDescription: "Dim palette that reduces glare for focus.",
lightblueDescription: "A crisp light palette built around blue.",
- lightgreenDescription: "A crisp light palette built around WeKnora green.",
+ lightgreenDescription: "A crisp light palette built around fresh green tones.",
iceblueDescription: "An airy palette built around #9BCEF1 with accessible blue actions.",
lightyellowDescription: "A soft palette built around #F3FBD9 with readable olive actions.",
languageTitle: "Language",
diff --git a/frontend-web/src/core/i18n/locales/zh-CN.ts b/frontend-web/src/core/i18n/locales/zh-CN.ts
index ab06ad4a..8610ac88 100644
--- a/frontend-web/src/core/i18n/locales/zh-CN.ts
+++ b/frontend-web/src/core/i18n/locales/zh-CN.ts
@@ -437,7 +437,7 @@ export const zhCN: Translations = {
lightDescription: "更明亮的配色,适合日间使用。",
darkDescription: "更暗的配色,减少眩光方便专注。",
lightblueDescription: "以蓝色为主色的清爽浅色配色。",
- lightgreenDescription: "采用 WeKnora 绿色的清爽浅色配色。",
+ lightgreenDescription: "采用清新绿色的浅色配色。",
iceblueDescription: "以 #9BCEF1 为主色,搭配冰蓝留白与沉稳交互色。",
lightyellowDescription: "以 #F3FBD9 为主色,搭配柔和浅黄留白与深橄榄交互色。",
languageTitle: "语言",
diff --git a/frontend-web/src/core/llmwiki/source-links.ts b/frontend-web/src/core/llmwiki/source-links.ts
index d91c0845..60f7f677 100644
--- a/frontend-web/src/core/llmwiki/source-links.ts
+++ b/frontend-web/src/core/llmwiki/source-links.ts
@@ -44,7 +44,7 @@ export function parseWeKnoraPreviewUrl(url?: string | null): WeKnoraSourceTarget
params.get("wikiSlug")?.trim() ||
params.get("slug")?.trim() ||
"";
- return knowledgeBaseId && chunkId
+ return knowledgeBaseId && (wikiSlug || chunkId || documentId)
? {
knowledgeBaseId,
chunkId,
@@ -75,6 +75,7 @@ export function resolveWeKnoraSourceTarget(source: KnowledgeSourceLike): WeKnora
fromPreview?.documentId ||
"";
const chunkId = source.chunk_id || source.chunkId || fromPreview?.chunkId || "";
+ if (!wikiSlug && !chunkId && !documentId) return null;
return {
knowledgeBaseId,
chunkId: chunkId || undefined,
@@ -94,6 +95,43 @@ export function buildWeKnoraSourceIframeUrl(target: WeKnoraSourceTarget): string
return `/#/page/workspace/knowledge/${encodeURIComponent(target.knowledgeBaseId)}${query ? `?${query}` : ""}`;
}
+/**
+ * Lightweight native reader for one Wiki page. Unlike the knowledge-base
+ * detail route this does not create a WeKnora iframe session.
+ */
+export function buildWeKnoraWikiPageUrl(
+ knowledgeBaseId: string,
+ wikiSlug: string,
+): string {
+ return `/#${buildWeKnoraWikiPagePath(knowledgeBaseId, wikiSlug)}`;
+}
+
+export function buildWeKnoraWikiPagePath(
+ knowledgeBaseId: string,
+ wikiSlug: string,
+): string {
+ const params = new URLSearchParams({ wikiSlug });
+ return `/page/workspace/knowledge/${encodeURIComponent(knowledgeBaseId)}/wiki/view?${params.toString()}`;
+}
+
+/** Public, chrome-free Wiki reader intended for third-party iframe embedding. */
+export function buildPublicKnowledgeWikiPath(
+ knowledgeBaseId: string,
+ wikiSlug?: string,
+): string {
+ const params = new URLSearchParams();
+ if (wikiSlug) params.set("wikiSlug", wikiSlug);
+ const query = params.toString();
+ return `/embed/knowledge/${encodeURIComponent(knowledgeBaseId)}${query ? `?${query}` : ""}`;
+}
+
+export function buildPublicKnowledgeWikiUrl(
+ knowledgeBaseId: string,
+ wikiSlug?: string,
+): string {
+ return `/#${buildPublicKnowledgeWikiPath(knowledgeBaseId, wikiSlug)}`;
+}
+
export function buildLegacyKnowledgeSourceUrl(source: KnowledgeSourceLike): string {
const recUuid = source.recUuid || source.rec_uuid || source.url || "";
if (!recUuid) return "";
@@ -108,7 +146,9 @@ export function buildLegacyKnowledgeSourceUrl(source: KnowledgeSourceLike): stri
export function openKnowledgeSource(source: KnowledgeSourceLike): boolean {
const weknoraTarget = resolveWeKnoraSourceTarget(source);
const href = weknoraTarget
- ? buildWeKnoraSourceIframeUrl(weknoraTarget)
+ ? weknoraTarget.wikiSlug
+ ? buildWeKnoraWikiPageUrl(weknoraTarget.knowledgeBaseId, weknoraTarget.wikiSlug)
+ : buildWeKnoraSourceIframeUrl(weknoraTarget)
: buildLegacyKnowledgeSourceUrl(source);
if (!href) return false;
window.open(href, "_blank", "noopener,noreferrer");
diff --git a/frontend-web/src/core/llmwiki/wiki-page-markdown.ts b/frontend-web/src/core/llmwiki/wiki-page-markdown.ts
new file mode 100644
index 00000000..d4a24078
--- /dev/null
+++ b/frontend-web/src/core/llmwiki/wiki-page-markdown.ts
@@ -0,0 +1,56 @@
+import { buildWeKnoraWikiPageUrl } from "@/core/llmwiki/source-links";
+
+export type WikiPageUrlBuilder = (
+ knowledgeBaseId: string,
+ wikiSlug: string,
+) => string;
+
+export function stripWikiFrontmatter(value: string) {
+ return value.replace(/^\s*---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, "").trim();
+}
+
+export function safeWikiFileName(value: string) {
+ return value.replace(/[\\/:*?"<>|\r\n]+/g, "_").replace(/\s+/g, " ").trim() || "wiki-page";
+}
+
+export function exportableWikiMarkdown(title: string, summary: string, content: string) {
+ const body = content.trim();
+ const hasTitle = /^#\s+/m.test(body);
+ return [hasTitle ? "" : `# ${title}`, summary && !body.includes(summary) ? `> ${summary}` : "", body]
+ .filter(Boolean)
+ .join("\n\n");
+}
+
+export function wikiLinkMarkdown(
+ markdown: string,
+ knowledgeBaseId: string,
+ buildPageUrl: WikiPageUrlBuilder = buildWeKnoraWikiPageUrl,
+) {
+ const link = (slugValue: string, labelValue?: string) => {
+ const slug = slugValue.trim();
+ const fallbackLabel = slug.split("/").filter(Boolean).at(-1) || slug;
+ const label = (labelValue || fallbackLabel).trim().replace(/\]/g, "\\]");
+ return `[${label}](${buildPageUrl(knowledgeBaseId, slug)})`;
+ };
+ return markdown
+ .replace(/\[\[([^\]|]+)\|([^\]]+)\]\]/g, (_match, slug, label) => link(slug, label))
+ .replace(/\[\[([^\]\s]+)\s+([^\]]+)\]\]/g, (_match, slug, label) => link(slug, label))
+ .replace(/\[\[([^\]]+)\]\]/g, (_match, slug) => link(slug));
+}
+
+export function renderableWikiMarkdown(
+ title: string,
+ content: string,
+ knowledgeBaseId: string,
+ buildPageUrl?: WikiPageUrlBuilder,
+) {
+ const lines = content.trim().split("\n");
+ if (lines[0]?.replace(/^#+\s*/, "").trim() === title.trim()) {
+ lines[0] = `# ${title}`;
+ }
+ return wikiLinkMarkdown(
+ exportableWikiMarkdown(title, "", lines.join("\n")),
+ knowledgeBaseId,
+ buildPageUrl,
+ );
+}
diff --git a/frontend-web/src/core/llmwiki/wiki-tree.ts b/frontend-web/src/core/llmwiki/wiki-tree.ts
new file mode 100644
index 00000000..1db7ebc6
--- /dev/null
+++ b/frontend-web/src/core/llmwiki/wiki-tree.ts
@@ -0,0 +1,146 @@
+import type { LlmWikiWikiPage } from "@/strategy-components/api/llmwiki";
+
+export type WikiTreeNode = {
+ key: string;
+ label: string;
+ page: LlmWikiWikiPage | null;
+ children: WikiTreeNode[];
+ count: number;
+};
+
+export type WikiTree = {
+ roots: WikiTreeNode[];
+ parentKeysBySlug: Map;
+};
+
+function pageDirectoryParts(page: LlmWikiWikiPage): string[] {
+ const wikiPath = (page.wiki_path || "").split("/").map((item) => item.trim()).filter(Boolean);
+ if (wikiPath.length > 2) return wikiPath.slice(1, -1);
+ const category = page.category_path?.filter(Boolean) || [];
+ if (category.length > 0) return category;
+ const parts = (page.slug || "").split("/").filter(Boolean);
+ return parts.length > 1 ? parts.slice(0, -1) : ["未分类"];
+}
+
+export function filterWikiPagesWithAncestors(
+ pages: LlmWikiWikiPage[],
+ query: string,
+): LlmWikiWikiPage[] {
+ const words = query.trim().toLowerCase().split(/\s+/).filter(Boolean);
+ if (words.length === 0) return pages;
+ const matched = pages.filter((page) => {
+ const haystack = [
+ page.title,
+ page.slug,
+ page.summary,
+ page.page_type,
+ page.parent_slug,
+ ...(page.aliases || []),
+ ...(page.category_path || []),
+ ]
+ .filter(Boolean)
+ .join(" ")
+ .toLowerCase();
+ return words.every((word) => haystack.includes(word));
+ });
+ const bySlug = new Map(pages.map((page) => [page.slug, page]));
+ const result = new Map();
+ for (const page of matched) {
+ let current: LlmWikiWikiPage | undefined = page;
+ const visited = new Set();
+ while (current?.slug && !visited.has(current.slug)) {
+ visited.add(current.slug);
+ result.set(current.slug, current);
+ current = current.parent_slug ? bySlug.get(current.parent_slug) : undefined;
+ }
+ }
+ return Array.from(result.values());
+}
+
+function attachNode(parent: WikiTreeNode | null, roots: WikiTreeNode[], node: WikiTreeNode) {
+ const siblings = parent ? parent.children : roots;
+ if (!siblings.some((item) => item.key === node.key)) siblings.push(node);
+}
+
+function sortWikiNodes(nodes: WikiTreeNode[]) {
+ nodes.sort((left, right) => {
+ if (!!left.children.length !== !!right.children.length) return left.children.length ? -1 : 1;
+ return left.label.localeCompare(right.label, "zh-Hans-CN");
+ });
+ nodes.forEach((node) => sortWikiNodes(node.children));
+}
+
+function countPages(node: WikiTreeNode): number {
+ node.count = (node.page ? 1 : 0) + node.children.reduce((sum, child) => sum + countPages(child), 0);
+ return node.count;
+}
+
+export function buildWikiTree(pages: LlmWikiWikiPage[]): WikiTree {
+ const roots: WikiTreeNode[] = [];
+ const pageNodes = new Map();
+ const directoryNodes = new Map();
+ const parentKeysBySlug = new Map();
+
+ for (const page of pages) {
+ if (!page.slug) continue;
+ pageNodes.set(page.slug, {
+ key: `page:${page.slug}`,
+ label: page.title || page.slug,
+ page,
+ children: [],
+ count: 1,
+ });
+ }
+
+ const directoryNode = (path: string[]) => {
+ const key = `dir:${path.join("/")}`;
+ const existing = directoryNodes.get(key);
+ if (existing) return existing;
+ const node: WikiTreeNode = {
+ key,
+ label: path[path.length - 1] || "未分类",
+ page: null,
+ children: [],
+ count: 0,
+ };
+ directoryNodes.set(key, node);
+ return node;
+ };
+
+ for (const page of pages) {
+ const node = pageNodes.get(page.slug);
+ if (!node) continue;
+ const ancestorKeys: string[] = [];
+ const parentSlug = page.parent_slug?.trim();
+ if (parentSlug && pageNodes.has(parentSlug)) {
+ const parent = pageNodes.get(parentSlug)!;
+ attachNode(parent, roots, node);
+ ancestorKeys.push(parent.key);
+ let current = parent.page?.parent_slug ? pageNodes.get(parent.page.parent_slug) : undefined;
+ const seen = new Set([parent.key]);
+ while (current && !seen.has(current.key)) {
+ seen.add(current.key);
+ ancestorKeys.unshift(current.key);
+ current = current.page?.parent_slug ? pageNodes.get(current.page.parent_slug) : undefined;
+ }
+ parentKeysBySlug.set(page.slug, ancestorKeys);
+ continue;
+ }
+
+ let parent: WikiTreeNode | null = null;
+ const path: string[] = [];
+ for (const part of pageDirectoryParts(page)) {
+ path.push(part);
+ const directory = directoryNode(path);
+ attachNode(parent, roots, directory);
+ parent = directory;
+ ancestorKeys.push(directory.key);
+ }
+ attachNode(parent, roots, node);
+ parentKeysBySlug.set(page.slug, ancestorKeys);
+ }
+
+ roots.forEach(countPages);
+ sortWikiNodes(roots);
+ return { roots, parentKeysBySlug };
+}
diff --git a/frontend-web/src/core/settings/local.ts b/frontend-web/src/core/settings/local.ts
index b9c86665..c79ec805 100644
--- a/frontend-web/src/core/settings/local.ts
+++ b/frontend-web/src/core/settings/local.ts
@@ -4,10 +4,11 @@ export const DEFAULT_LOCAL_SETTINGS: LocalSettings = {
notification: {
enabled: true,
},
- display: {
- streamSpeedMeterEnabled: true,
- ragModeEnabled: false,
- contextCompressionEnabled: false,
+ display: {
+ streamSpeedMeterEnabled: true,
+ ragModeEnabled: false,
+ artifactDirectoryModeEnabled: false,
+ contextCompressionEnabled: false,
citationCleanupWords: [],
fontScale: 1,
},
@@ -43,7 +44,12 @@ export interface LocalSettings {
* Global switch for skill result display adapters. When disabled, chat
* messages render tool results with the legacy generic search UI only.
*/
- ragModeEnabled: boolean;
+ ragModeEnabled: boolean;
+ /**
+ * When enabled, the artifact panel scans every generated output and keeps
+ * a directory tree visible beside the selected file preview.
+ */
+ artifactDirectoryModeEnabled: boolean;
/**
* Per-user switch for context compression (上下文压缩). When enabled, long
* conversations are summarized transiently before the model call to save
diff --git a/frontend-web/src/core/threads/hooks.ts b/frontend-web/src/core/threads/hooks.ts
index a46229e0..e5cc5bce 100644
--- a/frontend-web/src/core/threads/hooks.ts
+++ b/frontend-web/src/core/threads/hooks.ts
@@ -761,17 +761,15 @@ export function useThreadStream({
}
return;
}
- // A "thread not found" 404 almost always means the URL points at a
- // thread from a previous login session — either this tab outlived a
- // logout/re-login or the thread genuinely belongs to a different user.
- // Re-frame the error around the *current* logged-in identity and bounce
- // the user to a fresh chat instead of leaving them stuck on the stale
- // route looking at a raw HTTP 404.
- if (isThreadNotAccessibleError(message)) {
- const userLabel = buildCurrentUserLabel();
- toast.info(
- `当前登录用户为 ${userLabel},原对话不存在或无权访问,正在为你刷新页面…`,
- );
+ // A "thread not found" 404 means this route cannot accept another turn.
+ // Move to a fresh chat without a full browser reload; the old reload path
+ // made every click look like a crash and could loop indefinitely when a
+ // different tab's cookie kept overriding this tab's Bearer token.
+ if (isThreadNotAccessibleError(message)) {
+ const userLabel = buildCurrentUserLabel();
+ toast.info(
+ `当前登录用户为 ${userLabel},原对话不存在或无权访问,已为你打开新对话。`,
+ );
// Prefer staying inside an agent chat context when the user is on an
// agent thread URL — otherwise fall back to the global new chat.
const hash = typeof window !== "undefined" ? window.location.hash : "";
@@ -782,18 +780,7 @@ export function useThreadStream({
: iframeMatch
? "/page/workspace/chats_iframe/new"
: "/page/workspace/chats/new";
- // First navigate to a safe URL so the post-reload page lands on a
- // valid route. Then full-reload so TanStack Query cache, context
- // providers and any other state captured under the previous user's
- // identity get rebuilt against the current login.
- if (typeof window !== "undefined") {
- window.location.hash = target;
- window.setTimeout(() => {
- window.location.reload();
- }, 900);
- } else {
- navigate(target, { replace: true });
- }
+ navigate(target, { replace: true });
return;
}
// 上一轮还没结束又发了新问题(409)或并发达到上限(429):
diff --git a/frontend-web/src/core/threads/thread-access-error.ts b/frontend-web/src/core/threads/thread-access-error.ts
index 4b01f74f..266b0f2a 100644
--- a/frontend-web/src/core/threads/thread-access-error.ts
+++ b/frontend-web/src/core/threads/thread-access-error.ts
@@ -13,8 +13,9 @@ import { getStoredAuth } from "@/core/auth";
* 404 just leaves the user staring at a scary error. Both the chat stream
* (``core/threads/hooks.ts``) and the AI-writing stream
* (``open-canvas/contexts/AIWritingContext.tsx``) funnel their stream errors
- * through here so the UX is identical: detect the login-change 404 and prompt
- * the user to refresh against the current identity instead of erroring out.
+ * through here so the UX is identical: detect the login-change 404 and reset
+ * the stale in-memory session against the current identity instead of showing
+ * a raw error or reloading the browser page.
*/
/** Normalize an unknown stream error into a human-readable message string. */
@@ -108,22 +109,15 @@ export function buildCurrentUserLabel(): string {
}
/**
- * Re-frame a login-change 404 around the current identity and reload the page.
+ * Re-frame a login-change 404 around the current identity and reset its view.
*
- * Shows a friendly toast naming the current login, then full-reloads after a
- * short delay so the toast is visible and so TanStack Query cache / context
- * providers / any state captured under the previous user get rebuilt against
- * the current login. Used by streams whose session id lives in component state
- * (not the URL) — a plain reload lands them back on a fresh, valid page.
+ * Streams whose session id lives in component state pass a local reset callback
+ * so they can discard the inaccessible session without a browser reload.
*/
-export function notifyThreadAccessChangedAndReload(): void {
+export function notifyThreadAccessChanged(onReset: () => void): void {
const userLabel = buildCurrentUserLabel();
toast.info(
- `当前登录用户为 ${userLabel},原会话不存在或无权访问,正在为你刷新页面…`,
+ `当前登录用户为 ${userLabel},原会话不存在或无权访问,已为你重置为新会话。`,
);
- if (typeof window !== "undefined") {
- window.setTimeout(() => {
- window.location.reload();
- }, 900);
- }
+ onReset();
}
diff --git a/frontend-web/src/open-canvas/contexts/AIWritingContext.tsx b/frontend-web/src/open-canvas/contexts/AIWritingContext.tsx
index 2b0b9f0a..de4e300f 100644
--- a/frontend-web/src/open-canvas/contexts/AIWritingContext.tsx
+++ b/frontend-web/src/open-canvas/contexts/AIWritingContext.tsx
@@ -18,7 +18,7 @@ import { getAPIClient } from '@/core/api';
import {
getStreamErrorMessage,
isThreadNotAccessibleError,
- notifyThreadAccessChangedAndReload,
+ notifyThreadAccessChanged,
} from '@/core/threads/thread-access-error';
import { useCanvasGraphContext } from '../contexts/CanvasGraphContext';
@@ -299,10 +299,13 @@ const AIWritingStreamHost = memo(function AIWritingStreamHost({
// thread belongs to a previous login session — the user switched accounts
// (or this tab outlived a re-login), so the current account legitimately
// can't access the old thread. Instead of dumping the raw HTTP 404 into
- // the panel, re-frame it around the current identity and reload — same UX
- // the chat stream gives (see core/threads/hooks.ts).
+ // the panel, re-frame it around the current identity and discard the
+ // stale component-state session without reloading the browser page.
if (isThreadNotAccessibleError(message)) {
- notifyThreadAccessChangedAndReload();
+ notifyThreadAccessChanged(() => {
+ continueTargetRef.current = null;
+ dispatchRef.current?.({ type: 'reset' });
+ });
return;
}
dispatchRef.current?.({ type: 'set_error', message });
@@ -410,6 +413,13 @@ export function AIWritingProvider({ children }: { children: ReactNode }) {
// (刷新会断开 SSE,后端默认 on_disconnect=cancel 取消该 run,不留孤儿)。
const threadId = state.sessionId ?? undefined;
+ const resetAfterThreadAccessError = useCallback(() => {
+ setArtifact(null);
+ seenInterruptRef.current = null;
+ continueTargetRef.current = null;
+ dispatch({ type: 'reset' });
+ }, [setArtifact]);
+
useEffect(() => {
const s = state.status;
const savable = s === 'done' || s === 'error' || s.startsWith('awaiting_');
@@ -473,12 +483,12 @@ export function AIWritingProvider({ children }: { children: ReactNode }) {
} catch (err) {
const message = getStreamErrorMessage(err);
if (isThreadNotAccessibleError(message)) {
- notifyThreadAccessChangedAndReload();
+ notifyThreadAccessChanged(resetAfterThreadAccessError);
return;
}
dispatch({ type: 'set_error', message });
}
- }, [state.status]);
+ }, [state.status, resetAfterThreadAccessError]);
const submitIntervention = useCallback(async (payload: UserInterventionPayload) => {
if (!state.sessionId || !state.currentPause) return;
@@ -585,14 +595,14 @@ export function AIWritingProvider({ children }: { children: ReactNode }) {
} catch (err) {
const message = getStreamErrorMessage(err);
if (isThreadNotAccessibleError(message)) {
- notifyThreadAccessChangedAndReload();
+ notifyThreadAccessChanged(resetAfterThreadAccessError);
return;
}
dispatch({ type: 'set_error', message });
}
}, [state.sessionId, state.currentPause, state.materials, state.outline,
state.currentDraftTitle, state.currentDraftMarkdown, state.reviewResult,
- state.status]);
+ state.status, resetAfterThreadAccessError]);
const pauseWriting = useCallback(() => {
// 真停:取消当前 run(graph 已完成的阶段已落 checkpoint)。继续时从该 checkpoint 续跑。
@@ -626,13 +636,13 @@ export function AIWritingProvider({ children }: { children: ReactNode }) {
} catch (err) {
const message = getStreamErrorMessage(err);
if (isThreadNotAccessibleError(message)) {
- notifyThreadAccessChangedAndReload();
+ notifyThreadAccessChanged(resetAfterThreadAccessError);
return;
}
dispatch({ type: 'set_error', message });
}
})();
- }, [state.request]);
+ }, [state.request, resetAfterThreadAccessError]);
// Restore an unfinished history session: make it active, load its checkpoint, then let
// the host skip completed awaiting_* stages or re-run the current unfinished stage.
diff --git a/frontend-web/src/pages/AgentChatPage.tsx b/frontend-web/src/pages/AgentChatPage.tsx
index 86c4583d..4487813a 100644
--- a/frontend-web/src/pages/AgentChatPage.tsx
+++ b/frontend-web/src/pages/AgentChatPage.tsx
@@ -776,6 +776,17 @@ export default function AgentChatPage({
onContextChange={(context) =>
setSettings("context", context)
}
+ knowledgeSelector={
+
+ }
onFollowupsVisibilityChange={setShowFollowups}
onSubmit={handleSubmit}
onStop={handleStop}
diff --git a/frontend-web/src/pages/ChatPage.tsx b/frontend-web/src/pages/ChatPage.tsx
index 5e95df8e..260d62fd 100644
--- a/frontend-web/src/pages/ChatPage.tsx
+++ b/frontend-web/src/pages/ChatPage.tsx
@@ -838,20 +838,19 @@ export default function ChatPage() {
: undefined
}
knowledgeSelector={
- isLandingState ? (
-
- ) : undefined
+
}
onFollowupsVisibilityChange={setShowFollowups}
onSubmit={handleSubmit}
diff --git a/frontend-web/src/pages/EnterpriseResearchPage.tsx b/frontend-web/src/pages/EnterpriseResearchPage.tsx
index 9c2118bb..1f5bdcdb 100644
--- a/frontend-web/src/pages/EnterpriseResearchPage.tsx
+++ b/frontend-web/src/pages/EnterpriseResearchPage.tsx
@@ -37,11 +37,11 @@ import { cn } from "@/lib/utils";
import { openKnowledgeSource } from "@/core/llmwiki/source-links";
import MarkdownRenderer from "@/strategy-components/MarkdownRenderer";
import {
- type LlmWikiSearchResult,
useLlmWikiKnowledgeBases,
useLlmWikiRuntime,
} from "@/strategy-components/api/llmwiki";
import {
+ type EnterpriseResearchEvidence,
type EnterpriseResearchTask,
type EnterpriseResearchTaskStatus,
type EnterpriseResearchReportEvent,
@@ -636,7 +636,7 @@ function EvidencePane({
onStartReport, onCancelReport, onRestoreReport, onSourceSelect,
}: {
task: EnterpriseResearchTask | null;
- source: LlmWikiSearchResult | null;
+ source: EnterpriseResearchEvidence | null;
reportJob: EnterpriseResearchReportJob | null;
reportJobs: EnterpriseResearchReportJob[];
reportDraft: string;
@@ -665,7 +665,7 @@ function EvidencePane({
const latestRewrite = reportJobs.find((job) => job.action === "rewrite" && job.status === "completed" && job.original_report);
const latestCommittedJob = reportJobs.find((job) => job.action !== "followup" && job.status === "completed");
const completedFollowups = reportJobs.filter((job) => job.action === "followup" && job.status === "completed");
- const selectedSources = task.selected_source_ids.map((id) => task.sources.find((item) => item.chunk_id === id)).filter(Boolean) as LlmWikiSearchResult[];
+ const selectedSources = task.selected_source_ids.map((id) => task.sources.find((item) => item.chunk_id === id)).filter(Boolean) as EnterpriseResearchEvidence[];
const citationIndex = new Map(selectedSources.map((item, index) => [item.chunk_id, index + 1]));
const renderedReport = currentReport.replace(/\[资料:([^\]]+)\]/g, (match, id: string) => {
const index = citationIndex.get(id.trim());
diff --git a/frontend-web/src/pages/FixedQuestionsPage.tsx b/frontend-web/src/pages/FixedQuestionsPage.tsx
new file mode 100644
index 00000000..98e276da
--- /dev/null
+++ b/frontend-web/src/pages/FixedQuestionsPage.tsx
@@ -0,0 +1,310 @@
+"use client";
+
+import { useEffect, useMemo, useState } from "react";
+import { Navigate } from "react-router-dom";
+import {
+ Info,
+ MessageSquareText,
+ Plus,
+ Save,
+ Search,
+ Trash2,
+ Zap,
+} from "lucide-react";
+import { toast } from "sonner";
+
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Switch } from "@/components/ui/switch";
+import { Textarea } from "@/components/ui/textarea";
+import { getStoredAuth } from "@/core/auth";
+import {
+ type FixedQuestion,
+ useFixedQuestionActions,
+ useFixedQuestions,
+} from "@/strategy-components/api/fixed-questions";
+
+let sequence = 1;
+
+function newId(): string {
+ if (typeof crypto !== "undefined" && "randomUUID" in crypto) {
+ return crypto.randomUUID();
+ }
+ return `fixed-${Date.now()}-${sequence++}`;
+}
+
+function serialize(rows: FixedQuestion[]): string {
+ return JSON.stringify(
+ rows
+ .map(({ id, question, answer, enabled, tokensPerSecond, sortOrder }) => ({
+ id,
+ question,
+ answer,
+ enabled,
+ tokensPerSecond,
+ sortOrder,
+ }))
+ .sort((a, b) => a.id.localeCompare(b.id)),
+ );
+}
+
+export default function FixedQuestionsPage() {
+ const isAdmin = getStoredAuth()?.system_role === "admin";
+ const { data: serverQuestions = [], isLoading } = useFixedQuestions();
+ const { save } = useFixedQuestionActions();
+ const [rows, setRows] = useState([]);
+ const [loadedKey, setLoadedKey] = useState(null);
+ const [query, setQuery] = useState("");
+
+ const sourceKey = useMemo(
+ () => serialize(serverQuestions),
+ [serverQuestions],
+ );
+ useEffect(() => {
+ if (loadedKey !== sourceKey) {
+ setRows(serverQuestions);
+ setLoadedKey(sourceKey);
+ }
+ }, [loadedKey, serverQuestions, sourceKey]);
+
+ const dirty = useMemo(() => serialize(rows) !== sourceKey, [rows, sourceKey]);
+ const search = query.trim().toLowerCase();
+ const visibleRows = useMemo(
+ () =>
+ search
+ ? rows.filter(
+ (row) =>
+ row.question.toLowerCase().includes(search) ||
+ row.answer.toLowerCase().includes(search),
+ )
+ : rows,
+ [rows, search],
+ );
+ const enabledCount = rows.filter(
+ (row) => row.enabled && row.question.trim() && row.answer.trim(),
+ ).length;
+
+ if (!isAdmin) {
+ return ;
+ }
+
+ const patchRow = (id: string, patch: Partial) => {
+ setRows((current) =>
+ current.map((row) => (row.id === id ? { ...row, ...patch } : row)),
+ );
+ };
+
+ const addRow = () => {
+ setRows((current) => [
+ {
+ id: newId(),
+ question: "",
+ answer: "",
+ enabled: true,
+ tokensPerSecond: 100,
+ sortOrder: current.length,
+ },
+ ...current,
+ ]);
+ };
+
+ const removeRow = (id: string) => {
+ setRows((current) => current.filter((row) => row.id !== id));
+ };
+
+ const handleSave = async () => {
+ const cleaned = rows.map((row, index) => ({
+ ...row,
+ question: row.question.trim(),
+ tokensPerSecond: Math.round(row.tokensPerSecond),
+ sortOrder: index,
+ }));
+ for (const row of cleaned) {
+ if (!row.question) {
+ toast.error("问题不能为空");
+ return;
+ }
+ if (!row.answer.trim()) {
+ toast.error(`问题「${row.question}」的回答不能为空`);
+ return;
+ }
+ if (
+ !Number.isFinite(row.tokensPerSecond) ||
+ row.tokensPerSecond < 1 ||
+ row.tokensPerSecond > 1000
+ ) {
+ toast.error(`问题「${row.question}」的流式速度须为 1–1000 token/s`);
+ return;
+ }
+ }
+ const seen = new Set();
+ for (const row of cleaned) {
+ if (seen.has(row.question)) {
+ toast.error(`问题「${row.question}」重复,请删除或修改重复项`);
+ return;
+ }
+ seen.add(row.question);
+ }
+ try {
+ await save.mutateAsync(cleaned);
+ toast.success("问题配置已保存并生效");
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : "保存失败");
+ }
+ };
+
+ return (
+
+
+
+
+
+
+
+
+
+
+
问题配置
+
+ {rows.length} 条 / {enabledCount} 条启用
+
+
+
+ 为完全相同的问题配置固定回答,命中时直接流式返回,不调用大模型。
+
+
+
+
+
+
+
+
+
+
+
+ 匹配区分大小写、标点和空格;停用的配置会保留但不参与匹配。带附件或图片的提问不会使用固定回答。
+
+
+
+
+
+
+ {isLoading ? (
+
+ 加载中…
+
+ ) : visibleRows.length === 0 ? (
+
+ {rows.length === 0
+ ? "暂无问题配置,点击「新增问题」开始配置"
+ : "没有匹配的问题或回答"}
+
+ ) : (
+
+ {visibleRows.map((row, index) => (
+
+
+
+
+ {index + 1}
+
+ 固定问答
+
+
+
+
+
+
+
+
+
+
+
+
+ ))}
+
+ )}
+
+
+ );
+}
diff --git a/frontend-web/src/pages/LlmWikiKnowledgePage.tsx b/frontend-web/src/pages/LlmWikiKnowledgePage.tsx
index a54b097e..2e3d745d 100644
--- a/frontend-web/src/pages/LlmWikiKnowledgePage.tsx
+++ b/frontend-web/src/pages/LlmWikiKnowledgePage.tsx
@@ -1,6 +1,6 @@
import { Loader2, Settings2 } from "lucide-react";
import { useState } from "react";
-import { Navigate } from "react-router-dom";
+import { Navigate, useParams, useSearchParams } from "react-router-dom";
import { Button } from "@/components/ui/button";
import {
@@ -10,6 +10,7 @@ import {
DialogTitle,
} from "@/components/ui/dialog";
import { getStoredAuth } from "@/core/auth";
+import { buildWeKnoraWikiPagePath } from "@/core/llmwiki/source-links";
import { useLlmWikiRuntime } from "@/strategy-components/api/llmwiki";
import WeKnoraKnowledgeWorkspace, {
ConnectionSettings,
@@ -66,7 +67,14 @@ export default function LlmWikiKnowledgePage({
}: {
mode?: LlmWikiKnowledgePageMode;
}) {
+ const { noteId = "" } = useParams();
+ const [searchParams] = useSearchParams();
const runtime = useLlmWikiRuntime();
+ const wikiSlug = searchParams.get("wikiSlug")?.trim() || "";
+
+ if (mode === "detail" && noteId && wikiSlug) {
+ return ;
+ }
if (runtime.isLoading) {
return ;
diff --git a/frontend-web/src/pages/StrategyRoutes.tsx b/frontend-web/src/pages/StrategyRoutes.tsx
index f8843bb4..792017cd 100644
--- a/frontend-web/src/pages/StrategyRoutes.tsx
+++ b/frontend-web/src/pages/StrategyRoutes.tsx
@@ -47,6 +47,7 @@ import LightAppListPage from "./LightAppListPage";
import LightAppManagementPage from "./LightAppManagementPage";
import MenuManagementPage from "./MenuManagementPage";
import LoginWhitelistPage from "./LoginWhitelistPage";
+import FixedQuestionsPage from "./FixedQuestionsPage";
import SensitiveWordsPage from "./SensitiveWordsPage";
import SkillAddressesPage from "./SkillAddressesPage";
import TaskButtonsPage from "./TaskButtonsPage";
@@ -454,6 +455,16 @@ export default function StrategyRoutes() {
)
}
/>
+
+ ) : (
+
+ )
+ }
+ />
}
/>
+
+
+
+ }
+ />
{
+ if (clarificationSeen) return;
setInteractiveMessages((prev) =>
prev.map((m) =>
m.id === agentMsgId
@@ -504,6 +510,7 @@ export function useStep1Intent(options: UseStep1IntentOptions): UseStep1IntentRe
);
},
onClarification: (clarification, index) => {
+ clarificationSeen = true;
// 实时把每条澄清问题灌进气泡 —— 用户能边流边看到「协助过程」逐个出现,
// 而不是等终态帧(终态帧只带第 1 个问题)。同时往步骤链补一条
// 「追问 N:<问题>」,与「强化步骤链」诉求对齐。
diff --git a/frontend-web/src/strategy-components/api/enterprise-research.ts b/frontend-web/src/strategy-components/api/enterprise-research.ts
index 1e5f4801..8a5dce55 100644
--- a/frontend-web/src/strategy-components/api/enterprise-research.ts
+++ b/frontend-web/src/strategy-components/api/enterprise-research.ts
@@ -1,10 +1,26 @@
import { apiFetch } from "@/core/api/fetch-client";
import { getBackendBaseURL } from "@/core/config";
-import type { LlmWikiSearchResult } from "./llmwiki";
-
const API = () => `${getBackendBaseURL()}/api/enterprise-research`;
+/**
+ * Enterprise research snapshots raw WeKnora evidence chunks. This is
+ * intentionally separate from LlmWikiSearchResult, whose public search shape
+ * now represents processed Wiki pages only.
+ */
+export interface EnterpriseResearchEvidence {
+ chunk_id: string;
+ content: string;
+ knowledge_id: string;
+ knowledge_base_id: string;
+ knowledge_base_name: string;
+ title: string;
+ filename: string;
+ score: number;
+ chunk_index: number;
+ url: string;
+}
+
export type EnterpriseResearchTaskStatus = "draft" | "collecting" | "ready" | "partial" | "failed";
export interface EnterpriseResearchTask {
@@ -15,7 +31,7 @@ export interface EnterpriseResearchTask {
knowledge_base_ids: string[];
queries: string[];
selected_source_ids: string[];
- sources: LlmWikiSearchResult[];
+ sources: EnterpriseResearchEvidence[];
plan_markdown?: string | null;
report_markdown?: string | null;
report_summary?: string | null;
diff --git a/frontend-web/src/strategy-components/api/fixed-questions.ts b/frontend-web/src/strategy-components/api/fixed-questions.ts
new file mode 100644
index 00000000..90e29cda
--- /dev/null
+++ b/frontend-web/src/strategy-components/api/fixed-questions.ts
@@ -0,0 +1,122 @@
+/** Administrator-managed exact-match fixed question/answer API. */
+
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
+
+import { apiFetch } from "@/core/api/fetch-client";
+import { getBackendBaseURL } from "@/core/config";
+import { OptType, writeYLog } from "@/core/log/y-log";
+
+const BASE = () => `${getBackendBaseURL()}/api/fixed-questions`;
+
+export interface FixedQuestion {
+ id: string;
+ question: string;
+ answer: string;
+ enabled: boolean;
+ tokensPerSecond: number;
+ sortOrder: number;
+ updatedBy?: string | null;
+ updatedAt?: string | null;
+}
+
+interface ServerFixedQuestion {
+ id: string;
+ question: string;
+ answer: string;
+ enabled: boolean;
+ tokens_per_second: number;
+ sort_order: number;
+ updated_by?: string | null;
+ updated_at?: string | null;
+}
+
+function fromServer(row: ServerFixedQuestion): FixedQuestion {
+ return {
+ id: row.id,
+ question: row.question ?? "",
+ answer: row.answer ?? "",
+ enabled: row.enabled ?? true,
+ tokensPerSecond: row.tokens_per_second ?? 100,
+ sortOrder: row.sort_order ?? 0,
+ updatedBy: row.updated_by ?? null,
+ updatedAt: row.updated_at ?? null,
+ };
+}
+
+function toServer(row: FixedQuestion): ServerFixedQuestion {
+ return {
+ id: row.id,
+ question: row.question,
+ answer: row.answer,
+ enabled: row.enabled,
+ tokens_per_second: row.tokensPerSecond,
+ sort_order: row.sortOrder,
+ };
+}
+
+async function readJson(response: Response, context: string): Promise {
+ if (!response.ok) {
+ const data = await response.json().catch(() => null as unknown);
+ const detail =
+ data && typeof data === "object" && "detail" in data
+ ? String((data as { detail: unknown }).detail)
+ : `${context} failed (${response.status})`;
+ throw new Error(detail);
+ }
+ return (await response.json()) as T;
+}
+
+export async function listFixedQuestions(): Promise {
+ const response = await apiFetch(BASE(), { method: "GET" });
+ const data = await readJson<{ questions: ServerFixedQuestion[] }>(
+ response,
+ "list fixed questions",
+ );
+ return (data.questions ?? []).map(fromServer);
+}
+
+export async function replaceFixedQuestions(
+ questions: FixedQuestion[],
+): Promise {
+ const response = await apiFetch(BASE(), {
+ method: "PUT",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ questions: questions.map(toServer) }),
+ });
+ const data = await readJson<{ questions: ServerFixedQuestion[] }>(
+ response,
+ "save fixed questions",
+ );
+ const saved = (data.questions ?? []).map(fromServer);
+ writeYLog({
+ optType: OptType.edit,
+ optDetail: "CM助手-全量-问题配置-保存",
+ url: "/api/fixed-questions",
+ reqParam: { questions },
+ returnParam: saved,
+ });
+ return saved;
+}
+
+const FIXED_QUESTIONS_KEY = "fixed-questions";
+
+export function useFixedQuestions() {
+ return useQuery({
+ queryKey: [FIXED_QUESTIONS_KEY],
+ queryFn: listFixedQuestions,
+ refetchOnWindowFocus: false,
+ staleTime: 30_000,
+ });
+}
+
+export function useFixedQuestionActions() {
+ const queryClient = useQueryClient();
+ return {
+ save: useMutation({
+ mutationFn: replaceFixedQuestions,
+ onSuccess: (questions) => {
+ queryClient.setQueryData([FIXED_QUESTIONS_KEY], questions);
+ },
+ }),
+ };
+}
diff --git a/frontend-web/src/strategy-components/api/llmwiki.ts b/frontend-web/src/strategy-components/api/llmwiki.ts
index 9f899052..481b2967 100644
--- a/frontend-web/src/strategy-components/api/llmwiki.ts
+++ b/frontend-web/src/strategy-components/api/llmwiki.ts
@@ -5,6 +5,7 @@ import { useLiveAccount } from "@/core/auth/use-live-account";
import { getBackendBaseURL } from "@/core/config";
const API = () => `${getBackendBaseURL()}/api/llmwiki`;
+const PUBLIC_API = () => `${getBackendBaseURL()}/api/public/llmwiki`;
const SETTINGS_API = () => `${getBackendBaseURL()}/api/system-settings/llmwiki`;
export type LlmWikiProvider = "legacy" | "weknora";
@@ -143,6 +144,22 @@ export interface LlmWikiWikiPageList {
total_pages: number;
}
+export interface LlmWikiWikiIndexStatus {
+ knowledge_base_id: string;
+ state: "not_synced" | "idle" | "running" | "partial" | "failed" | "rebuilding" | "rebuild_required";
+ local_index_enabled: boolean;
+ wiki_index_enabled: boolean;
+ fully_vectorized: boolean;
+ retrieval_mode: "vector" | "wiki_api";
+ local_page_count?: number;
+ ready_page_count?: number;
+ failed_page_count?: number;
+ vector_count?: number;
+ last_error?: string | null;
+ last_started_at?: string | null;
+ last_completed_at?: string | null;
+}
+
export interface LlmWikiSourceContext {
knowledge_base_name?: string;
document_title?: string;
@@ -159,16 +176,38 @@ export interface LlmWikiSourceContext {
}
export interface LlmWikiSearchResult {
- chunk_id: string;
+ kind: "wiki_page";
+ id: string;
+ wiki_page_id: string;
+ wiki_slug: string;
content: string;
- knowledge_id: string;
knowledge_base_id: string;
knowledge_base_name: string;
title: string;
- filename: string;
+ summary?: string;
+ page_type?: string;
+ page_version?: number;
+ updated_at?: string | null;
+ matched_heading?: string | null;
+ matched_sections?: Array<{
+ section_index: number;
+ heading?: string | null;
+ content: string;
+ score: number;
+ }>;
score: number;
- chunk_index: number;
url: string;
+ retrieval_mode?: "vector" | "wiki_api";
+}
+
+export interface LlmWikiSearchResponse {
+ query: string;
+ mode: "vector" | "wiki_api" | "mixed";
+ results: LlmWikiSearchResult[];
+ knowledge_base_count: number;
+ partial: boolean;
+ warnings: string[];
+ fallback_knowledge_base_ids: string[];
}
export interface LlmWikiWeKnoraEmbedSession {
@@ -203,14 +242,22 @@ export interface LlmWikiWikiPageUpdate {
wiki_path?: string;
}
+function userFacingKnowledgeMessage(value: string): string {
+ return value.replace(/weknora/gi, "知识服务");
+}
+
async function readJson(response: Response, context: string): Promise {
if (!response.ok) {
const payload = await response.json().catch(() => null as unknown);
- const detail =
- payload && typeof payload === "object" && "detail" in payload
- ? String((payload as { detail: unknown }).detail)
+ const rawDetail = payload && typeof payload === "object" && "detail" in payload
+ ? (payload as { detail: unknown }).detail
+ : null;
+ const detail = rawDetail && typeof rawDetail === "object" && "message" in rawDetail
+ ? String((rawDetail as { message: unknown }).message)
+ : rawDetail
+ ? String(rawDetail)
: `${context}失败(${response.status})`;
- throw new Error(detail);
+ throw new Error(userFacingKnowledgeMessage(detail));
}
if (response.status === 204) return undefined as T;
return (await response.json()) as T;
@@ -266,7 +313,7 @@ export async function createLlmWikiWeKnoraEmbedSession(
`${API()}/knowledge-bases/${encodeURIComponent(id)}/weknora/session?tab=${encodeURIComponent(tab)}`,
{ method: "POST" },
),
- "创建 WeKnora 详情会话",
+ "创建知识库详情会话",
);
}
@@ -412,6 +459,66 @@ export async function getLlmWikiWikiPage(
);
}
+export async function getLlmWikiWikiIndexStatus(
+ knowledgeBaseId: string,
+): Promise {
+ return readJson(
+ await apiFetch(
+ `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/wiki-index-status`,
+ ),
+ "读取 Wiki 向量化状态",
+ );
+}
+
+export async function vectorizeLlmWikiKnowledgeBase(
+ knowledgeBaseId: string,
+): Promise<{ job_id: string; status: string; knowledge_base_id: string }> {
+ return readJson(
+ await apiFetch(
+ `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/vectorize`,
+ { method: "POST" },
+ ),
+ "触发 Wiki 向量化",
+ );
+}
+
+export async function exportLlmWikiWikiExcel(
+ knowledgeBaseId: string,
+ knowledgeBaseName = "LLMWiki",
+): Promise {
+ const response = await apiFetch(
+ `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/wiki/export.xlsx`,
+ );
+ if (!response.ok) {
+ await readJson(response, "导出 Wiki Excel");
+ return;
+ }
+ const blob = await response.blob();
+ const objectUrl = URL.createObjectURL(blob);
+ const link = document.createElement("a");
+ link.href = objectUrl;
+ link.download = `${knowledgeBaseName.replace(/[\\/:*?"<>|]/g, "_") || "LLMWiki"}-Wiki.xlsx`;
+ document.body.appendChild(link);
+ link.click();
+ link.remove();
+ URL.revokeObjectURL(objectUrl);
+}
+
+/** Read the DeerFlow-local Wiki mirror used by new vector citations. */
+export async function getLocalLlmWikiWikiPage(
+ knowledgeBaseId: string,
+ slug: string,
+): Promise {
+ const safeSlug = slug.split("/").map(encodeURIComponent).join("/");
+ const page = await readJson(
+ await apiFetch(
+ `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/local-wiki/pages/${safeSlug}`,
+ ),
+ "读取本地 Wiki 页面详情",
+ );
+ return { ...page, version: page.version ?? page.page_version ?? 0 };
+}
+
export async function createLlmWikiWikiPage(
knowledgeBaseId: string,
input: LlmWikiWikiPageCreate,
@@ -554,8 +661,8 @@ export async function searchLlmWiki(input: {
query: string;
scope: LlmWikiScope;
knowledge_base_ids?: string[];
-}): Promise {
- const payload = await readJson<{ results: LlmWikiSearchResult[] }>(
+}): Promise {
+ return readJson(
await apiFetch(`${API()}/search`, {
method: "POST",
headers: { "Content-Type": "application/json" },
@@ -563,7 +670,44 @@ export async function searchLlmWiki(input: {
}),
"检索知识库",
);
- return payload.results ?? [];
+}
+
+export async function getPublicLlmWikiKnowledgeBase(
+ id: string,
+): Promise {
+ return readJson(
+ await apiFetch(`${PUBLIC_API()}/knowledge-bases/${encodeURIComponent(id)}`),
+ "读取公开知识库详情",
+ );
+}
+
+export async function listPublicLlmWikiWikiPages(
+ knowledgeBaseId: string,
+ page: number,
+ pageSize: number,
+ query = "",
+): Promise {
+ const params = new URLSearchParams({ page: String(page), page_size: String(pageSize) });
+ if (query.trim()) params.set("query", query.trim());
+ return readJson(
+ await apiFetch(
+ `${PUBLIC_API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/wiki/pages?${params.toString()}`,
+ ),
+ "读取公开 Wiki 页面",
+ );
+}
+
+export async function getPublicLlmWikiWikiPage(
+ knowledgeBaseId: string,
+ slug: string,
+): Promise {
+ const safeSlug = slug.split("/").map(encodeURIComponent).join("/");
+ return readJson(
+ await apiFetch(
+ `${PUBLIC_API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/wiki/pages/${safeSlug}`,
+ ),
+ "读取公开 Wiki 页面详情",
+ );
}
export async function createLlmWikiConversationDeposit(input: {
@@ -586,7 +730,7 @@ export async function createLlmWikiConversationDeposit(input: {
}
export async function getLlmWikiSettings(): Promise {
- return readJson(await apiFetch(SETTINGS_API()), "读取 WeKnora 连接配置");
+ return readJson(await apiFetch(SETTINGS_API()), "读取知识服务连接配置");
}
export async function saveLlmWikiSettings(input: {
@@ -600,7 +744,7 @@ export async function saveLlmWikiSettings(input: {
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
- "保存 WeKnora 连接配置",
+ "保存知识服务连接配置",
);
}
@@ -620,7 +764,7 @@ export async function testLlmWikiConnection(
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ api_base_url: apiBaseUrl }),
}),
- "测试 WeKnora 连接",
+ "测试知识服务连接",
);
}
@@ -755,6 +899,62 @@ export function useLlmWikiWikiPage(
});
}
+export function usePublicLlmWikiKnowledgeBase(
+ knowledgeBaseId: string | null,
+ enabled = true,
+) {
+ return useQuery({
+ queryKey: ["public-llmwiki", "knowledge-base", knowledgeBaseId],
+ queryFn: () => getPublicLlmWikiKnowledgeBase(knowledgeBaseId!),
+ enabled: enabled && !!knowledgeBaseId,
+ staleTime: 60_000,
+ });
+}
+
+export function usePublicLlmWikiWikiPages(
+ knowledgeBaseId: string | null,
+ page: number,
+ pageSize = 50,
+ query = "",
+ enabled = true,
+) {
+ return useQuery({
+ queryKey: ["public-llmwiki", "wiki-pages", knowledgeBaseId, page, pageSize, query],
+ queryFn: () => listPublicLlmWikiWikiPages(knowledgeBaseId!, page, pageSize, query),
+ enabled: enabled && !!knowledgeBaseId,
+ staleTime: 60_000,
+ });
+}
+
+export function usePublicLlmWikiWikiPage(
+ knowledgeBaseId: string | null,
+ slug: string | null,
+ enabled = true,
+) {
+ return useQuery({
+ queryKey: ["public-llmwiki", "wiki-page", knowledgeBaseId, slug],
+ queryFn: () => getPublicLlmWikiWikiPage(knowledgeBaseId!, slug!),
+ enabled: enabled && !!knowledgeBaseId && !!slug,
+ staleTime: 60_000,
+ });
+}
+
+export function useLlmWikiWikiIndexStatus(
+ knowledgeBaseId: string | null,
+ enabled = true,
+) {
+ const { userId } = useLiveAccount();
+ return useQuery({
+ queryKey: ["llmwiki", "wiki-index-status", knowledgeBaseId, userId ?? "anonymous"],
+ queryFn: () => getLlmWikiWikiIndexStatus(knowledgeBaseId!),
+ enabled: enabled && !!knowledgeBaseId,
+ refetchInterval: (query) => {
+ const state = query.state.data?.state;
+ return state === "running" || state === "rebuilding" ? 2_000 : 10_000;
+ },
+ });
+}
+
export function useLlmWikiSettings(enabled = true) {
const { userId } = useLiveAccount();
return useQuery({
@@ -822,6 +1022,14 @@ export function useLlmWikiActions() {
deleteLlmWikiWikiPage(id, slug),
onSuccess: invalidateKnowledge,
}),
+ vectorizeWiki: useMutation({
+ mutationFn: ({ id }: { id: string }) => vectorizeLlmWikiKnowledgeBase(id),
+ onSuccess: invalidateKnowledge,
+ }),
+ exportWikiExcel: useMutation({
+ mutationFn: ({ id, name }: { id: string; name?: string }) =>
+ exportLlmWikiWikiExcel(id, name),
+ }),
publication: useMutation({
mutationFn: ({ id, publish }: { id: string; publish: boolean }) =>
setLlmWikiPublication(id, publish),
diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeDetailPage.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeDetailPage.tsx
index 9f2ae5b4..8def6a95 100644
--- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeDetailPage.tsx
+++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeDetailPage.tsx
@@ -1,4 +1,4 @@
-import { AlertCircle, Edit3, FileText, Loader2, RefreshCw, Trash2, Upload } from "lucide-react";
+import { AlertCircle, Edit3, FileText, Loader2, RefreshCw, Trash2 } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { Navigate, useNavigate, useParams, useSearchParams } from "react-router-dom";
import { toast } from "sonner";
@@ -14,6 +14,7 @@ import { Input } from "@/components/ui/input";
import { Textarea } from "@/components/ui/textarea";
import { MarkdownCodeEditor } from "@/components/wiki/MarkdownCodeEditor";
import { getBackendBaseURL } from "@/core/config";
+import { buildWeKnoraWikiPagePath } from "@/core/llmwiki/source-links";
import {
createLlmWikiWeKnoraEmbedSession,
getLlmWikiSourceContext,
@@ -100,14 +101,11 @@ export function WeKnoraKnowledgeDetailPage() {
const [searchParams] = useSearchParams();
const navigate = useNavigate();
const frameRef = useRef(null);
- const fileInputRef = useRef(null);
const [frameSrc, setFrameSrc] = useState("");
const [frameVersion, setFrameVersion] = useState(0);
const [frameError, setFrameError] = useState(null);
- const [uploading, setUploading] = useState(false);
const runtime = useLlmWikiRuntime();
const details = useLlmWikiKnowledgeBase(knowledgeBaseId || null, !!knowledgeBaseId);
- const actions = useLlmWikiActions();
const detailsReady = !!details.data;
const requestedTab = searchParams.get("tab");
@@ -120,12 +118,15 @@ export function WeKnoraKnowledgeDetailPage() {
const sourceChunkId = searchParams.get("sourceChunkId") || searchParams.get("chunkId") || "";
const documentId = searchParams.get("documentId") || "";
const wikiSlug = searchParams.get("wikiSlug") || "";
+ const nativeWikiPageUrl = knowledgeBaseId && wikiSlug && preferredTab === "wiki"
+ ? buildWeKnoraWikiPagePath(knowledgeBaseId, wikiSlug)
+ : "";
useEffect(() => {
let cancelled = false;
setFrameSrc("");
setFrameError(null);
- if (!knowledgeBaseId || runtime.data?.provider !== "weknora" || !detailsReady) {
+ if (nativeWikiPageUrl || !knowledgeBaseId || runtime.data?.provider !== "weknora" || !detailsReady) {
return () => {
cancelled = true;
};
@@ -158,7 +159,7 @@ export function WeKnoraKnowledgeDetailPage() {
return () => {
cancelled = true;
};
- }, [detailsReady, documentId, knowledgeBaseId, preferredTab, runtime.data?.provider, sourceChunkId, wikiSlug]);
+ }, [detailsReady, documentId, knowledgeBaseId, nativeWikiPageUrl, preferredTab, runtime.data?.provider, sourceChunkId, wikiSlug]);
useEffect(() => {
if (!frameSrc) return;
@@ -178,6 +179,10 @@ export function WeKnoraKnowledgeDetailPage() {
};
}, [frameSrc]);
+ if (nativeWikiPageUrl) {
+ return ;
+ }
+
if (runtime.isLoading || details.isLoading) {
return (
@@ -225,53 +230,15 @@ export function WeKnoraKnowledgeDetailPage() {
);
}
- const canUpload = details.data.can_write && !details.data.is_conversation_deposit;
+ const canWrite = details.data.can_write && !details.data.is_conversation_deposit;
const canEditCurrentWikiArticle =
- canUpload && !!details.data.indexing_strategy?.wiki_enabled && !!wikiSlug;
- const canManageWiki = canUpload && !!details.data.indexing_strategy?.wiki_enabled;
+ canWrite && !!details.data.indexing_strategy?.wiki_enabled && !!wikiSlug;
+ const canManageWiki = canWrite && !!details.data.indexing_strategy?.wiki_enabled;
return (
- {(canUpload || canEditCurrentWikiArticle) && (
+ {canManageWiki && (
-
{
- const files = Array.from(event.currentTarget.files ?? []);
- event.currentTarget.value = "";
- if (files.length === 0) return;
- void (async () => {
- setUploading(true);
- const failed: string[] = [];
- try {
- for (const file of files) {
- try {
- await actions.upload.mutateAsync({ id: knowledgeBaseId, file });
- } catch {
- failed.push(file.name);
- }
- }
- const succeeded = files.length - failed.length;
- if (succeeded > 0) {
- await details.refetch();
- setFrameVersion((value) => value + 1);
- }
- if (failed.length === 0) {
- toast.success(`已上传 ${succeeded} 个文件,正在处理`);
- } else if (succeeded > 0) {
- toast.warning(`上传完成:成功 ${succeeded} 个,失败 ${failed.length} 个`);
- } else {
- toast.error("文件上传失败");
- }
- } finally {
- setUploading(false);
- }
- })();
- }}
- />
{canEditCurrentWikiArticle && (
- {canManageWiki && (
Wiki管理
- )}
- {canUpload && (
-
- )}
)}
@@ -323,7 +273,7 @@ export function WeKnoraKnowledgeDetailPage() {
title={`${details.data.name || "知识库"} 详情`}
src={frameSrc}
className="h-full min-h-0 w-full flex-1 border-0 bg-background"
- sandbox="allow-same-origin allow-scripts allow-forms allow-downloads allow-popups"
+ sandbox="allow-same-origin allow-scripts allow-forms allow-downloads allow-popups allow-popups-to-escape-sandbox"
referrerPolicy="no-referrer"
onLoad={() => postThemeToFrame(frameRef.current)}
/>
diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeWorkspace.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeWorkspace.tsx
index 04a20482..e320a272 100644
--- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeWorkspace.tsx
+++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeWorkspace.tsx
@@ -1,5 +1,7 @@
import {
BookOpen,
+ BookOpenText,
+ ChevronRight,
Database,
FileText,
Globe2,
@@ -29,8 +31,11 @@ import { Input } from "@/components/ui/input";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Textarea } from "@/components/ui/textarea";
import { getStoredAuth } from "@/core/auth";
+import { WeKnoraSourceDrawer } from "@/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer";
import {
type LlmWikiKnowledgeBase,
+ type LlmWikiSearchResponse,
+ type LlmWikiSearchResult,
type LlmWikiScope,
searchLlmWiki,
useLlmWikiActions,
@@ -53,6 +58,49 @@ function messageOf(error: unknown): string {
return error instanceof Error ? error.message : "操作失败,请稍后重试";
}
+function searchModeLabel(mode: LlmWikiSearchResponse["mode"]) {
+ if (mode === "vector") return "本地 Wiki 向量检索";
+ if (mode === "mixed") return "Wiki 混合检索";
+ return "Wiki 接口检索";
+}
+
+function wikiPageTypeLabel(value?: string) {
+ const normalized = (value || "").trim().toLowerCase();
+ if (normalized === "directory") return "目录页";
+ if (normalized === "overview") return "概览页";
+ if (normalized === "article") return "文章页";
+ return value?.trim() || "Wiki 页面";
+}
+
+function formatSearchTime(value?: string | null) {
+ if (!value) return "";
+ const date = new Date(value);
+ return Number.isNaN(date.getTime())
+ ? value
+ : date.toLocaleString("zh-CN", { hour12: false });
+}
+
+function resultPreview(result: LlmWikiSearchResult) {
+ const text = (
+ result.matched_sections?.map((section) => section.content).filter(Boolean).join("\n\n") ||
+ result.summary ||
+ result.content ||
+ ""
+ );
+ return text
+ .replace(/^---[\s\S]*?---\s*/m, "")
+ .replace(/```[\s\S]*?```/g, " ")
+ .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")
+ .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
+ .replace(/\[\[[^\]\s|]+[|\s]+([^\]]+)\]\]/g, "$1")
+ .replace(/\[\[([^\]]+)\]\]/g, "$1")
+ .replace(/^#{1,6}\s+/gm, "")
+ .replace(/^\s*[-*+]\s+/gm, "")
+ .replace(/[>*_`~|]/g, " ")
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
export function WeKnoraKnowledgeWorkspace() {
const navigate = useNavigate();
const isAdmin = getStoredAuth()?.system_role === "admin";
@@ -63,15 +111,13 @@ export function WeKnoraKnowledgeWorkspace() {
const [createOpen, setCreateOpen] = useState(false);
const [searchText, setSearchText] = useState("");
const [searching, setSearching] = useState(false);
- const [searchResults, setSearchResults] = useState
>([]);
+ const [searchResponse, setSearchResponse] = useState(null);
+ const [selectedSearchResult, setSelectedSearchResult] = useState(null);
- useEffect(() => setSearchResults([]), [tab]);
+ useEffect(() => {
+ setSearchResponse(null);
+ setSelectedSearchResult(null);
+ }, [tab]);
const busy = actions.create.isPending || actions.remove.isPending || actions.publication.isPending;
const openDetails = (item: LlmWikiKnowledgeBase) => {
@@ -87,7 +133,7 @@ export function WeKnoraKnowledgeWorkspace() {
if (!query) return;
setSearching(true);
try {
- setSearchResults(await searchLlmWiki({ query, scope }));
+ setSearchResponse(await searchLlmWiki({ query, scope }));
} catch (error) {
toast.error(messageOf(error));
} finally {
@@ -154,9 +200,9 @@ export function WeKnoraKnowledgeWorkspace() {
)}
@@ -166,15 +212,95 @@ export function WeKnoraKnowledgeWorkspace() {
{isAdmin &&
}
- {searchResults.length > 0 && tab !== "settings" && (
-
- 检索结果结果由知识服务检索返回,问答中使用同一授权链路。
- {searchResults.map((result) => {result.title || "未命名来源"}{result.knowledge_base_name}
{result.content}
)}
+ {searchResponse && tab !== "settings" && (
+
+
+
+
+
+
+
+
+
+ Wiki 文档检索
+
+ {searchResponse.results.length} 个 Wiki 页面
+
+
+
+ 本次直接检索知识库生成的 Wiki 页面,不检索上传原文或原文分块。
+
+
+
+
+ {searchModeLabel(searchResponse.mode)}
+
+
+
+
+ {searchResponse.results.length === 0 ? (
+
+
+
已完成 Wiki 文档检索,暂未命中相关页面
+
可以缩短关键词或换一个主题后重试。
+
+ ) : searchResponse.results.map((result) => {
+ const heading = result.matched_heading || result.matched_sections?.find((section) => section.heading)?.heading;
+ const updatedAt = formatSearchTime(result.updated_at);
+ return (
+
+ );
+ })}
+
)}
+ {
+ if (!open) setSelectedSearchResult(null);
+ }}
+ />
+
{
try {
const created = await actions.create.mutateAsync(input);
diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx
index b88ee2d6..61a63d6e 100644
--- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx
+++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx
@@ -1,5 +1,14 @@
-import { Download, FileText, Loader2, Maximize2, Minimize2 } from "lucide-react";
+import {
+ BookOpenText,
+ Download,
+ FileDown,
+ FileText,
+ Loader2,
+ Maximize2,
+ Minimize2,
+} from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
+import { Link } from "react-router-dom";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
@@ -17,10 +26,21 @@ import { SpreadsheetPreview } from "@/components/workspace/artifacts/spreadsheet
import { MarkdownContent } from "@/components/workspace/messages/markdown-content";
import { cn } from "@/lib/utils";
import { reasoningPlugins } from "@/core/streamdown";
+import { buildWeKnoraWikiPagePath } from "@/core/llmwiki/source-links";
+import {
+ exportableWikiMarkdown,
+ renderableWikiMarkdown,
+ safeWikiFileName,
+ stripWikiFrontmatter,
+ wikiLinkMarkdown,
+} from "@/core/llmwiki/wiki-page-markdown";
+import { markdownToDocxBlob } from "@/core/scheduled-tasks/markdown-docx";
import {
getLlmWikiDocument,
getLlmWikiDocumentPreview,
+ getLlmWikiWikiIndexStatus,
getLlmWikiWikiPage,
+ getLocalLlmWikiWikiPage,
listLlmWikiDocumentChunks,
type LlmWikiChunk,
type LlmWikiDocument,
@@ -61,8 +81,34 @@ type CachedSourceDetail = {
const sourceDetailCache = new Map();
+async function getAvailableWikiPage(
+ knowledgeBaseId: string,
+ wikiSlug: string,
+): Promise {
+ const status = await getLlmWikiWikiIndexStatus(knowledgeBaseId).catch(() => null);
+ if (status?.local_index_enabled) {
+ return getLocalLlmWikiWikiPage(knowledgeBaseId, wikiSlug).catch(() =>
+ getLlmWikiWikiPage(knowledgeBaseId, wikiSlug),
+ );
+ }
+ if (status?.local_index_enabled === false) {
+ return getLlmWikiWikiPage(knowledgeBaseId, wikiSlug);
+ }
+ try {
+ return await getLlmWikiWikiPage(knowledgeBaseId, wikiSlug);
+ } catch (remoteError) {
+ try {
+ return await getLocalLlmWikiWikiPage(knowledgeBaseId, wikiSlug);
+ } catch {
+ // Do not mask the useful remote-page error with a local-index-disabled
+ // response when the status request itself was unavailable.
+ throw remoteError;
+ }
+ }
+}
+
async function resolveTargetDetail(target: WeKnoraSourceTarget): Promise {
- if (target.documentId || target.wikiSlug || !target.chunkId) {
+ if (target.wikiSlug || !target.chunkId) {
return target;
}
const params = new URLSearchParams({
@@ -74,12 +120,14 @@ async function resolveTargetDetail(target: WeKnoraSourceTarget): Promise null)) as SourceContextResponse | null;
if (!response.ok || payload?.success !== true) {
- throw new Error("无法定位原文文档详情");
+ if (target.documentId) return target;
+ throw new Error("无法定位当前引用详情");
}
const documentId = payload.data?.document_id?.trim();
const wikiSlug = payload.data?.wiki_page?.slug?.trim();
if (!documentId && !wikiSlug) {
- throw new Error("当前引用缺少原文文档标识,无法打开详情");
+ if (target.documentId) return target;
+ throw new Error("当前引用缺少 Wiki 页面或文档标识,无法打开详情");
}
return {
...target,
@@ -96,8 +144,14 @@ function formatDate(value?: string | null) {
}
function displayType(document?: LlmWikiDocument | null, wiki?: LlmWikiWikiPage | null) {
+ if (wiki) {
+ const pageType = String(wiki.page_type || "").toLowerCase();
+ if (pageType === "directory") return "Wiki 目录页";
+ if (pageType === "overview") return "Wiki 概览页";
+ if (pageType === "article") return "Wiki 文章页";
+ return "Wiki 文档";
+ }
return (
- wiki?.page_type ||
document?.file_type ||
document?.type ||
document?.source ||
@@ -109,8 +163,23 @@ function cleanText(value?: string | null) {
return (value || "").replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
}
-function stripFrontmatter(value: string) {
- return value.replace(/^\s*---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, "").trim();
+function downloadMarkdown(markdown: string, fileName: string) {
+ downloadBlob(
+ new Blob([markdown], { type: "text/markdown;charset=utf-8" }),
+ `${safeWikiFileName(fileName)}.md`,
+ );
+}
+
+function downloadBlob(blob: Blob, fileName: string) {
+ const url = URL.createObjectURL(blob);
+ const anchor = window.document.createElement("a");
+ anchor.href = url;
+ anchor.download = fileName;
+ anchor.rel = "noopener";
+ window.document.body.appendChild(anchor);
+ anchor.click();
+ anchor.remove();
+ window.setTimeout(() => URL.revokeObjectURL(url), 30_000);
}
function fileExtension(document?: LlmWikiDocument | null) {
@@ -183,7 +252,7 @@ function isWordDocument(document?: LlmWikiDocument | null) {
}
function markdownFromChunks(chunks: LlmWikiChunk[]) {
- return stripFrontmatter(
+ return stripWikiFrontmatter(
chunks.map((chunk) => cleanText(chunk.content)).filter(Boolean).join("\n\n"),
);
}
@@ -468,6 +537,7 @@ export function WeKnoraSourceDrawer({
const [previewLoading, setPreviewLoading] = useState(false);
const [fullscreen, setFullscreen] = useState(false);
const [downloading, setDownloading] = useState(false);
+ const [wikiDownload, setWikiDownload] = useState<"md" | "word" | null>(null);
const [resolvedDocumentId, setResolvedDocumentId] = useState("");
const targetKey = useMemo(() => {
@@ -518,7 +588,9 @@ export function WeKnoraSourceDrawer({
void (async () => {
const detailTarget = await resolveTargetDetail(target);
- if (!cancelled) setResolvedDocumentId(detailTarget.documentId || "");
+ const isWikiTarget = Boolean(detailTarget.wikiSlug);
+ const previewDocumentId = isWikiTarget ? "" : detailTarget.documentId || "";
+ if (!cancelled) setResolvedDocumentId(previewDocumentId);
const tasks: Promise[] = [];
let loadedDocument: LlmWikiDocument | null = null;
let loadedWikiPage: LlmWikiWikiPage | null = null;
@@ -528,24 +600,23 @@ export function WeKnoraSourceDrawer({
if (detailTarget.wikiSlug) {
tasks.push(
- getLlmWikiWikiPage(detailTarget.knowledgeBaseId, detailTarget.wikiSlug)
+ getAvailableWikiPage(detailTarget.knowledgeBaseId, detailTarget.wikiSlug)
.then((page) => {
loadedWikiPage = page;
if (!cancelled) setWikiPage(page);
- })
- .catch(() => undefined),
+ }),
);
}
- if (detailTarget.documentId) {
+ if (previewDocumentId) {
loadedDocument = await getLlmWikiDocument(
detailTarget.knowledgeBaseId,
- detailTarget.documentId,
+ previewDocumentId,
);
if (!cancelled) setDocument(loadedDocument);
loadedChunks = await listLlmWikiDocumentChunks(
detailTarget.knowledgeBaseId,
- detailTarget.documentId,
+ previewDocumentId,
1,
200,
)
@@ -557,7 +628,7 @@ export function WeKnoraSourceDrawer({
}
const previewBlobPromise = isWordDocument(loadedDocument)
? Promise.resolve(new Blob())
- : getLlmWikiDocumentPreview(detailTarget.knowledgeBaseId, detailTarget.documentId);
+ : getLlmWikiDocumentPreview(detailTarget.knowledgeBaseId, previewDocumentId);
const previewTask = previewBlobPromise
.then((blob) => {
return buildPreviewState(
@@ -585,11 +656,11 @@ export function WeKnoraSourceDrawer({
}
if (tasks.length === 0) {
- throw new Error("当前引用缺少原文文档标识,无法打开详情");
+ throw new Error("当前引用缺少 Wiki 页面或文档标识,无法打开详情");
}
await Promise.all(tasks);
sourceDetailCache.set(targetKey, {
- documentId: detailTarget.documentId || "",
+ documentId: previewDocumentId,
document: loadedDocument,
wikiPage: loadedWikiPage,
preview: loadedPreview,
@@ -617,7 +688,20 @@ export function WeKnoraSourceDrawer({
title ||
"文档详情";
const summary = cleanText(wikiPage?.summary || document?.description);
- const wikiContent = stripFrontmatter(cleanText(wikiPage?.content));
+ const wikiContent = stripWikiFrontmatter(cleanText(wikiPage?.content));
+ const wikiDisplayContent = wikiPage && target
+ ? renderableWikiMarkdown(resolvedTitle, wikiContent, target.knowledgeBaseId)
+ : "";
+ const wikiMarkdown = wikiPage
+ ? wikiLinkMarkdown(
+ exportableWikiMarkdown(resolvedTitle, summary, wikiContent),
+ target?.knowledgeBaseId || "",
+ )
+ : "";
+ const resolvedWikiSlug = wikiPage?.slug || target?.wikiSlug || "";
+ const wikiWorkspacePath = target && resolvedWikiSlug
+ ? buildWeKnoraWikiPagePath(target.knowledgeBaseId, resolvedWikiSlug)
+ : "";
const downloadOriginal = async () => {
if (!target || !resolvedDocumentId) return;
@@ -639,6 +723,28 @@ export function WeKnoraSourceDrawer({
}
};
+ const downloadWiki = async (format: "md" | "word") => {
+ if (!wikiMarkdown) {
+ toast.error("当前 Wiki 页面没有可下载内容");
+ return;
+ }
+ setWikiDownload(format);
+ try {
+ if (format === "md") {
+ downloadMarkdown(wikiMarkdown, resolvedTitle);
+ toast.success("Markdown 文件已生成");
+ } else {
+ const blob = await markdownToDocxBlob(wikiMarkdown, resolvedTitle);
+ downloadBlob(blob, `${safeWikiFileName(resolvedTitle)}.docx`);
+ toast.success("Word 文件已生成");
+ }
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : "下载 Wiki 页面失败");
+ } finally {
+ setWikiDownload(null);
+ }
+ };
+
return (
{resolvedTitle}
- 知识库文档详情
+ 知识库 Wiki 页面详情
-
-
-
-
-
-
-
{resolvedTitle}
-
-
{displayType(document, wikiPage)}
- {formatDate(document?.created_at || wikiPage?.created_at) && (
-
{formatDate(document?.created_at || wikiPage?.created_at)}
- )}
- {document?.parse_status &&
{document.parse_status}}
+
+
+
+
+ {wikiPage ? : }
+
+
+
{resolvedTitle}
+
+ {displayType(document, wikiPage)}
+ {formatDate(document?.created_at || wikiPage?.created_at) && (
+ {formatDate(document?.created_at || wikiPage?.created_at)}
+ )}
+ {document?.parse_status && {document.parse_status}}
+
-
-
- {resolvedDocumentId && (
-
{loading && (
- 正在加载完整文档预览
+ 正在加载完整 Wiki 文档
)}
@@ -780,13 +912,16 @@ export function WeKnoraSourceDrawer({
) : null}
- {wikiContent && (
+ {wikiDisplayContent && (
- Wiki 原文
+
+
+ Wiki 文档正文
+
{
+ if (!canWrite) return;
+ try {
+ await actions.vectorizeWiki.mutateAsync({ id: noteId });
+ toast.success("已提交整库 Wiki 向量化任务");
+ await indexStatus.refetch();
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : "触发向量化失败");
+ }
+ };
+
+ const exportExcel = async () => {
+ try {
+ await actions.exportWikiExcel.mutateAsync({ id: noteId, name: title });
+ toast.success("Wiki 已全部导出为 Excel");
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : "导出 Wiki Excel 失败");
+ }
+ };
+
+ const vectorStatusText = !indexStatus.data?.local_index_enabled
+ ? "本地向量未启用,检索使用 Wiki 接口"
+ : indexStatus.data.fully_vectorized
+ ? `整库已向量化(${indexStatus.data.vector_count ?? 0} 个向量章节)`
+ : indexStatus.data.state === "running" || indexStatus.data.state === "rebuilding"
+ ? "整库向量化进行中,完成前检索使用 Wiki 接口"
+ : "整库尚未完成向量化,检索使用 Wiki 接口";
+
return (
@@ -571,9 +601,26 @@ export function WeKnoraWikiManagementPage() {
{title}
- Wiki 文档新增、编辑、删除
+ Wiki 文档新增、编辑、删除 · {vectorStatusText}
+
void vectorize()}
+ disabled={!canWrite || actions.vectorizeWiki.isPending || indexStatus.data?.state === "running" || indexStatus.data?.state === "rebuilding"}
+ >
+ {actions.vectorizeWiki.isPending || indexStatus.data?.state === "running" || indexStatus.data?.state === "rebuilding" ? (
+
+ ) : (
+
+ )}
+ 手动向量化
+
+
void exportExcel()} disabled={actions.exportWikiExcel.isPending}>
+ {actions.exportWikiExcel.isPending ? : }
+ 导出 Excel
+
void pages.refetch()}>
刷新
diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx
new file mode 100644
index 00000000..489ff7d6
--- /dev/null
+++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx
@@ -0,0 +1,479 @@
+import {
+ ArrowLeft,
+ BookOpenText,
+ ChevronDown,
+ ChevronRight,
+ Download,
+ FileText,
+ Folder,
+ Loader2,
+ RefreshCw,
+ Search,
+} from "lucide-react";
+import { type ReactNode, useEffect, useMemo, useState } from "react";
+import { Link, useLocation, useNavigate, useParams, useSearchParams } from "react-router-dom";
+import { toast } from "sonner";
+
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { MarkdownContent } from "@/components/workspace/messages/markdown-content";
+import {
+ buildPublicKnowledgeWikiPath,
+ buildWeKnoraWikiPagePath,
+} from "@/core/llmwiki/source-links";
+import {
+ buildWikiTree,
+ filterWikiPagesWithAncestors,
+ type WikiTreeNode,
+} from "@/core/llmwiki/wiki-tree";
+import { usePublicBackground } from "@/core/embed/background";
+import { reasoningPlugins } from "@/core/streamdown";
+import { markdownToDocxBlob } from "@/core/scheduled-tasks/markdown-docx";
+import {
+ exportableWikiMarkdown,
+ renderableWikiMarkdown,
+ safeWikiFileName,
+ stripWikiFrontmatter,
+ wikiLinkMarkdown,
+} from "@/core/llmwiki/wiki-page-markdown";
+import {
+ type LlmWikiWikiPage,
+ useLlmWikiKnowledgeBase,
+ useLlmWikiWikiPage,
+ useLlmWikiWikiPages,
+ usePublicLlmWikiKnowledgeBase,
+ usePublicLlmWikiWikiPage,
+ usePublicLlmWikiWikiPages,
+} from "@/strategy-components/api/llmwiki";
+
+const LIST_PATH = "/page/workspace/knowledge";
+
+function cleanText(value?: string | null) {
+ return (value || "").replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
+}
+
+function formatDate(value?: string | null) {
+ if (!value) return "";
+ const date = new Date(value);
+ return Number.isNaN(date.getTime())
+ ? value
+ : date.toLocaleString("zh-CN", { hour12: false });
+}
+
+function pageTypeLabel(value?: string) {
+ const pageType = String(value || "").toLowerCase();
+ if (pageType === "directory") return "Wiki 目录页";
+ if (pageType === "overview") return "Wiki 概览页";
+ if (pageType === "article") return "Wiki 文章页";
+ return "Wiki 文档";
+}
+
+function directoryLabel(value: string) {
+ const labels: Record = {
+ concept: "概念",
+ entity: "实体",
+ summary: "摘要",
+ index: "索引",
+ "未分类": "其他页面",
+ };
+ return labels[value.toLowerCase()] || value;
+}
+
+function downloadBlob(blob: Blob, fileName: string) {
+ const url = URL.createObjectURL(blob);
+ const anchor = window.document.createElement("a");
+ anchor.href = url;
+ anchor.download = fileName;
+ anchor.rel = "noopener";
+ window.document.body.appendChild(anchor);
+ anchor.click();
+ anchor.remove();
+ window.setTimeout(() => URL.revokeObjectURL(url), 30_000);
+}
+
+function selectLandingPage(pages: LlmWikiWikiPage[]): LlmWikiWikiPage | undefined {
+ return (
+ pages.find((page) => page.page_type?.toLowerCase() === "overview") ||
+ pages.find((page) => /^(?:index|home|readme)$/i.test(page.slug)) ||
+ pages.find((page) => !page.parent_slug && (page.depth ?? 0) === 0) ||
+ pages[0]
+ );
+}
+
+export function WeKnoraWikiPageView({
+ publicEmbed = false,
+}: {
+ publicEmbed?: boolean;
+}) {
+ const { noteId = "" } = useParams();
+ const [searchParams] = useSearchParams();
+ const location = useLocation();
+ const navigate = useNavigate();
+ const publicBackground = usePublicBackground();
+ const requestedWikiSlug = searchParams.get("wikiSlug")?.trim() || "";
+ const privateKnowledgeBase = useLlmWikiKnowledgeBase(
+ noteId || null,
+ !!noteId && !publicEmbed,
+ );
+ const publicKnowledgeBase = usePublicLlmWikiKnowledgeBase(
+ noteId || null,
+ !!noteId && publicEmbed,
+ );
+ const knowledgeBase = publicEmbed ? publicKnowledgeBase : privateKnowledgeBase;
+ const privatePages = useLlmWikiWikiPages(
+ noteId || null,
+ 1,
+ 500,
+ "",
+ !!noteId && !publicEmbed,
+ );
+ const publicPages = usePublicLlmWikiWikiPages(
+ noteId || null,
+ 1,
+ 500,
+ "",
+ !!noteId && publicEmbed,
+ );
+ const pages = publicEmbed ? publicPages : privatePages;
+ const pagesList = useMemo(() => pages.data?.pages || [], [pages.data?.pages]);
+ const landingPage = useMemo(() => selectLandingPage(pagesList), [pagesList]);
+ const wikiSlug = requestedWikiSlug || (publicEmbed ? landingPage?.slug || "" : "");
+ const privatePage = useLlmWikiWikiPage(
+ noteId || null,
+ wikiSlug || null,
+ !!noteId && !!wikiSlug && !publicEmbed,
+ );
+ const publicPage = usePublicLlmWikiWikiPage(
+ noteId || null,
+ wikiSlug || null,
+ !!noteId && !!wikiSlug && publicEmbed,
+ );
+ const page = publicEmbed ? publicPage : privatePage;
+ const [downloading, setDownloading] = useState<"md" | "word" | null>(null);
+ const [directoryQuery, setDirectoryQuery] = useState("");
+ const [collapsedDirectories, setCollapsedDirectories] = useState>(() => new Set());
+ const routedPage = (location.state as { wikiPage?: LlmWikiWikiPage } | null)?.wikiPage;
+ const pageData = page.data || (routedPage?.slug === wikiSlug ? routedPage : undefined);
+ const filteredPages = useMemo(
+ () => filterWikiPagesWithAncestors(pagesList, directoryQuery),
+ [directoryQuery, pagesList],
+ );
+ const wikiTree = useMemo(() => buildWikiTree(filteredPages), [filteredPages]);
+
+ useEffect(() => {
+ if (!publicEmbed || requestedWikiSlug || !wikiSlug) return;
+ const next = new URLSearchParams(searchParams);
+ next.set("wikiSlug", wikiSlug);
+ navigate(`${buildPublicKnowledgeWikiPath(noteId)}?${next.toString()}`, {
+ replace: true,
+ });
+ }, [navigate, noteId, publicEmbed, requestedWikiSlug, searchParams, wikiSlug]);
+
+ useEffect(() => {
+ const activeParents = wikiTree.parentKeysBySlug.get(wikiSlug) || [];
+ if (activeParents.length === 0) return;
+ setCollapsedDirectories((previous) => {
+ if (activeParents.every((key) => !previous.has(key))) return previous;
+ const next = new Set(previous);
+ activeParents.forEach((key) => next.delete(key));
+ return next;
+ });
+ }, [wikiSlug, wikiTree.parentKeysBySlug]);
+
+ const title = pageData?.title || wikiSlug || "Wiki 页面";
+ const summary = cleanText(pageData?.summary);
+ const content = stripWikiFrontmatter(cleanText(pageData?.content));
+ const displayMarkdown = pageData
+ ? renderableWikiMarkdown(
+ title,
+ content,
+ noteId,
+ publicEmbed
+ ? (knowledgeBaseId, slug) => {
+ const next = new URLSearchParams(searchParams);
+ next.set("wikiSlug", slug);
+ return `/#${buildPublicKnowledgeWikiPath(knowledgeBaseId)}?${next.toString()}`;
+ }
+ : undefined,
+ )
+ : "";
+ const exportMarkdown = pageData
+ ? wikiLinkMarkdown(exportableWikiMarkdown(title, summary, content), noteId)
+ : "";
+
+ const download = async (format: "md" | "word") => {
+ if (!exportMarkdown) return;
+ setDownloading(format);
+ try {
+ const fileName = safeWikiFileName(title);
+ if (format === "md") {
+ downloadBlob(
+ new Blob([exportMarkdown], { type: "text/markdown;charset=utf-8" }),
+ `${fileName}.md`,
+ );
+ } else {
+ downloadBlob(
+ await markdownToDocxBlob(exportMarkdown, title),
+ `${fileName}.docx`,
+ );
+ }
+ toast.success(format === "md" ? "Markdown 文件已生成" : "Word 文件已生成");
+ } catch (error) {
+ toast.error(error instanceof Error ? error.message : "下载 Wiki 页面失败");
+ } finally {
+ setDownloading(null);
+ }
+ };
+
+ const toggleDirectory = (key: string) => {
+ setCollapsedDirectories((previous) => {
+ const next = new Set(previous);
+ if (next.has(key)) next.delete(key);
+ else next.add(key);
+ return next;
+ });
+ };
+
+ const renderDirectoryNode = (node: WikiTreeNode, depth = 0): ReactNode => {
+ const hasChildren = node.children.length > 0;
+ const expanded = directoryQuery.trim() ? true : !collapsedDirectories.has(node.key);
+ const active = node.page?.slug === wikiSlug;
+ const label = node.page ? node.label : directoryLabel(node.label);
+ return (
+
+
+ {hasChildren ? (
+ toggleDirectory(node.key)}
+ aria-label={expanded ? "收起目录" : "展开目录"}
+ >
+ {expanded ? : }
+
+ ) : (
+
+ )}
+ {node.page ? (
+ {
+ const next = new URLSearchParams(searchParams);
+ next.set("wikiSlug", node.page.slug);
+ return `${buildPublicKnowledgeWikiPath(noteId)}?${next.toString()}`;
+ })()
+ : buildWeKnoraWikiPagePath(noteId, node.page.slug)
+ }
+ state={{ wikiPage: node.page }}
+ className="flex min-w-0 flex-1 items-center gap-2 py-2"
+ aria-current={active ? "page" : undefined}
+ title={node.page.title || node.page.slug}
+ >
+
+ {label}
+
+ ) : (
+ toggleDirectory(node.key)}
+ title={label}
+ >
+
+ {label}
+ {node.count}
+
+ )}
+
+ {hasChildren && expanded && (
+
{node.children.map((child) => renderDirectoryNode(child, depth + 1))}
+ )}
+
+ );
+ };
+
+ if (!wikiSlug) {
+ if (publicEmbed && pages.isLoading) {
+ return (
+
+
+ 正在加载公开 Wiki
+
+ );
+ }
+ return (
+
+ {publicEmbed ? "该知识库暂无可展示的 Wiki 页面" : "当前链接缺少 Wiki 页面标识"}
+ {!publicEmbed && (
+ navigate(LIST_PATH)}>返回知识库
+ )}
+
+ );
+ }
+
+ if (page.isLoading && !pageData) {
+ return (
+
+
+ 正在加载 Wiki 页面
+
+ );
+ }
+
+ if (!pageData) {
+ return (
+
+
+
+
+
+
无法打开 Wiki 页面
+
+ {page.error instanceof Error
+ ? page.error.message
+ : publicEmbed
+ ? "知识库未发布、页面不存在或已停止公开访问"
+ : "页面不存在或当前账号无权访问"}
+
+
+
+ {!publicEmbed && (
+ navigate(LIST_PATH)}>返回知识库
+ )}
+ void page.refetch()}>
+ 重试
+
+
+
+ );
+ }
+
+ return (
+
+ {publicEmbed ? (
+
+
+
+
+
+
+ {knowledgeBase.data?.name || "知识库 Wiki"}
+
+
{title}
+
+
+ 公开 Wiki
+
+
+ ) : (
+
+ )}
+
+
+
+
+
+
+ {summary && (
+
+ )}
+
+
+
+
+
+ );
+}
+
+export default WeKnoraWikiPageView;
diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md
index 71376570..0beb183a 100644
--- a/offline-backend-20260512/backend/CLAUDE.md
+++ b/offline-backend-20260512/backend/CLAUDE.md
@@ -347,6 +347,7 @@ FastAPI application on port 8001 with health checks at `GET|HEAD /health`, `/api
| **Dashboard Sessions** (`/api/dashboard-sessions`) | 「大屏绘制智能体」独立页面会话,**按外部 `task_id` 存取,不按 user 分权**(open read + shared writes;`user_id` audit-only)。一个 task 一行(upsert):`transcript`(聊天记录 JSON)+ `report_json`(最新结构化数据,可解析 = 「有效结果」)。`GET /by-task/{task_id}` - 取该 task 会话(无则 404);`PUT /by-task/{task_id}` - upsert(`{transcript?, report_json?}`,只覆盖传入项)。Backed by `deerflow.persistence.dashboard_sessions`(table `dashboard_sessions`, auto-created by `create_all`,无迁移);store wired in `deps.py` as `app.state.dashboard_session_store`。前端独立页 `roundtable-planning/pages/DashboardAgentPage.tsx`(路由 `/page/strategy/dashboard-agent?taskId=`,侧边栏「问答管理 → 大屏绘制」)让智能体分析任务报告 JSON → report-json → 渲染 `DashboardApp`。Tests: `tests/test_dashboard_sessions.py`。 |
| **Task Reports** (`/api/task-reports`) | 任务级研判情报报告详情,**按 `task_id` 共享、不按 user 分权**(写入者只留 `updated_by` 审计)。`GET /by-task/{task_id}` 始终返回原 Consumer 兼容外壳 `{state:"200", msg:"操作成功!", data:[...]}`(未保存为 `data: []`);`POST /by-task/{task_id}` 首次保存,已有报告返回 409;`PUT /by-task/{task_id}` 原子替换完整 `data` 数组。`POST /import/by-task/{task_id}` 面向 TaskCOP 导入页,接收 `sq-report-mock.json` 兼容的 `data`/`records` 数组并完整覆盖目标任务;路径中的选定任务 id 优先于文件内的 `taskId`,避免误写。每一项严格是 `{id, taskId, createTime, updateTime, sessionId, contentJson, categoryType}`;`contentJson` 以 JSON **字符串**原样落在 `records_json`,避免破坏大屏按类目解析的输入格式。保存或导入非空 `data` 后,router 会通过 `app.state.taskcop_task_store.mark_analysis_completed` 将同 id 的 TaskCOP 任务升级至状态 `25`,使旧任务列表显示并筛选为“已完成”;普通保存时不存在匹配任务的报告仍可独立保存,导入则要求目标任务存在。Backed by `deerflow.persistence.task_reports`(table `task_reports`, auto-created by `create_all`);store wired in `deps.py` as `app.state.task_report_store`。前端客户端 `strategy-components/api/task-reports.ts`,`core/auth/dashboard-input.ts` 与 rwfx 3q 完成检查均直接读此接口,不再依赖 Consumer 或本地 sq-report mock。Tests: `tests/test_task_reports.py`。 |
| **Task Buttons** (`/api/task-buttons`) | Task-workspace configurable jump buttons (按钮管理), **global/shared** (no user scoping; `updated_by` audit-only). Each row = one jump button in a task deep-link workspace's bottom 快捷跳转 bar: `{id, business (rwfx/3qfx/xdfx), label, link_type (url/business/purpose/host), target, append_task_id, id_param (editable query-param name, e.g. id/task_id), id_kind (task/action — only xdfx differs, its deep-link id is an 行动id), open_mode (blank/self), login_params, append_auth, auth_token_param, auth_name_param, enabled, sort_order}`. **Two independent, coexisting login-append mechanisms for `url` buttons** — both resolved frontend-side: (1) `login_params` (JSON column `login_params_json`, `PortableJSON`, migration `20260625_01`) = a list of `{key, source, custom_value}` resolved at click time (`source` = a `userInfo` field or `other`→literal `custom_value`) and appended to the target URL — mirrors the light-app login params; empty = nothing appended. (2) `append_auth` + `auth_token_param` + `auth_name_param` (columns, migration `20260625_02`, default off) — appends `?{auth_token_param}={token1}&{auth_name_param}={username}` so the target system can identify the current user; gated on **token login** — the frontend only stashes `token1` (the `?authToken=`) + the raw `getTokenInfo` username (`login.token1`/`login.username`) when login went through `/login/token`, so a non-token-login session appends neither. The **frontend** holds code-seed defaults (rwfx's 4 historical buttons built from the live `task_deeplink` config); only when this table is empty do the workspace + admin page fall back to them, else this table is the sole source of truth. `GET /` - list all (open to any authenticated user — the workspace renders them); `PUT /` - **admin only** (`_require_admin`), atomically **replaces the whole set** (`{buttons: [...]}`, de-duped by `id`, last wins). Backed by `deerflow.persistence.task_buttons` (table `task_buttons`, auto-created by `create_all`); store wired in `deps.py` as `app.state.task_button_store`. Frontend: helpers `core/auth/task-buttons.ts`, client `strategy-components/api/task-buttons.ts`, admin page `pages/TaskButtonsPage.tsx` (route `/page/strategy/admin/task-buttons`, entry「按钮管理」in the bottom-left settings dropdown, admin-only, both sidebar modes; per-business tabs + 「导入默认」). Tests: `tests/test_task_buttons.py`. |
+| **Fixed Questions** (`/api/fixed-questions`) | Administrator-managed exact-match fixed Q&A (问题配置), **global/shared**. Each row is `{id, question, answer, enabled, tokens_per_second, sort_order}`; stream speed is configurable per row from `1–1000 token/s` and defaults to `100`. `GET /` and whole-set atomic `PUT /` are both admin-only. At the start of an external chat run, `services._find_fixed_answer` checks the latest eligible plain-text human message with exact, case/space/punctuation-sensitive equality. Enabled matches switch the run to `make_fixed_answer_agent`: a local `BaseChatModel` tokenizes with `cl100k_base`, replays the answer in rate-limited token chunks, and keeps the existing LangGraph SSE `messages`/`values` protocol, run journal, checkpoint history, deterministic local title, and refresh behavior intact while no provider/model request or model-concurrency slot is used. File/multimodal/context-prefixed turns, `canvas_agent`, `ai_writing`, writing mode, resume commands, and internal orchestration/background child runs bypass matching. Storage failures fail open to the normal model path. Backed by `deerflow.persistence.fixed_questions` (table `fixed_questions`, SHA-256 indexed lookup plus exact original-string collision check, migrations `20260821_01` + `20260821_02`), wired as `app.state.fixed_question_store`. Frontend: `strategy-components/api/fixed-questions.ts`, admin page `pages/FixedQuestionsPage.tsx`, route `/page/strategy/admin/fixed-questions`, entries in both full-layout and compact-workspace settings menus. Tests: `tests/test_fixed_questions.py`. |
| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup). **Defaults to `background=true`**: the slow per-thread extraction loop is deferred to a FastAPI background task and the endpoint returns `{queued:true, processed:N}` immediately (each thread's transcript is truncated to `max_input_chars` before the LLM call). Running the loop inline for a large batch outlives the nginx gateway timeout (504), which the frontend's `parseResponse` surfaces as a generic "Request failed"; `background=false` keeps the blocking path (full per-thread `created/skipped/failed` tally) for small batches/scripts. Shared loop: `_run_batch_ingest`. **Optional model selection**: both ingest endpoints accept `model_name` (overrides `knowledge.extract_model_name` for that capture; threaded through `KnowledgeService.capture_thread` ? `extract_thread_llm_multi`). **Model-error surfacing**: when the chosen model produces nothing usable (timeout / API error / non-JSON) capture still **falls back to rule-based extraction and lands a draft** (`llm_fallback:true`); the single-thread (non-background) response and the batch response (`llm_fallback` count) carry this so the UI tells the user "???????". The "??????" chat button stays background/fire-and-forget (passes `model_name`, no per-save error toast); the batch-import dialog is synchronous and shows the fallback count. `POST /ingest/file` - **import an uploaded file** (multipart: `file` + optional `title`/`tags`/`status`/`model_name`) into wiki note(s): the file is converted to Markdown (md/txt read directly, pdf/word/ppt/excel via `convert_file_to_markdown`), distilled by the same wiki-capture LLM pipeline as thread capture (`source_type="file"`), and falls back to a verbatim **draft** note when the model returns nothing usable (`llm_fallback:true`); `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector?keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities, graph-blacklist filtered); `GET/POST /graph/blacklist` + `DELETE /graph/blacklist/{id}` - graph node blacklist (labels = entity names / note titles, case-insensitive match; blacklisted nodes and their edges are dropped from `/graph` and the wiki-export artifacts; table `knowledge_graph_blacklist`, auto-created); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[?]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). `GET/POST /folders` + `PUT/DELETE /folders/{id}` - user-managed directories (notes carry a `folder` path; `GET /notes?folder=` filters a dir + sub-dirs); `GET/POST /extract-templates` + `GET/PUT/DELETE /extract-templates/{id}` - editable extraction-prompt templates (thread/document ingest accept `template_id`; capture/PUT-note accept `folder`). Backed by the `deerflow.knowledge` module (see below). |
Proxied through nginx: `/api/langgraph/*` ? LangGraph, all other `/api/*` ? Gateway.
@@ -718,6 +719,13 @@ Operators upgrading an existing DB run the whole file (`USE ;` then execute)
- **Loading**: `load_skills()` recursively scans `skills/{public,custom}` for `SKILL.md`, parses metadata, and reads enabled state from extensions_config.json. **A directory containing `SKILL.md` is a self-contained leaf** — discovery does not descend into its subdirectories, so a skill *package* that bundles its own `examples/`/`references/` sub-skills (each with a SKILL.md) surfaces as **one** skill, not a swarm of phantom entries.
- **Discovery↔delete consistency**: a skill is keyed for discovery by its frontmatter `name`, but its on-disk directory name may differ (hand-dropped / mis-extracted package). `LocalSkillStorage._resolve_skill_dir(name, category)` resolves the *discovered* directory (canonical `/` fast-path, else scan-by-frontmatter-name); `custom_skill_exists`/`read_custom_skill`/`delete_custom_skill` all go through it, so **anything that gets listed can also be read and deleted** regardless of directory name (the old code assumed `custom//` and 404'd "Custom skill 'x' not found." on any non-canonical layout). Delete is path-guarded to never escape the `custom/` root. Tests: `tests/test_skill_storage_delete.py`.
- **Injection**: Enabled skills listed in agent system prompt with container paths
+- **Configurable `es_query` routing prompt**: `config.yaml → skills.es_query_routing`
+ contains an `enabled` switch and editable `prompt`. When enabled,
+ `lead_agent.prompt._build_es_query_routing_section` injects the rule into the
+ default lead-agent system prompt. Custom-agent-only prompts receive it only
+ when their explicit skill allowlist contains `es_query`; disabling the switch
+ or leaving the prompt blank emits no block. Tests:
+ `tests/test_es_query_routing_prompt.py`.
- **Installation**: `POST /api/skills/install` extracts .skill ZIP archive to custom/ directory
- **Installable sentiment analysis package**: `skill-packages/sentiment-analysis.skill` is deliberately separate from the existing frontend virtual-agent path. Its self-contained `config/sentiment-agent.json` owns the external AG-UI address; Basic Auth is resolved from `SENTIMENT_AGENT_AUTH_USERNAME` / `SENTIMENT_AGENT_AUTH_PASSWORD` at runtime (not committed into the package). `scripts/sentiment_stream.py` uses only the Python standard library and converts upstream SSE to stable NDJSON events (`thinking_delta`, `answer_delta`, `done`), with a Markdown convenience mode for normal agent calls. Installation/calling guide: `skill-packages/sentiment-analysis/docs/调用与安装说明.md`.
- **Chinese name (`name_zh`)**: a Chinese display name stored in the **`skills` DB table** (`SkillRow.name_zh`, `String(128)`, migration `20260609_01`) ? never written to the on-disk SKILL.md, so it works for built-in skills too. Threaded through `SkillStore` create/ensure_legacy/update/update_admin exactly like `detail`; surfaced on `SkillResponse.name_zh` from the ownership record in `_skill_to_response`. Authorization mirrors `detail`: custom skill ? owner or admin; built-in/public ? admin only (row materialized on demand via `ensure_legacy`). Endpoints (in `app/gateway/routers/skills.py`): `PUT /api/skills/{name}/name-zh` (set/clear), `POST /api/skills/{name}/name-zh/generate` (LLM generates one), `POST /api/skills/name-zh/generate-missing` (????: **admin-only** bulk-generate for every skill that lacks one, bounded-concurrency `asyncio.gather`). **Optional model selection**: both generate endpoints accept an optional JSON body `{model_name}` ? an explicit name wins, else the curator model setting, else the main model (`_resolve_skill_name_zh_model_name`). **Model-error surfacing**: `_generate_skill_name_zh` raises `SkillNameZhModelError` when the model itself fails (build/timeout/API error) ? distinct from the model returning an unusable name (? `None`). The single endpoint maps that to `502 ?????,?????????:?`; the bulk endpoint fails fast on a model-build error (502) and otherwise carries a representative `model_error` string in `SkillNameZhBulkResponse` so the UI can say "N ?????????". The frontend (`skill-settings-page.tsx` list + `skill-detail-dialog.tsx` editor, plus the agent skill pickers/badges in `NewAgentPage.tsx` + `agent-card.tsx`) shows `name_zh || name`, offers per-skill "AI ??" plus an **admin-only** header "???????" button, and a compact model dropdown (`components/workspace/model-name-select.tsx`, default = configured model) beside each action.
@@ -1419,6 +1427,88 @@ ordinary evidence questions must remain non-mutating, and cancellation or a
stream error must never save a partial report. The rewrite prompt receives only
the selected range plus the selected evidence set, never a new web search.
+### DeerFlow-local LLMWiki Wiki index
+
+`llmwiki.local_wiki_index` is an opt-in local-vector subsystem. WeKnora is the
+Wiki generation/sync source and the Wiki-only fallback while an entire mapping
+has not completed vectorization. The readiness gate requires `state=idle`, a
+successful full scan, matching embedding fingerprint, no failed pages,
+`ready_page_count == local_page_count`, and vectors for a non-empty library.
+Ready mappings read DeerFlow's `llmwiki_wiki_pages` /
+`llmwiki_wiki_vectors` / `llmwiki_wiki_sync_states` tables; incomplete mappings
+call only WeKnora `/knowledgebase/{id}/wiki/pages?query=...` plus Wiki-page
+detail. Never call `/knowledge-search` from internal search, RAG, or the
+`llmwiki_search` tool, and never return raw chunks as the fallback. Core code lives under
+`deerflow.integrations.weknora.local_index` and
+`deerflow.persistence.llmwiki_index`; preserve the harness → app import
+firewall. Gateway composition injects the store/search/sync services through
+`local_index.runtime`, and the lease-protected scheduler starts only when both
+`enabled` and `sync_enabled` are true.
+
+Vectors are normalized little-endian float32 BLOBs. A page mirror and all of
+its vectors are replaced in one transaction only after strict batched
+embedding succeeds. Never delete the old generation before a remote embedding
+call and never fall back to WeKnora raw chunks while the local feature flag is
+enabled. NumPy matrices are per-mapping LRU snapshots invalidated by
+`index_revision`; large decode/matrix work runs through `asyncio.to_thread`.
+During a fingerprint rebuild the mapping uses the Wiki-page API until every
+page is successfully rebuilt. `WikiRetrievalService` partitions mixed requests
+per mapping, so ready libraries stay local while incomplete libraries use that
+Wiki-only fallback.
+
+The frontend must keep Wiki citations visibly distinct from raw-document
+citations while hiding provider-brand wording from user-facing labels, status
+text, and errors. Any click on a Wiki source card opens the shared right-side
+source drawer, which renders the complete Markdown page. When the local index
+is disabled, the drawer reads the remote WeKnora Wiki-page endpoint directly
+and must not surface the local-index-disabled response. A Wiki target must not
+also load or render the legacy original-document preview, even when its citation
+still carries a document id. The drawer supports single-page Markdown and Word
+downloads, and its **Open in Wiki** link carries both the DeerFlow knowledge-base
+mapping id and the exact `wikiSlug`. Wiki search-result cards use this same
+drawer instead of navigating immediately. Exact-page navigation uses the native
+`knowledge/{mapping_id}/wiki/view?wikiSlug=...` reader, not the WeKnora detail
+iframe. Keep legacy `knowledge/{mapping_id}?tab=wiki&wikiSlug=...` links fast by
+redirecting them in `LlmWikiKnowledgePage` before its runtime-loading gate.
+Drawer-to-reader navigation stays in the current SPA and passes the fetched
+Wiki page in router state so the article can render before the background query
+refresh completes. The native reader must retain a searchable/collapsible left
+Wiki directory, reconstruct hierarchy from `parent_slug`, `wiki_path`,
+`category_path`, and slug fallback, expand/highlight the active page, and keep
+directory navigation on the same lightweight route rather than mounting the
+WeKnora iframe.
+
+Third-party pages embed published Wiki content through the chrome-free route
+`/#/embed/knowledge/{mapping_id}` (optional exact-page `wikiSlug`). It defaults
+to an overview/root Wiki page, retains directory navigation, and supports the
+shared public-embed `theme` / `bg` / `text` appearance inputs. All anonymous
+reader calls use `/api/public/llmwiki/knowledge-bases/{id}` and its Wiki-page
+children. Those endpoints must resolve the mapping with a synthetic anonymous
+user and `is_admin=False`, explicitly require `publication_status=published`,
+and hard-deny the system conversation-deposit mapping. Never point this page at
+the authenticated `/api/llmwiki` reads or weaken the published-only gate merely
+because the mapping id is hard to guess.
+
+Owners manually force a full sync/vectorization through `POST
+/api/llmwiki/knowledge-bases/{id}/vectorize`; status is available through `GET
+/api/llmwiki/knowledge-bases/{id}/wiki-index-status`. The Wiki management page
+also exports all remote processed Wiki pages via `GET
+/api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx`. Directory export must keep
+the full path, current directory, parent path, and dynamic per-level columns;
+long Markdown is split into Excel-safe continuation columns instead of being
+truncated. Workbook creation runs in `asyncio.to_thread`.
+
+Internal search authorization always starts from DeerFlow mapping ids and is
+rechecked by the Gateway. External endpoints use the isolated
+`/api/external/llmwiki/wiki` namespace and `X-API-Key`; require published +
+external-enabled + index-enabled on every request and response, and hard-deny
+the `system` / `对话沉淀` mapping. Never return remote WeKnora ids, provenance
+refs, credentials, or vector bytes externally. Regression tests:
+`tests/test_llmwiki_local_index_repository.py`,
+`tests/test_llmwiki_wiki_sync.py`, `tests/test_llmwiki_vector_search.py`,
+`tests/test_external_llmwiki_search.py`, and
+`tests/test_llmwiki_wiki_retrieval.py`.
+
The sandbox whole-report rewrite may carry an optional `reportOutline` Markdown
template from the shared report-structure library. Persist the original template
in the durable input snapshot and append it as an explicit chapter-structure
diff --git a/offline-backend-20260512/backend/README.md b/offline-backend-20260512/backend/README.md
index aef83e99..fe2f9ee4 100644
--- a/offline-backend-20260512/backend/README.md
+++ b/offline-backend-20260512/backend/README.md
@@ -504,7 +504,9 @@ Key sections:
- `tools` - Tool definitions with module paths and groups
- `tool_groups` - Logical tool groupings
- `sandbox` - Execution environment provider
-- `skills` - Skills directory paths
+- `skills` - Skills directory paths plus optional prompt routing; `skills.es_query_routing`
+ can inject an editable system-prompt rule that sends flexible Elasticsearch Query DSL
+ requests to the `es_query` skill, and can be disabled without changing code
- `title` - Auto-title generation settings
- `summarization` - Context summarization settings
- `subagents` - Subagent system (enabled/disabled)
@@ -768,6 +770,72 @@ without deleting the audit row. DeerFlow stores only the artifact reference
never the Markdown body and never a task-system delivery. Continue to preview
the file through the existing thread-artifact endpoint.
+## DeerFlow-local LLMWiki Wiki vector index
+
+LLMWiki can mirror processed WeKnora Wiki pages into DeerFlow and search only
+processed Wiki content. A knowledge base uses exact NumPy cosine similarity
+only after its whole Wiki has completed one coherent vectorization pass; until
+then internal search calls WeKnora's Wiki-page search/list API and reads the
+matched Wiki pages. It never falls back to raw documents or chunks. The local
+vector feature is disabled by default. Configure
+`llmwiki.local_wiki_index.embedding` first, then enable `enabled`,
+`sync_enabled`, and `internal_search_enabled` in `config.yaml`.
+
+The migration creates `llmwiki_wiki_pages`, `llmwiki_wiki_vectors`, and
+`llmwiki_wiki_sync_states`. Wiki Markdown and normalized float32 vectors are
+stored in DeerFlow's configured SQL database. Multi-worker sync is protected by
+database leases; searches use revision-checked per-knowledge-base LRU snapshots
+after the full-library readiness gate succeeds. Knowledge-base owners can click
+**手动向量化** in Wiki management (`POST
+/api/llmwiki/knowledge-bases/{id}/vectorize`) and follow its status at
+`GET /api/llmwiki/knowledge-bases/{id}/wiki-index-status`. Existing chunk
+citations remain readable while all new retrieval citations open Wiki pages.
+
+The frontend labels these results simply as **Wiki** references; provider-brand
+wording is intentionally hidden from user-facing labels and status text. Clicking any
+part of a Wiki reference card opens one right-side drawer (rather than a second
+detail dialog), renders the complete page as Markdown, and reads the remote
+WeKnora Wiki page directly when the local index is disabled. Even if a Wiki
+citation carries a legacy document id, the drawer does not mix in the uploaded
+original-file preview. A single page can be downloaded as Markdown or Word from
+the drawer. **Open in Wiki** navigates to the exact DeerFlow knowledge-base
+mapping and Wiki slug. Wiki search-result cards use the same drawer interaction.
+Exact Wiki links now open a lightweight native reader at
+`knowledge/{mapping_id}/wiki/view?wikiSlug=...`; it fetches only the selected
+page and never creates a WeKnora iframe session. The former
+`knowledge/{mapping_id}?tab=wiki&wikiSlug=...` links redirect to the reader
+before waiting for the LLMWiki runtime check. Drawer navigation stays in the
+same SPA and passes the already-loaded page as route state for immediate paint.
+The native reader also keeps a searchable, collapsible Wiki directory on the
+left. It reconstructs hierarchy from `parent_slug`, `wiki_path`,
+`category_path`, and the slug fallback, highlights and expands the current
+page, and switches directory entries through the same lightweight reader route
+without mounting the WeKnora iframe.
+
+Published knowledge bases also have a login-free, chrome-free reader at
+`/#/embed/knowledge/{mapping_id}` for direct third-party iframe use. With no
+`wikiSlug`, it selects an overview/root page and keeps directory navigation
+inside the embed route; `?wikiSlug=...` deep-links one page. The reader supports
+`theme=light|dark`, `bg`, and `text` embed appearance parameters. Its data comes
+only from the read-only `/api/public/llmwiki/knowledge-bases/{id}/...` namespace,
+which rechecks `publication_status=published` on every request and denies the
+system conversation-deposit knowledge base. Private or unpublished bases are
+never exposed by this route.
+
+Wiki management also provides **导出 Excel**. `GET
+/api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx` exports every processed Wiki
+page, including its full directory path, current directory, parent-directory
+path, one column per directory level, metadata, and complete Markdown content
+(split across continuation columns when Excel's per-cell limit is reached).
+
+Authenticated operations are available below `/api/llmwiki/wiki-index` and
+`POST /api/llmwiki/wiki-vector-search`. The service-to-service endpoints below
+`/api/external/llmwiki/wiki` require `X-API-Key`, are rate-limited, and only
+expose mappings that are both internally published and explicitly marked
+`external_search_enabled`; the system `对话沉淀` mapping is always denied.
+See `docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md` for configuration,
+security boundaries, rollout, and API contracts.
+
## Document rewrite fallback
Whole-document rewrite commits the artifact/report before it attempts to store
diff --git a/offline-backend-20260512/backend/app/gateway/app.py b/offline-backend-20260512/backend/app/gateway/app.py
index eb67e5fc..315d1729 100644
--- a/offline-backend-20260512/backend/app/gateway/app.py
+++ b/offline-backend-20260512/backend/app/gateway/app.py
@@ -1,1692 +1,1730 @@
-import asyncio
-import logging
-import os
-import queue
-import sys
-import threading
-from collections.abc import AsyncGenerator
-from contextlib import asynccontextmanager
-from datetime import datetime
-from time import perf_counter
-
-from dotenv import load_dotenv
-from fastapi import FastAPI
-from fastapi.middleware.cors import CORSMiddleware
-
-# Load .env BEFORE any module that reads os.environ at import time.
-# Without this, GATEWAY_CORS_ORIGINS / AUTH_JWT_SECRET / DEER_FLOW_AUTH_DISABLED
-# placed in /workspace/.env are silently ignored when the container is started
-# without explicit `-e` flags, leading to ephemeral JWT secrets, missing CORS
-# origins, and "Failed to fetch" in the browser.
-load_dotenv()
-
-from app.gateway.auth.disabled_mode import is_auth_disabled # noqa: E402
-from app.gateway.auth_middleware import AuthMiddleware
-from app.gateway.config import get_gateway_config
-from app.gateway.csrf_middleware import CSRFMiddleware
-from app.gateway.deps import langgraph_runtime
-from app.gateway.routers import (
- ai_writing,
- admin_active_runs,
- admin_users,
- agents,
- artifacts,
- assistants_compat,
- auth,
- browser_context,
- business_mapping,
- channels,
+import asyncio
+import logging
+import os
+import queue
+import sys
+import threading
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+from datetime import datetime
+from time import perf_counter
+
+from dotenv import load_dotenv
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+# Load .env BEFORE any module that reads os.environ at import time.
+# Without this, GATEWAY_CORS_ORIGINS / AUTH_JWT_SECRET / DEER_FLOW_AUTH_DISABLED
+# placed in /workspace/.env are silently ignored when the container is started
+# without explicit `-e` flags, leading to ephemeral JWT secrets, missing CORS
+# origins, and "Failed to fetch" in the browser.
+load_dotenv()
+
+from app.gateway.auth.disabled_mode import is_auth_disabled # noqa: E402
+from app.gateway.auth_middleware import AuthMiddleware
+from app.gateway.config import get_gateway_config
+from app.gateway.csrf_middleware import CSRFMiddleware
+from app.gateway.deps import langgraph_runtime
+from app.gateway.routers import (
+ ai_writing,
+ admin_active_runs,
+ admin_users,
+ agents,
+ artifacts,
+ assistants_compat,
+ auth,
+ browser_context,
+ business_mapping,
+ channels,
checkpoint_migration,
compact_menu_overrides,
+ external_llmwiki,
feedback,
- dashboard_sessions,
- deep_research,
- html_page_favorites,
- intent,
- knowledge,
- knowledge_ingest,
- light_apps,
+ fixed_questions,
+ dashboard_sessions,
+ deep_research,
+ html_page_favorites,
+ intent,
+ knowledge,
+ knowledge_ingest,
+ light_apps,
llm_metrics,
llmwiki,
- mcp,
- memory,
- menu_overrides,
- model_management,
- models,
- multi_agent,
- notifications,
- open_3qfx,
- open_chat,
- parallel_agent_tasks,
- parallel_agents,
- position_roundtable,
- positions,
- position_roles,
- public_config,
- public_embed,
- public_leaderboard,
- public_scheduled_tasks,
- recommend,
- flow_extract,
- recommended_questions,
- roundtable_chains,
- roundtable_artifact_submissions,
- roundtable_diagnostics,
- roundtable_draft_shares,
- roundtable_drafts,
- roundtable_jobs,
- roundtable_task_drafts,
- runs,
- scheduled_tasks,
- sensitive_words,
- skill_addresses,
- skills,
- suggestions,
- system_settings,
- tags,
- task_buttons,
- taskcop_tasks,
- report_structures,
- task_reports,
- thread_runs,
- tool_metrics,
- thread_shares,
- threads,
- uploads,
- user_preferences,
- user_prompts,
- writing,
-)
-from deerflow.config import app_config as deerflow_app_config
-from deerflow.config.app_config import apply_logging_level
-from deerflow.runtime.scheduler import ScheduledTaskService, set_scheduled_task_service
-
-AppConfig = deerflow_app_config.AppConfig
-get_app_config = deerflow_app_config.get_app_config
-
-# Default logging; lifespan overrides from config.yaml log_level.
-logging.basicConfig(
- level=logging.INFO,
- format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
- datefmt="%Y-%m-%d %H:%M:%S",
-)
-
-logger = logging.getLogger(__name__)
-
-# Upper bound (seconds) each lifespan shutdown hook is allowed to run.
-# Bounds worker exit time so uvicorn's reload supervisor does not keep
-# firing signals into a worker that is stuck waiting for shutdown cleanup.
-_SHUTDOWN_HOOK_TIMEOUT_SECONDS = 5.0
-
-
-async def _ensure_admin_user(app: FastAPI) -> None:
- """Startup hook: handle first boot and migrate orphan threads otherwise.
-
- After admin creation, migrate orphan threads from the LangGraph
- store (metadata.user_id unset) to the admin account. This is the
- "no-auth ? with-auth" upgrade path: users who ran DeerFlow without
- authentication have existing LangGraph thread data that needs an
- owner assigned.
- First boot (no admin exists):
- - Does NOT create any user accounts automatically.
- - The operator must visit ``/setup`` to create the first admin.
-
- Subsequent boots (admin already exists):
- - Runs the one-time "no-auth ? with-auth" orphan thread migration for
- existing LangGraph thread metadata that has no owner_id.
-
- No SQL persistence migration is needed: the four user_id columns
- (threads_meta, runs, run_events, feedback) only come into existence
- alongside the auth module via create_all, so freshly created tables
- never contain NULL-owner rows.
- """
- from sqlalchemy import select
-
- from app.gateway.deps import get_local_provider
- from deerflow.persistence.engine import get_session_factory
- from deerflow.persistence.user.model import UserRow
-
- try:
- provider = get_local_provider()
- except RuntimeError:
- # Auth persistence may not be initialized in some test/boot paths.
- # Skip admin migration work rather than failing gateway startup.
- logger.warning("Auth persistence not ready; skipping admin bootstrap check")
- return
-
- sf = get_session_factory()
- if sf is None:
- return
-
- admin_count = await provider.count_admin_users()
-
- if admin_count == 0:
- logger.info("=" * 60)
- logger.info(" First boot detected ? no admin account exists.")
- logger.info(" Visit /setup to complete admin account creation.")
- logger.info("=" * 60)
- return
-
- # Admin already exists ? run orphan thread migration for any
- # LangGraph thread metadata that pre-dates the auth module.
- async with sf() as session:
- stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
- row = (await session.execute(stmt)).scalar_one_or_none()
-
- if row is None:
- return # Should not happen (admin_count > 0 above), but be safe.
-
- admin_id = str(row.id)
-
- # LangGraph store orphan migration ? non-fatal.
- # This covers the "no-auth ? with-auth" upgrade path for users
- # whose existing LangGraph thread metadata has no user_id set.
- store = getattr(app.state, "store", None)
- if store is not None:
- try:
- migrated = await _migrate_orphaned_threads(store, admin_id)
- if migrated:
- logger.info("Migrated %d orphan LangGraph thread(s) to admin", migrated)
- except Exception:
- logger.exception("LangGraph thread migration failed (non-fatal)")
-
-
-async def _iter_store_items(store, namespace, *, page_size: int = 500):
- """Paginated async iterator over a LangGraph store namespace.
-
- Replaces the old hardcoded ``limit=1000`` call with a cursor-style
- loop so that environments with more than one page of orphans do
- not silently lose data. Terminates when a page is empty OR when a
- short page arrives (indicating the last page).
- """
- offset = 0
- while True:
- batch = await store.asearch(namespace, limit=page_size, offset=offset)
- if not batch:
- return
- for item in batch:
- yield item
- if len(batch) < page_size:
- return
- offset += page_size
-
-
-async def _sync_legacy_skills(app: FastAPI) -> None:
- """Idempotently register existing on-disk ``skills/custom/*`` directories
- in the ``skills`` DB table, owned by the first admin and published=True.
-
- Without this step, after upgrading to per-user skill ownership the existing
- custom skill folders would not be visible to anyone (the SkillStore
- visibility filter relies on a DB row). Running this once on every boot is
- cheap (folder scan + one ``ensure_legacy`` per name) and idempotent.
-
- Non-fatal: a failure here MUST NOT block server startup.
- """
- from sqlalchemy import select
-
- from deerflow.persistence.engine import get_session_factory
- from deerflow.persistence.user.model import UserRow
- from deerflow.skills.storage import get_or_new_skill_storage
- from deerflow.skills.types import SkillCategory
-
- sf = get_session_factory()
- if sf is None:
- return
-
- skill_store = getattr(app.state, "skill_store", None)
- if skill_store is None:
- return
-
- async with sf() as session:
- stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
- row = (await session.execute(stmt)).scalar_one_or_none()
- if row is None:
- # No admin yet ? leave ownership null; first admin is responsible for
- # claiming via the UI once they exist.
- admin_id: str | None = None
- else:
- admin_id = str(row.id)
-
- config = getattr(app.state, "config", None)
- storage = get_or_new_skill_storage(app_config=config)
- try:
- skills = storage.load_skills(enabled_only=False)
- except Exception:
- logger.exception("_sync_legacy_skills: could not list on-disk skills")
- return
-
- inserted = 0
- for skill in skills:
- if skill.category != SkillCategory.CUSTOM:
- continue
- try:
- existing = await skill_store.get_any(skill.name)
- if existing is not None:
- continue
- await skill_store.ensure_legacy(
- {"name": skill.name, "owner_user_id": admin_id, "published": True}
- )
- inserted += 1
- except Exception:
- logger.warning("_sync_legacy_skills: could not register %s", skill.name, exc_info=True)
- if inserted:
- logger.info("Skill ownership migration: registered %d legacy custom skill(s)", inserted)
-
-
-async def _migrate_orphaned_threads(store, admin_user_id: str) -> int:
- """Migrate LangGraph store threads with no user_id to the given admin.
-
- Uses cursor pagination so all orphans are migrated regardless of
- count. Returns the number of rows migrated.
- """
- migrated = 0
- async for item in _iter_store_items(store, ("threads",)):
- metadata = item.value.get("metadata", {})
- if not metadata.get("user_id"):
- metadata["user_id"] = admin_user_id
- item.value["metadata"] = metadata
- await store.aput(("threads",), item.key, item.value)
- migrated += 1
- return migrated
-
-
-async def _run_db_migrations() -> None:
- """Run Alembic migrations at startup so schema changes are applied automatically."""
- from pathlib import Path
-
- from alembic import command
- from alembic.config import Config
-
- import deerflow.persistence as _persistence_pkg
- from deerflow.persistence.engine import get_engine
-
- engine = get_engine()
- if engine is None:
- logger.info("Persistence backend=memory ? skipping Alembic migrations")
- return
-
- if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in {"1", "true", "yes"}:
- logger.warning("DEER_FLOW_SKIP_ALEMBIC is set; skipping Alembic migrations")
- return
-
- db_url = engine.url.render_as_string(hide_password=False).replace("%", "%%")
- migrations_dir = str(Path(_persistence_pkg.__file__).parent / "migrations")
- result_queue: queue.Queue[BaseException | None] = queue.Queue(maxsize=1)
-
- def _run_sync() -> None:
- try:
- cfg = Config()
- cfg.set_main_option("script_location", migrations_dir)
- cfg.set_main_option("sqlalchemy.url", db_url)
- command.upgrade(cfg, "head")
- except BaseException as exc:
- result_queue.put(exc)
- else:
- result_queue.put(None)
-
- try:
- try:
- timeout_s = float(os.getenv("DEER_FLOW_ALEMBIC_TIMEOUT", "60"))
- except ValueError:
- timeout_s = 60.0
-
- worker = threading.Thread(target=_run_sync, name="deerflow-alembic-upgrade", daemon=True)
- worker.start()
- try:
- deadline = asyncio.get_running_loop().time() + timeout_s
- while result_queue.empty():
- if asyncio.get_running_loop().time() >= deadline:
- raise asyncio.TimeoutError
- await asyncio.sleep(0.2)
- result = result_queue.get_nowait()
- except asyncio.TimeoutError:
- logger.error(
- "Database migration did not finish within %.0fs; continuing startup on create_all schema. "
- "Likely cause: MySQL metadata-lock wait or a slow DDL. Set DEER_FLOW_SKIP_ALEMBIC=1 "
- "to skip startup migrations, or inspect MySQL with SHOW FULL PROCESSLIST.",
- timeout_s,
- )
- return
- if result is not None:
- raise result
- logger.info("Database migrations applied (head)")
- except Exception:
- logger.exception("Database migration failed ? schema may be out of date")
-
-
-# Skill curator ?????
-# - ????:?????????"??????";??????? should_run_now() ???
-# - ????:?????????????,???????
-_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 ??
-_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 ??
-
-
-def _acquire_singleton_lock(lock_path):
- """???????????,??? worker ?? leader ???
-
- ?? -> ??????????(?????????,??????/????);
- ????(???? worker ??)????? -> ?? None?
- ??????? OS ????????
- """
- try:
- lock_path.parent.mkdir(parents=True, exist_ok=True)
- handle = open(lock_path, "a+")
- except OSError:
- logger.exception("Curator scheduler: ???????")
- return None
- try:
- if sys.platform == "win32":
- import msvcrt
-
- handle.seek(0)
- msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
- else:
- import fcntl
-
- fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
- except OSError:
- handle.close()
- return None
- return handle
-
-
-def _release_singleton_lock(handle) -> None:
- """????? _acquire_singleton_lock ?????????"""
- try:
- if sys.platform == "win32":
- import msvcrt
-
- handle.seek(0)
- msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
- else:
- import fcntl
-
- fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
- except OSError:
- pass
- finally:
- try:
- handle.close()
- except OSError:
- pass
-
-
-async def _curator_scheduler_loop() -> None:
- """??????:?? _CURATOR_CHECK_INTERVAL_SECONDS ???????????
-
- ? worker ???????? leader ?? ?? ???????? worker ????
- ????,?? worker ????,?? curator ?????????? LLM?
- ? cancel ? CancelledError ??? ``except Exception``(?? BaseException
- ??,?????),?????????? finally ?????
- """
- from deerflow.config.runtime_paths import runtime_home
- from deerflow.skills.curator import maybe_run_curator
-
- lock_handle = _acquire_singleton_lock(runtime_home() / ".curator_scheduler.lock")
- if lock_handle is None:
- logger.info("Curator scheduler: ??? worker ??????,? worker ??")
- return
-
- logger.info("Curator scheduler: ?????,? worker ?? curator ??")
- try:
- # ?????????????,??????
- await asyncio.sleep(_CURATOR_STARTUP_DELAY_SECONDS)
- while True:
- try:
- await maybe_run_curator()
- except Exception:
- logger.exception("Curator scheduler tick failed (non-fatal)")
- await asyncio.sleep(_CURATOR_CHECK_INTERVAL_SECONDS)
- finally:
- _release_singleton_lock(lock_handle)
-
-
-def _env_bool(name: str, default: bool) -> bool:
- raw = os.getenv(name)
- if raw is None:
- return default
- return raw.strip().lower() not in {"0", "false", "no", "off"}
-
-
-def _env_int(name: str, default: int, *, alias: str | None = None, minimum: int = 1) -> int:
- raw = os.getenv(name)
- if raw is None and alias:
- raw = os.getenv(alias)
- if raw is None:
- raw = str(default)
- try:
- value = int(raw)
- except ValueError:
- return default
- return max(minimum, value)
-
-
-def _env_float(name: str, default: float, *, alias: str | None = None, minimum: float = 0.0) -> float:
- raw = os.getenv(name)
- if raw is None and alias:
- raw = os.getenv(alias)
- if raw is None:
- raw = str(default)
- try:
- value = float(raw)
- except ValueError:
- return default
- return max(minimum, value)
-
-
-_LEADERBOARD_SNAPSHOT_BACKGROUND_ENABLED = _env_bool("ADMIN_LEADERBOARD_BACKGROUND_ENABLED", True)
-_LEADERBOARD_SNAPSHOT_PREWARM_DAYS = _env_int(
- "ADMIN_LEADERBOARD_BACKGROUND_PREWARM_DAYS",
- 30,
-)
-_LEADERBOARD_SNAPSHOT_BATCH_SIZE = _env_int(
- "ADMIN_LEADERBOARD_BACKGROUND_BATCH_SIZE",
- 1,
-)
-_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS = _env_float(
- "ADMIN_LEADERBOARD_BACKGROUND_INTERVAL_SECONDS",
- 300.0,
-)
-_LEADERBOARD_SNAPSHOT_SLEEP_SECONDS = _env_float(
- "ADMIN_LEADERBOARD_BACKGROUND_SLEEP_SECONDS",
- 2.0,
-)
-_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS = _env_int(
- "ADMIN_LEADERBOARD_BACKGROUND_RUNNING_TTL_SECONDS",
- 1800,
-)
-_LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
- "ADMIN_LEADERBOARD_BACKGROUND_STARTUP_DELAY_SECONDS",
- 60.0,
-)
-# Concurrency (system-pressure) sampler: how often to snapshot the in-flight run
-# count, and how long to keep samples (drives the admin 并发量 line chart, which
-# aggregates per minute/hour). Sampling is in-memory cheap (one tiny INSERT).
-_CONCURRENCY_SAMPLE_ENABLED = _env_bool("CONCURRENCY_SAMPLE_ENABLED", True)
-_CONCURRENCY_SAMPLE_INTERVAL_SECONDS = _env_float(
- "CONCURRENCY_SAMPLE_INTERVAL_SECONDS",
- 15.0,
-)
-_CONCURRENCY_SAMPLE_RETENTION_HOURS = _env_float(
- "CONCURRENCY_SAMPLE_RETENTION_HOURS",
- 168.0, # 7 days
-)
-# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
-# *previous* day's snapshot once, so records that landed near midnight are fully
-# captured. Default 1 ? "???? 1 ????????".
-_LEADERBOARD_FINALIZE_HOUR = min(
- 23,
- _env_int("ADMIN_LEADERBOARD_FINALIZE_HOUR", 1, minimum=0),
-)
-
-
-def _leaderboard_scheduler_sql_for_log(stmt) -> str:
- try:
- return str(stmt.compile(compile_kwargs={"literal_binds": True}))
- except Exception:
- return str(stmt)
-
-
-async def _execute_leaderboard_scheduler_query(session, step: str, stmt, **context):
- sql = _leaderboard_scheduler_sql_for_log(stmt)
- logger.info(
- "Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
- step,
- context.get("stat_date") or context.get("stat_dates") or "",
- context.get("job_key") or "",
- context.get("settings_hash") or "",
- datetime.now().isoformat(),
- sql,
- )
- start = perf_counter()
- try:
- result = await session.execute(stmt)
- except Exception:
- logger.exception(
- "Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
- step,
- context.get("stat_date") or context.get("stat_dates") or "",
- context.get("job_key") or "",
- context.get("settings_hash") or "",
- (perf_counter() - start) * 1000,
- )
- raise
- logger.info(
- "Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
- step,
- context.get("stat_date") or context.get("stat_dates") or "",
- context.get("job_key") or "",
- context.get("settings_hash") or "",
- datetime.now().isoformat(),
- (perf_counter() - start) * 1000,
- )
- return result
-
-
-async def _time_leaderboard_scheduler_db_operation(step: str, operation, *, sql: str = "", **context):
- logger.info(
- "Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
- step,
- context.get("stat_date") or context.get("stat_dates") or "",
- context.get("job_key") or "",
- context.get("settings_hash") or "",
- datetime.now().isoformat(),
- sql,
- )
- start = perf_counter()
- try:
- result = await operation()
- except Exception:
- logger.exception(
- "Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
- step,
- context.get("stat_date") or context.get("stat_dates") or "",
- context.get("job_key") or "",
- context.get("settings_hash") or "",
- (perf_counter() - start) * 1000,
- )
- raise
- logger.info(
- "Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
- step,
- context.get("stat_date") or context.get("stat_dates") or "",
- context.get("job_key") or "",
- context.get("settings_hash") or "",
- datetime.now().isoformat(),
- (perf_counter() - start) * 1000,
- )
- return result
-
-
-def _recent_leaderboard_stat_dates(days: int) -> list[str]:
- from datetime import datetime, timedelta
-
- from deerflow.persistence.types import BEIJING_TZ
-
- today = datetime.now(BEIJING_TZ).date()
- start = today - timedelta(days=max(1, days) - 1)
- return [(start + timedelta(days=i)).isoformat() for i in range((today - start).days + 1)]
-
-
-async def _leaderboard_snapshot_due_dates(
- session_factory,
- *,
- settings_hash: str,
- limit: int,
- running_ttl_seconds: int,
-) -> list[str]:
- from datetime import UTC, datetime, timedelta
-
- from sqlalchemy import and_, or_, select
-
- from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
-
- cutoff = datetime.now(UTC) - timedelta(seconds=running_ttl_seconds)
- async with session_factory() as session:
- stmt = (
- select(AdminLeaderboardDailyStatRow.stat_date)
- .where(
- AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
- or_(
- AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
- and_(
- AdminLeaderboardDailyStatRow.status == "running",
- AdminLeaderboardDailyStatRow.updated_at < cutoff,
- ),
- ),
- )
- .order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
- .limit(limit)
- )
- rows = (
- await _execute_leaderboard_scheduler_query(
- session,
- "scheduler.due_dates",
- stmt,
- settings_hash=settings_hash,
- )
- ).all()
- return [str(row[0]) for row in rows]
-
-
-async def _claim_leaderboard_snapshot(
- session_factory,
- *,
- stat_date: str,
- settings_hash: str,
- running_ttl_seconds: int,
-) -> bool:
- from datetime import UTC, datetime
-
- from app.gateway.routers.admin_users import _snapshot_is_fresh
- from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
- from deerflow.persistence.types import BEIJING_TZ
-
- async with session_factory() as session:
- row = await _time_leaderboard_scheduler_db_operation(
- "scheduler.claim.get",
- lambda: session.get(
- AdminLeaderboardDailyStatRow,
- {"stat_date": stat_date, "settings_hash": settings_hash},
- ),
- sql="SELECT admin_leaderboard_daily_stats by primary key",
- stat_date=stat_date,
- settings_hash=settings_hash,
- )
- if row is None or _snapshot_is_fresh(row, stat_date):
- return False
- if row.status == "running":
- updated_at = row.updated_at
- if isinstance(updated_at, datetime):
- if updated_at.tzinfo is None:
- updated_at = updated_at.replace(tzinfo=BEIJING_TZ)
- age = (datetime.now(BEIJING_TZ) - updated_at.astimezone(BEIJING_TZ)).total_seconds()
- if age < running_ttl_seconds:
- return False
- row.status = "running"
- row.message = "?????????????"
- row.error = None
- row.updated_at = datetime.now(UTC)
- await _time_leaderboard_scheduler_db_operation(
- "scheduler.claim.commit",
- lambda: session.commit(),
- sql="COMMIT",
- stat_date=stat_date,
- settings_hash=settings_hash,
- )
- return True
-
-
-async def _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash: str) -> bool:
- """Once per day after ``_LEADERBOARD_FINALIZE_HOUR``, force-recompute yesterday.
-
- A ``ready`` past-day snapshot is otherwise treated as fresh forever and never
- rebuilt. But the previous day's snapshot is usually generated before midnight,
- so it can miss runs/tool-calls that only land in the DB around the day
- boundary. This re-queues yesterday (``force=True``) a single time; the rebuild
- stamps ``generated_at`` to today, which stops it from re-firing for the rest of
- the day.
- """
- from datetime import timedelta
-
- from app.gateway.routers.admin_users import (
- _as_beijing,
- _queue_daily_leaderboard_snapshots,
- )
- from deerflow.persistence.admin_stats.finalize import should_finalize_previous_day
- from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
- from deerflow.persistence.types import BEIJING_TZ
-
- now = datetime.now(BEIJING_TZ)
- yesterday = (now.date() - timedelta(days=1)).isoformat()
- async with session_factory() as session:
- row = await session.get(
- AdminLeaderboardDailyStatRow,
- {"stat_date": yesterday, "settings_hash": settings_hash},
- )
- generated_at = _as_beijing(row.generated_at) if (row is not None and row.generated_at is not None) else None
- if not should_finalize_previous_day(
- status=(row.status if row is not None else None),
- generated_at=generated_at,
- now=now,
- finalize_hour=_LEADERBOARD_FINALIZE_HOUR,
- ):
- return False
- queued = await _queue_daily_leaderboard_snapshots(
- session_factory,
- [yesterday],
- settings_hash,
- force=True,
- message="???????????????????????????",
- )
- if queued:
- logger.info(
- "Leaderboard snapshot finalize: queued previous day %s for full recompute (hash=%s)",
- yesterday,
- settings_hash[:10],
- )
- return True
- return False
-
-
-async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
- from app.gateway.routers.admin_users import (
- _daily_job_key,
- _queue_daily_leaderboard_snapshots,
- _refresh_daily_leaderboard_snapshot,
- _resolve_leaderboard_snapshot_scope,
- )
- from deerflow.config.system_settings import load_system_settings
- from deerflow.persistence.engine import get_session_factory
-
- session_factory = get_session_factory()
- if session_factory is None:
- return 0, 0
-
- settings = load_system_settings().leaderboard
- excluded, include_scheduled, include_admins, include_failed, settings_hash = _resolve_leaderboard_snapshot_scope(settings)
- stat_dates = _recent_leaderboard_stat_dates(_LEADERBOARD_SNAPSHOT_PREWARM_DAYS)
- queued_dates = await _queue_daily_leaderboard_snapshots(
- session_factory,
- stat_dates,
- settings_hash,
- message="?????????????/?????",
- )
-
- # ??????? 1 ???????"???"????????????????????
- try:
- await _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash)
- except Exception:
- logger.exception("Leaderboard snapshot finalize check failed (non-fatal)")
-
- due_dates = await _leaderboard_snapshot_due_dates(
- session_factory,
- settings_hash=settings_hash,
- limit=_LEADERBOARD_SNAPSHOT_BATCH_SIZE,
- running_ttl_seconds=_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS,
- )
- processed = 0
- for stat_date in due_dates:
- claimed = await _claim_leaderboard_snapshot(
- session_factory,
- stat_date=stat_date,
- settings_hash=settings_hash,
- running_ttl_seconds=_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS,
- )
- if not claimed:
- continue
- logger.info("Leaderboard snapshot scheduler: building date=%s hash=%s", stat_date, settings_hash[:10])
- await _refresh_daily_leaderboard_snapshot(
- _daily_job_key(stat_date, settings_hash),
- session_factory=session_factory,
- settings=settings,
- stat_date=stat_date,
- settings_hash=settings_hash,
- excluded=excluded,
- include_scheduled=include_scheduled,
- include_admins=include_admins,
- include_failed=include_failed,
- )
- processed += 1
- if _LEADERBOARD_SNAPSHOT_SLEEP_SECONDS > 0 and processed < len(due_dates):
- await asyncio.sleep(_LEADERBOARD_SNAPSHOT_SLEEP_SECONDS)
- return processed, len(queued_dates)
-
-
-async def _leaderboard_snapshot_scheduler_loop() -> None:
- """Background task inside the existing gateway process for daily snapshots."""
- if not _LEADERBOARD_SNAPSHOT_BACKGROUND_ENABLED:
- logger.info("Leaderboard snapshot scheduler disabled by ADMIN_LEADERBOARD_BACKGROUND_ENABLED")
- return
-
- from deerflow.config.runtime_paths import runtime_home
-
- lock_handle = _acquire_singleton_lock(runtime_home() / ".leaderboard_snapshot_scheduler.lock")
- if lock_handle is None:
- logger.info("Leaderboard snapshot scheduler: another gateway worker holds the lock; this worker skips")
- return
-
- logger.info(
- "Leaderboard snapshot scheduler started: prewarm_days=%d batch_size=%d interval=%.1fs",
- _LEADERBOARD_SNAPSHOT_PREWARM_DAYS,
- _LEADERBOARD_SNAPSHOT_BATCH_SIZE,
- _LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS,
- )
- try:
- if _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS > 0:
- await asyncio.sleep(_LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS)
- while True:
- try:
- processed, queued = await _run_leaderboard_snapshot_tick()
- if processed or queued:
- logger.info(
- "Leaderboard snapshot scheduler tick: queued=%d processed=%d",
- queued,
- processed,
- )
- except Exception:
- logger.exception("Leaderboard snapshot scheduler tick failed (non-fatal)")
- await asyncio.sleep(_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS)
- finally:
- _release_singleton_lock(lock_handle)
-
-
-async def _concurrency_sampler_loop(app: FastAPI) -> None:
- """Periodically record the in-flight run count for the admin pressure chart.
-
- Runs in the gateway process, singleton-locked so only one worker samples.
- Each tick records ``len(run_manager.list_active())`` and, roughly hourly,
- purges samples older than the retention window. Best-effort throughout — a
- failed tick is logged and the loop continues.
- """
- if not _CONCURRENCY_SAMPLE_ENABLED:
- logger.info("Concurrency sampler disabled by CONCURRENCY_SAMPLE_ENABLED")
- return
-
- store = getattr(app.state, "concurrency_sample_store", None)
- run_mgr = getattr(app.state, "run_manager", None)
- concurrency_gate = getattr(app.state, "concurrency_gate", None)
- if store is None or run_mgr is None:
- logger.info("Concurrency sampler not started (no store / run manager — memory backend?)")
- return
-
- from deerflow.config.runtime_paths import runtime_home
-
- lock_handle = _acquire_singleton_lock(runtime_home() / ".concurrency_sampler.lock")
- if lock_handle is None:
- logger.info("Concurrency sampler: another gateway worker holds the lock; this worker skips")
- return
-
- interval = max(1.0, _CONCURRENCY_SAMPLE_INTERVAL_SECONDS)
- purge_every_ticks = max(1, int(3600 / interval)) # ~once per hour
- logger.info("Concurrency sampler started: interval=%.1fs retention=%.1fh", interval, _CONCURRENCY_SAMPLE_RETENTION_HOURS)
- ticks = 0
- try:
- while True:
- try:
- # Runtime toggle (admin can turn sampling off without a restart).
- # mtime-cached read, so this is cheap every tick.
- from deerflow.config.system_settings import get_concurrency_monitor_settings
-
- if get_concurrency_monitor_settings().enabled:
- active_count = None
- if concurrency_gate is not None:
- active_count = await concurrency_gate.count_active()
- if active_count is None:
- active = await run_mgr.list_active()
- active_count = len(active)
- await store.record(active_count)
- ticks += 1
- if ticks % purge_every_ticks == 0:
- from datetime import UTC, datetime, timedelta
-
- cutoff = datetime.now(UTC) - timedelta(hours=_CONCURRENCY_SAMPLE_RETENTION_HOURS)
- await store.purge_older_than(cutoff)
- except Exception:
- logger.exception("Concurrency sampler tick failed (non-fatal)")
- await asyncio.sleep(interval)
- finally:
- _release_singleton_lock(lock_handle)
-
-
-def _warn_if_multi_worker() -> None:
- """?? uvicorn / gunicorn ? worker ???? ERROR ???
-
- ? agent / canvas / AI ??????? Gateway ????? ``StreamBridge``
- ?? run_id ? SSE ?????? map?? worker ????? worker A ??
- ``/runs/stream``???? worker B ? join/resume??? in-process map ??
- ???? 404 / ???
-
- ????? startup ? ERROR ????**???? fail-fast** ?? ??
- ???????????????? sticky session??????????
- """
- # uvicorn / gunicorn ??????? worker ?????????????
- # ?????? sibling ??????????? WEB_CONCURRENCY?gunicorn
- # / uvicorn ????
- web_concurrency = os.environ.get("WEB_CONCURRENCY", "").strip()
- if web_concurrency and web_concurrency.isdigit() and int(web_concurrency) > 1:
- logger.error(
- "? ??? WEB_CONCURRENCY=%s?? worker ????AI ?? / ? agent "
- "??????????????? worker ????????? / "
- "SSE ?? / resume 400??????????? WEB_CONCURRENCY=1 "
- "????????? sticky session??? AIWritingPage.tsx ????",
- web_concurrency,
- )
- return
-
- # ???? /proc ??? uvicorn ?????? Linux ??????
- if not sys.platform.startswith("linux"):
- return
- try:
- from pathlib import Path
-
- peers = 0
- for entry in Path("/proc").iterdir():
- if not entry.name.isdigit():
- continue
- try:
- cmdline = (entry / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "ignore")
- except OSError:
- continue
- if "uvicorn" in cmdline and "app.gateway.app" in cmdline:
- peers += 1
- if peers > 1:
- logger.error(
- "? ??? %d ??? uvicorn ???? worker ????? AI ?? "
- "????? / SSE ??????????? worker ??????? "
- "?? sticky session?",
- peers,
- )
- except Exception:
- # ????????? ?? ??????
- pass
-
-
-@asynccontextmanager
-async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
- """Application lifespan handler."""
-
- # ?????? worker ?? ?? ?? AI ?? / ? agent ???????
- # ??????? worker ???????? ERROR ?????
- _warn_if_multi_worker()
-
- # Load config and check necessary environment variables at startup
- try:
- app.state.config = get_app_config()
- apply_logging_level(app.state.config.log_level)
- logger.info("Configuration loaded successfully")
- except Exception as e:
- error_msg = f"Failed to load configuration during gateway startup: {e}"
- logger.exception(error_msg)
- raise RuntimeError(error_msg) from e
- config = get_gateway_config()
- logger.info(f"Starting API Gateway on {config.host}:{config.port}")
-
- # Initialize LangGraph runtime components (StreamBridge, RunManager, checkpointer, store)
- async with langgraph_runtime(app):
- logger.info("LangGraph runtime initialised")
-
+ llmwiki_index,
+ mcp,
+ memory,
+ menu_overrides,
+ model_management,
+ models,
+ multi_agent,
+ notifications,
+ open_3qfx,
+ open_chat,
+ parallel_agent_tasks,
+ parallel_agents,
+ position_roundtable,
+ positions,
+ position_roles,
+ public_config,
+ public_embed,
+ public_leaderboard,
+ public_scheduled_tasks,
+ recommend,
+ flow_extract,
+ recommended_questions,
+ roundtable_chains,
+ roundtable_artifact_submissions,
+ roundtable_diagnostics,
+ roundtable_draft_shares,
+ roundtable_drafts,
+ roundtable_jobs,
+ roundtable_task_drafts,
+ runs,
+ scheduled_tasks,
+ sensitive_words,
+ skill_addresses,
+ skills,
+ suggestions,
+ system_settings,
+ tags,
+ task_buttons,
+ taskcop_tasks,
+ report_structures,
+ task_reports,
+ thread_runs,
+ tool_metrics,
+ thread_shares,
+ threads,
+ uploads,
+ user_preferences,
+ user_prompts,
+ writing,
+)
+from deerflow.config import app_config as deerflow_app_config
+from deerflow.config.app_config import apply_logging_level
+from deerflow.runtime.scheduler import ScheduledTaskService, set_scheduled_task_service
+
+AppConfig = deerflow_app_config.AppConfig
+get_app_config = deerflow_app_config.get_app_config
+
+# Default logging; lifespan overrides from config.yaml log_level.
+logging.basicConfig(
+ level=logging.INFO,
+ format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+ datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+logger = logging.getLogger(__name__)
+
+# Upper bound (seconds) each lifespan shutdown hook is allowed to run.
+# Bounds worker exit time so uvicorn's reload supervisor does not keep
+# firing signals into a worker that is stuck waiting for shutdown cleanup.
+_SHUTDOWN_HOOK_TIMEOUT_SECONDS = 5.0
+
+
+async def _ensure_admin_user(app: FastAPI) -> None:
+ """Startup hook: handle first boot and migrate orphan threads otherwise.
+
+ After admin creation, migrate orphan threads from the LangGraph
+ store (metadata.user_id unset) to the admin account. This is the
+ "no-auth ? with-auth" upgrade path: users who ran DeerFlow without
+ authentication have existing LangGraph thread data that needs an
+ owner assigned.
+ First boot (no admin exists):
+ - Does NOT create any user accounts automatically.
+ - The operator must visit ``/setup`` to create the first admin.
+
+ Subsequent boots (admin already exists):
+ - Runs the one-time "no-auth ? with-auth" orphan thread migration for
+ existing LangGraph thread metadata that has no owner_id.
+
+ No SQL persistence migration is needed: the four user_id columns
+ (threads_meta, runs, run_events, feedback) only come into existence
+ alongside the auth module via create_all, so freshly created tables
+ never contain NULL-owner rows.
+ """
+ from sqlalchemy import select
+
+ from app.gateway.deps import get_local_provider
+ from deerflow.persistence.engine import get_session_factory
+ from deerflow.persistence.user.model import UserRow
+
+ try:
+ provider = get_local_provider()
+ except RuntimeError:
+ # Auth persistence may not be initialized in some test/boot paths.
+ # Skip admin migration work rather than failing gateway startup.
+ logger.warning("Auth persistence not ready; skipping admin bootstrap check")
+ return
+
+ sf = get_session_factory()
+ if sf is None:
+ return
+
+ admin_count = await provider.count_admin_users()
+
+ if admin_count == 0:
+ logger.info("=" * 60)
+ logger.info(" First boot detected ? no admin account exists.")
+ logger.info(" Visit /setup to complete admin account creation.")
+ logger.info("=" * 60)
+ return
+
+ # Admin already exists ? run orphan thread migration for any
+ # LangGraph thread metadata that pre-dates the auth module.
+ async with sf() as session:
+ stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
+ row = (await session.execute(stmt)).scalar_one_or_none()
+
+ if row is None:
+ return # Should not happen (admin_count > 0 above), but be safe.
+
+ admin_id = str(row.id)
+
+ # LangGraph store orphan migration ? non-fatal.
+ # This covers the "no-auth ? with-auth" upgrade path for users
+ # whose existing LangGraph thread metadata has no user_id set.
+ store = getattr(app.state, "store", None)
+ if store is not None:
+ try:
+ migrated = await _migrate_orphaned_threads(store, admin_id)
+ if migrated:
+ logger.info("Migrated %d orphan LangGraph thread(s) to admin", migrated)
+ except Exception:
+ logger.exception("LangGraph thread migration failed (non-fatal)")
+
+
+async def _iter_store_items(store, namespace, *, page_size: int = 500):
+ """Paginated async iterator over a LangGraph store namespace.
+
+ Replaces the old hardcoded ``limit=1000`` call with a cursor-style
+ loop so that environments with more than one page of orphans do
+ not silently lose data. Terminates when a page is empty OR when a
+ short page arrives (indicating the last page).
+ """
+ offset = 0
+ while True:
+ batch = await store.asearch(namespace, limit=page_size, offset=offset)
+ if not batch:
+ return
+ for item in batch:
+ yield item
+ if len(batch) < page_size:
+ return
+ offset += page_size
+
+
+async def _sync_legacy_skills(app: FastAPI) -> None:
+ """Idempotently register existing on-disk ``skills/custom/*`` directories
+ in the ``skills`` DB table, owned by the first admin and published=True.
+
+ Without this step, after upgrading to per-user skill ownership the existing
+ custom skill folders would not be visible to anyone (the SkillStore
+ visibility filter relies on a DB row). Running this once on every boot is
+ cheap (folder scan + one ``ensure_legacy`` per name) and idempotent.
+
+ Non-fatal: a failure here MUST NOT block server startup.
+ """
+ from sqlalchemy import select
+
+ from deerflow.persistence.engine import get_session_factory
+ from deerflow.persistence.user.model import UserRow
+ from deerflow.skills.storage import get_or_new_skill_storage
+ from deerflow.skills.types import SkillCategory
+
+ sf = get_session_factory()
+ if sf is None:
+ return
+
+ skill_store = getattr(app.state, "skill_store", None)
+ if skill_store is None:
+ return
+
+ async with sf() as session:
+ stmt = select(UserRow).where(UserRow.system_role == "admin").limit(1)
+ row = (await session.execute(stmt)).scalar_one_or_none()
+ if row is None:
+ # No admin yet ? leave ownership null; first admin is responsible for
+ # claiming via the UI once they exist.
+ admin_id: str | None = None
+ else:
+ admin_id = str(row.id)
+
+ config = getattr(app.state, "config", None)
+ storage = get_or_new_skill_storage(app_config=config)
+ try:
+ skills = storage.load_skills(enabled_only=False)
+ except Exception:
+ logger.exception("_sync_legacy_skills: could not list on-disk skills")
+ return
+
+ inserted = 0
+ for skill in skills:
+ if skill.category != SkillCategory.CUSTOM:
+ continue
+ try:
+ existing = await skill_store.get_any(skill.name)
+ if existing is not None:
+ continue
+ await skill_store.ensure_legacy(
+ {"name": skill.name, "owner_user_id": admin_id, "published": True}
+ )
+ inserted += 1
+ except Exception:
+ logger.warning("_sync_legacy_skills: could not register %s", skill.name, exc_info=True)
+ if inserted:
+ logger.info("Skill ownership migration: registered %d legacy custom skill(s)", inserted)
+
+
+async def _migrate_orphaned_threads(store, admin_user_id: str) -> int:
+ """Migrate LangGraph store threads with no user_id to the given admin.
+
+ Uses cursor pagination so all orphans are migrated regardless of
+ count. Returns the number of rows migrated.
+ """
+ migrated = 0
+ async for item in _iter_store_items(store, ("threads",)):
+ metadata = item.value.get("metadata", {})
+ if not metadata.get("user_id"):
+ metadata["user_id"] = admin_user_id
+ item.value["metadata"] = metadata
+ await store.aput(("threads",), item.key, item.value)
+ migrated += 1
+ return migrated
+
+
+async def _run_db_migrations() -> None:
+ """Run Alembic migrations at startup so schema changes are applied automatically."""
+ from pathlib import Path
+
+ from alembic import command
+ from alembic.config import Config
+
+ import deerflow.persistence as _persistence_pkg
+ from deerflow.persistence.engine import get_engine
+
+ engine = get_engine()
+ if engine is None:
+ logger.info("Persistence backend=memory ? skipping Alembic migrations")
+ return
+
+ if os.getenv("DEER_FLOW_SKIP_ALEMBIC", "").lower() in {"1", "true", "yes"}:
+ logger.warning("DEER_FLOW_SKIP_ALEMBIC is set; skipping Alembic migrations")
+ return
+
+ db_url = engine.url.render_as_string(hide_password=False).replace("%", "%%")
+ migrations_dir = str(Path(_persistence_pkg.__file__).parent / "migrations")
+ result_queue: queue.Queue[BaseException | None] = queue.Queue(maxsize=1)
+
+ def _run_sync() -> None:
+ try:
+ cfg = Config()
+ cfg.set_main_option("script_location", migrations_dir)
+ cfg.set_main_option("sqlalchemy.url", db_url)
+ command.upgrade(cfg, "head")
+ except BaseException as exc:
+ result_queue.put(exc)
+ else:
+ result_queue.put(None)
+
+ try:
+ try:
+ timeout_s = float(os.getenv("DEER_FLOW_ALEMBIC_TIMEOUT", "60"))
+ except ValueError:
+ timeout_s = 60.0
+
+ worker = threading.Thread(target=_run_sync, name="deerflow-alembic-upgrade", daemon=True)
+ worker.start()
+ try:
+ deadline = asyncio.get_running_loop().time() + timeout_s
+ while result_queue.empty():
+ if asyncio.get_running_loop().time() >= deadline:
+ raise asyncio.TimeoutError
+ await asyncio.sleep(0.2)
+ result = result_queue.get_nowait()
+ except asyncio.TimeoutError:
+ logger.error(
+ "Database migration did not finish within %.0fs; continuing startup on create_all schema. "
+ "Likely cause: MySQL metadata-lock wait or a slow DDL. Set DEER_FLOW_SKIP_ALEMBIC=1 "
+ "to skip startup migrations, or inspect MySQL with SHOW FULL PROCESSLIST.",
+ timeout_s,
+ )
+ return
+ if result is not None:
+ raise result
+ logger.info("Database migrations applied (head)")
+ except Exception:
+ logger.exception("Database migration failed ? schema may be out of date")
+
+
+# Skill curator ?????
+# - ????:?????????"??????";??????? should_run_now() ???
+# - ????:?????????????,???????
+_CURATOR_CHECK_INTERVAL_SECONDS = 3600 # 1 ??
+_CURATOR_STARTUP_DELAY_SECONDS = 300 # 5 ??
+
+
+def _acquire_singleton_lock(lock_path):
+ """???????????,??? worker ?? leader ???
+
+ ?? -> ??????????(?????????,??????/????);
+ ????(???? worker ??)????? -> ?? None?
+ ??????? OS ????????
+ """
+ try:
+ lock_path.parent.mkdir(parents=True, exist_ok=True)
+ handle = open(lock_path, "a+")
+ except OSError:
+ logger.exception("Curator scheduler: ???????")
+ return None
+ try:
+ if sys.platform == "win32":
+ import msvcrt
+
+ handle.seek(0)
+ msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
+ else:
+ import fcntl
+
+ fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except OSError:
+ handle.close()
+ return None
+ return handle
+
+
+def _release_singleton_lock(handle) -> None:
+ """????? _acquire_singleton_lock ?????????"""
+ try:
+ if sys.platform == "win32":
+ import msvcrt
+
+ handle.seek(0)
+ msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
+ else:
+ import fcntl
+
+ fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
+ except OSError:
+ pass
+ finally:
+ try:
+ handle.close()
+ except OSError:
+ pass
+
+
+async def _curator_scheduler_loop() -> None:
+ """??????:?? _CURATOR_CHECK_INTERVAL_SECONDS ???????????
+
+ ? worker ???????? leader ?? ?? ???????? worker ????
+ ????,?? worker ????,?? curator ?????????? LLM?
+ ? cancel ? CancelledError ??? ``except Exception``(?? BaseException
+ ??,?????),?????????? finally ?????
+ """
+ from deerflow.config.runtime_paths import runtime_home
+ from deerflow.skills.curator import maybe_run_curator
+
+ lock_handle = _acquire_singleton_lock(runtime_home() / ".curator_scheduler.lock")
+ if lock_handle is None:
+ logger.info("Curator scheduler: ??? worker ??????,? worker ??")
+ return
+
+ logger.info("Curator scheduler: ?????,? worker ?? curator ??")
+ try:
+ # ?????????????,??????
+ await asyncio.sleep(_CURATOR_STARTUP_DELAY_SECONDS)
+ while True:
+ try:
+ await maybe_run_curator()
+ except Exception:
+ logger.exception("Curator scheduler tick failed (non-fatal)")
+ await asyncio.sleep(_CURATOR_CHECK_INTERVAL_SECONDS)
+ finally:
+ _release_singleton_lock(lock_handle)
+
+
+def _env_bool(name: str, default: bool) -> bool:
+ raw = os.getenv(name)
+ if raw is None:
+ return default
+ return raw.strip().lower() not in {"0", "false", "no", "off"}
+
+
+def _env_int(name: str, default: int, *, alias: str | None = None, minimum: int = 1) -> int:
+ raw = os.getenv(name)
+ if raw is None and alias:
+ raw = os.getenv(alias)
+ if raw is None:
+ raw = str(default)
+ try:
+ value = int(raw)
+ except ValueError:
+ return default
+ return max(minimum, value)
+
+
+def _env_float(name: str, default: float, *, alias: str | None = None, minimum: float = 0.0) -> float:
+ raw = os.getenv(name)
+ if raw is None and alias:
+ raw = os.getenv(alias)
+ if raw is None:
+ raw = str(default)
+ try:
+ value = float(raw)
+ except ValueError:
+ return default
+ return max(minimum, value)
+
+
+_LEADERBOARD_SNAPSHOT_BACKGROUND_ENABLED = _env_bool("ADMIN_LEADERBOARD_BACKGROUND_ENABLED", True)
+_LEADERBOARD_SNAPSHOT_PREWARM_DAYS = _env_int(
+ "ADMIN_LEADERBOARD_BACKGROUND_PREWARM_DAYS",
+ 30,
+)
+_LEADERBOARD_SNAPSHOT_BATCH_SIZE = _env_int(
+ "ADMIN_LEADERBOARD_BACKGROUND_BATCH_SIZE",
+ 1,
+)
+_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS = _env_float(
+ "ADMIN_LEADERBOARD_BACKGROUND_INTERVAL_SECONDS",
+ 300.0,
+)
+_LEADERBOARD_SNAPSHOT_SLEEP_SECONDS = _env_float(
+ "ADMIN_LEADERBOARD_BACKGROUND_SLEEP_SECONDS",
+ 2.0,
+)
+_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS = _env_int(
+ "ADMIN_LEADERBOARD_BACKGROUND_RUNNING_TTL_SECONDS",
+ 1800,
+)
+_LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS = _env_float(
+ "ADMIN_LEADERBOARD_BACKGROUND_STARTUP_DELAY_SECONDS",
+ 60.0,
+)
+# Concurrency (system-pressure) sampler: how often to snapshot the in-flight run
+# count, and how long to keep samples (drives the admin 并发量 line chart, which
+# aggregates per minute/hour). Sampling is in-memory cheap (one tiny INSERT).
+_CONCURRENCY_SAMPLE_ENABLED = _env_bool("CONCURRENCY_SAMPLE_ENABLED", True)
+_CONCURRENCY_SAMPLE_INTERVAL_SECONDS = _env_float(
+ "CONCURRENCY_SAMPLE_INTERVAL_SECONDS",
+ 15.0,
+)
+_CONCURRENCY_SAMPLE_RETENTION_HOURS = _env_float(
+ "CONCURRENCY_SAMPLE_RETENTION_HOURS",
+ 168.0, # 7 days
+)
+# Hour (local Beijing time, 0-23) after which the scheduler force-recomputes the
+# *previous* day's snapshot once, so records that landed near midnight are fully
+# captured. Default 1 ? "???? 1 ????????".
+_LEADERBOARD_FINALIZE_HOUR = min(
+ 23,
+ _env_int("ADMIN_LEADERBOARD_FINALIZE_HOUR", 1, minimum=0),
+)
+
+
+def _leaderboard_scheduler_sql_for_log(stmt) -> str:
+ try:
+ return str(stmt.compile(compile_kwargs={"literal_binds": True}))
+ except Exception:
+ return str(stmt)
+
+
+async def _execute_leaderboard_scheduler_query(session, step: str, stmt, **context):
+ sql = _leaderboard_scheduler_sql_for_log(stmt)
+ logger.info(
+ "Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ datetime.now().isoformat(),
+ sql,
+ )
+ start = perf_counter()
+ try:
+ result = await session.execute(stmt)
+ except Exception:
+ logger.exception(
+ "Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ (perf_counter() - start) * 1000,
+ )
+ raise
+ logger.info(
+ "Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ datetime.now().isoformat(),
+ (perf_counter() - start) * 1000,
+ )
+ return result
+
+
+async def _time_leaderboard_scheduler_db_operation(step: str, operation, *, sql: str = "", **context):
+ logger.info(
+ "Leaderboard snapshot DB query start: step=%s stat_date=%s job=%s settings_hash=%s started_at=%s sql=%s",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ datetime.now().isoformat(),
+ sql,
+ )
+ start = perf_counter()
+ try:
+ result = await operation()
+ except Exception:
+ logger.exception(
+ "Leaderboard snapshot DB query failed: step=%s stat_date=%s job=%s settings_hash=%s elapsed_ms=%.2f",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ (perf_counter() - start) * 1000,
+ )
+ raise
+ logger.info(
+ "Leaderboard snapshot DB query done: step=%s stat_date=%s job=%s settings_hash=%s finished_at=%s elapsed_ms=%.2f",
+ step,
+ context.get("stat_date") or context.get("stat_dates") or "",
+ context.get("job_key") or "",
+ context.get("settings_hash") or "",
+ datetime.now().isoformat(),
+ (perf_counter() - start) * 1000,
+ )
+ return result
+
+
+def _recent_leaderboard_stat_dates(days: int) -> list[str]:
+ from datetime import datetime, timedelta
+
+ from deerflow.persistence.types import BEIJING_TZ
+
+ today = datetime.now(BEIJING_TZ).date()
+ start = today - timedelta(days=max(1, days) - 1)
+ return [(start + timedelta(days=i)).isoformat() for i in range((today - start).days + 1)]
+
+
+async def _leaderboard_snapshot_due_dates(
+ session_factory,
+ *,
+ settings_hash: str,
+ limit: int,
+ running_ttl_seconds: int,
+) -> list[str]:
+ from datetime import UTC, datetime, timedelta
+
+ from sqlalchemy import and_, or_, select
+
+ from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
+
+ cutoff = datetime.now(UTC) - timedelta(seconds=running_ttl_seconds)
+ async with session_factory() as session:
+ stmt = (
+ select(AdminLeaderboardDailyStatRow.stat_date)
+ .where(
+ AdminLeaderboardDailyStatRow.settings_hash == settings_hash,
+ or_(
+ AdminLeaderboardDailyStatRow.status.in_(["queued", "error"]),
+ and_(
+ AdminLeaderboardDailyStatRow.status == "running",
+ AdminLeaderboardDailyStatRow.updated_at < cutoff,
+ ),
+ ),
+ )
+ .order_by(AdminLeaderboardDailyStatRow.stat_date.desc())
+ .limit(limit)
+ )
+ rows = (
+ await _execute_leaderboard_scheduler_query(
+ session,
+ "scheduler.due_dates",
+ stmt,
+ settings_hash=settings_hash,
+ )
+ ).all()
+ return [str(row[0]) for row in rows]
+
+
+async def _claim_leaderboard_snapshot(
+ session_factory,
+ *,
+ stat_date: str,
+ settings_hash: str,
+ running_ttl_seconds: int,
+) -> bool:
+ from datetime import UTC, datetime
+
+ from app.gateway.routers.admin_users import _snapshot_is_fresh
+ from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
+ from deerflow.persistence.types import BEIJING_TZ
+
+ async with session_factory() as session:
+ row = await _time_leaderboard_scheduler_db_operation(
+ "scheduler.claim.get",
+ lambda: session.get(
+ AdminLeaderboardDailyStatRow,
+ {"stat_date": stat_date, "settings_hash": settings_hash},
+ ),
+ sql="SELECT admin_leaderboard_daily_stats by primary key",
+ stat_date=stat_date,
+ settings_hash=settings_hash,
+ )
+ if row is None or _snapshot_is_fresh(row, stat_date):
+ return False
+ if row.status == "running":
+ updated_at = row.updated_at
+ if isinstance(updated_at, datetime):
+ if updated_at.tzinfo is None:
+ updated_at = updated_at.replace(tzinfo=BEIJING_TZ)
+ age = (datetime.now(BEIJING_TZ) - updated_at.astimezone(BEIJING_TZ)).total_seconds()
+ if age < running_ttl_seconds:
+ return False
+ row.status = "running"
+ row.message = "?????????????"
+ row.error = None
+ row.updated_at = datetime.now(UTC)
+ await _time_leaderboard_scheduler_db_operation(
+ "scheduler.claim.commit",
+ lambda: session.commit(),
+ sql="COMMIT",
+ stat_date=stat_date,
+ settings_hash=settings_hash,
+ )
+ return True
+
+
+async def _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash: str) -> bool:
+ """Once per day after ``_LEADERBOARD_FINALIZE_HOUR``, force-recompute yesterday.
+
+ A ``ready`` past-day snapshot is otherwise treated as fresh forever and never
+ rebuilt. But the previous day's snapshot is usually generated before midnight,
+ so it can miss runs/tool-calls that only land in the DB around the day
+ boundary. This re-queues yesterday (``force=True``) a single time; the rebuild
+ stamps ``generated_at`` to today, which stops it from re-firing for the rest of
+ the day.
+ """
+ from datetime import timedelta
+
+ from app.gateway.routers.admin_users import (
+ _as_beijing,
+ _queue_daily_leaderboard_snapshots,
+ )
+ from deerflow.persistence.admin_stats.finalize import should_finalize_previous_day
+ from deerflow.persistence.admin_stats.model import AdminLeaderboardDailyStatRow
+ from deerflow.persistence.types import BEIJING_TZ
+
+ now = datetime.now(BEIJING_TZ)
+ yesterday = (now.date() - timedelta(days=1)).isoformat()
+ async with session_factory() as session:
+ row = await session.get(
+ AdminLeaderboardDailyStatRow,
+ {"stat_date": yesterday, "settings_hash": settings_hash},
+ )
+ generated_at = _as_beijing(row.generated_at) if (row is not None and row.generated_at is not None) else None
+ if not should_finalize_previous_day(
+ status=(row.status if row is not None else None),
+ generated_at=generated_at,
+ now=now,
+ finalize_hour=_LEADERBOARD_FINALIZE_HOUR,
+ ):
+ return False
+ queued = await _queue_daily_leaderboard_snapshots(
+ session_factory,
+ [yesterday],
+ settings_hash,
+ force=True,
+ message="???????????????????????????",
+ )
+ if queued:
+ logger.info(
+ "Leaderboard snapshot finalize: queued previous day %s for full recompute (hash=%s)",
+ yesterday,
+ settings_hash[:10],
+ )
+ return True
+ return False
+
+
+async def _run_leaderboard_snapshot_tick() -> tuple[int, int]:
+ from app.gateway.routers.admin_users import (
+ _daily_job_key,
+ _queue_daily_leaderboard_snapshots,
+ _refresh_daily_leaderboard_snapshot,
+ _resolve_leaderboard_snapshot_scope,
+ )
+ from deerflow.config.system_settings import load_system_settings
+ from deerflow.persistence.engine import get_session_factory
+
+ session_factory = get_session_factory()
+ if session_factory is None:
+ return 0, 0
+
+ settings = load_system_settings().leaderboard
+ excluded, include_scheduled, include_admins, include_failed, settings_hash = _resolve_leaderboard_snapshot_scope(settings)
+ stat_dates = _recent_leaderboard_stat_dates(_LEADERBOARD_SNAPSHOT_PREWARM_DAYS)
+ queued_dates = await _queue_daily_leaderboard_snapshots(
+ session_factory,
+ stat_dates,
+ settings_hash,
+ message="?????????????/?????",
+ )
+
+ # ??????? 1 ???????"???"????????????????????
+ try:
+ await _maybe_finalize_previous_leaderboard_day(session_factory, settings_hash)
+ except Exception:
+ logger.exception("Leaderboard snapshot finalize check failed (non-fatal)")
+
+ due_dates = await _leaderboard_snapshot_due_dates(
+ session_factory,
+ settings_hash=settings_hash,
+ limit=_LEADERBOARD_SNAPSHOT_BATCH_SIZE,
+ running_ttl_seconds=_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS,
+ )
+ processed = 0
+ for stat_date in due_dates:
+ claimed = await _claim_leaderboard_snapshot(
+ session_factory,
+ stat_date=stat_date,
+ settings_hash=settings_hash,
+ running_ttl_seconds=_LEADERBOARD_SNAPSHOT_RUNNING_TTL_SECONDS,
+ )
+ if not claimed:
+ continue
+ logger.info("Leaderboard snapshot scheduler: building date=%s hash=%s", stat_date, settings_hash[:10])
+ await _refresh_daily_leaderboard_snapshot(
+ _daily_job_key(stat_date, settings_hash),
+ session_factory=session_factory,
+ settings=settings,
+ stat_date=stat_date,
+ settings_hash=settings_hash,
+ excluded=excluded,
+ include_scheduled=include_scheduled,
+ include_admins=include_admins,
+ include_failed=include_failed,
+ )
+ processed += 1
+ if _LEADERBOARD_SNAPSHOT_SLEEP_SECONDS > 0 and processed < len(due_dates):
+ await asyncio.sleep(_LEADERBOARD_SNAPSHOT_SLEEP_SECONDS)
+ return processed, len(queued_dates)
+
+
+async def _leaderboard_snapshot_scheduler_loop() -> None:
+ """Background task inside the existing gateway process for daily snapshots."""
+ if not _LEADERBOARD_SNAPSHOT_BACKGROUND_ENABLED:
+ logger.info("Leaderboard snapshot scheduler disabled by ADMIN_LEADERBOARD_BACKGROUND_ENABLED")
+ return
+
+ from deerflow.config.runtime_paths import runtime_home
+
+ lock_handle = _acquire_singleton_lock(runtime_home() / ".leaderboard_snapshot_scheduler.lock")
+ if lock_handle is None:
+ logger.info("Leaderboard snapshot scheduler: another gateway worker holds the lock; this worker skips")
+ return
+
+ logger.info(
+ "Leaderboard snapshot scheduler started: prewarm_days=%d batch_size=%d interval=%.1fs",
+ _LEADERBOARD_SNAPSHOT_PREWARM_DAYS,
+ _LEADERBOARD_SNAPSHOT_BATCH_SIZE,
+ _LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS,
+ )
+ try:
+ if _LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS > 0:
+ await asyncio.sleep(_LEADERBOARD_SNAPSHOT_STARTUP_DELAY_SECONDS)
+ while True:
+ try:
+ processed, queued = await _run_leaderboard_snapshot_tick()
+ if processed or queued:
+ logger.info(
+ "Leaderboard snapshot scheduler tick: queued=%d processed=%d",
+ queued,
+ processed,
+ )
+ except Exception:
+ logger.exception("Leaderboard snapshot scheduler tick failed (non-fatal)")
+ await asyncio.sleep(_LEADERBOARD_SNAPSHOT_INTERVAL_SECONDS)
+ finally:
+ _release_singleton_lock(lock_handle)
+
+
+async def _concurrency_sampler_loop(app: FastAPI) -> None:
+ """Periodically record the in-flight run count for the admin pressure chart.
+
+ Runs in the gateway process, singleton-locked so only one worker samples.
+ Each tick records ``len(run_manager.list_active())`` and, roughly hourly,
+ purges samples older than the retention window. Best-effort throughout — a
+ failed tick is logged and the loop continues.
+ """
+ if not _CONCURRENCY_SAMPLE_ENABLED:
+ logger.info("Concurrency sampler disabled by CONCURRENCY_SAMPLE_ENABLED")
+ return
+
+ store = getattr(app.state, "concurrency_sample_store", None)
+ run_mgr = getattr(app.state, "run_manager", None)
+ concurrency_gate = getattr(app.state, "concurrency_gate", None)
+ if store is None or run_mgr is None:
+ logger.info("Concurrency sampler not started (no store / run manager — memory backend?)")
+ return
+
+ from deerflow.config.runtime_paths import runtime_home
+
+ lock_handle = _acquire_singleton_lock(runtime_home() / ".concurrency_sampler.lock")
+ if lock_handle is None:
+ logger.info("Concurrency sampler: another gateway worker holds the lock; this worker skips")
+ return
+
+ interval = max(1.0, _CONCURRENCY_SAMPLE_INTERVAL_SECONDS)
+ purge_every_ticks = max(1, int(3600 / interval)) # ~once per hour
+ logger.info("Concurrency sampler started: interval=%.1fs retention=%.1fh", interval, _CONCURRENCY_SAMPLE_RETENTION_HOURS)
+ ticks = 0
+ try:
+ while True:
+ try:
+ # Runtime toggle (admin can turn sampling off without a restart).
+ # mtime-cached read, so this is cheap every tick.
+ from deerflow.config.system_settings import get_concurrency_monitor_settings
+
+ if get_concurrency_monitor_settings().enabled:
+ active_count = None
+ if concurrency_gate is not None:
+ active_count = await concurrency_gate.count_active()
+ if active_count is None:
+ active = await run_mgr.list_active()
+ active_count = len(active)
+ await store.record(active_count)
+ ticks += 1
+ if ticks % purge_every_ticks == 0:
+ from datetime import UTC, datetime, timedelta
+
+ cutoff = datetime.now(UTC) - timedelta(hours=_CONCURRENCY_SAMPLE_RETENTION_HOURS)
+ await store.purge_older_than(cutoff)
+ except Exception:
+ logger.exception("Concurrency sampler tick failed (non-fatal)")
+ await asyncio.sleep(interval)
+ finally:
+ _release_singleton_lock(lock_handle)
+
+
+def _warn_if_multi_worker() -> None:
+ """?? uvicorn / gunicorn ? worker ???? ERROR ???
+
+ ? agent / canvas / AI ??????? Gateway ????? ``StreamBridge``
+ ?? run_id ? SSE ?????? map?? worker ????? worker A ??
+ ``/runs/stream``???? worker B ? join/resume??? in-process map ??
+ ???? 404 / ???
+
+ ????? startup ? ERROR ????**???? fail-fast** ?? ??
+ ???????????????? sticky session??????????
+ """
+ # uvicorn / gunicorn ??????? worker ?????????????
+ # ?????? sibling ??????????? WEB_CONCURRENCY?gunicorn
+ # / uvicorn ????
+ web_concurrency = os.environ.get("WEB_CONCURRENCY", "").strip()
+ if web_concurrency and web_concurrency.isdigit() and int(web_concurrency) > 1:
+ logger.error(
+ "? ??? WEB_CONCURRENCY=%s?? worker ????AI ?? / ? agent "
+ "??????????????? worker ????????? / "
+ "SSE ?? / resume 400??????????? WEB_CONCURRENCY=1 "
+ "????????? sticky session??? AIWritingPage.tsx ????",
+ web_concurrency,
+ )
+ return
+
+ # ???? /proc ??? uvicorn ?????? Linux ??????
+ if not sys.platform.startswith("linux"):
+ return
+ try:
+ from pathlib import Path
+
+ peers = 0
+ for entry in Path("/proc").iterdir():
+ if not entry.name.isdigit():
+ continue
+ try:
+ cmdline = (entry / "cmdline").read_bytes().replace(b"\x00", b" ").decode("utf-8", "ignore")
+ except OSError:
+ continue
+ if "uvicorn" in cmdline and "app.gateway.app" in cmdline:
+ peers += 1
+ if peers > 1:
+ logger.error(
+ "? ??? %d ??? uvicorn ???? worker ????? AI ?? "
+ "????? / SSE ??????????? worker ??????? "
+ "?? sticky session?",
+ peers,
+ )
+ except Exception:
+ # ????????? ?? ??????
+ pass
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
+ """Application lifespan handler."""
+
+ # ?????? worker ?? ?? ?? AI ?? / ? agent ???????
+ # ??????? worker ???????? ERROR ?????
+ _warn_if_multi_worker()
+
+ # Load config and check necessary environment variables at startup
+ try:
+ app.state.config = get_app_config()
+ apply_logging_level(app.state.config.log_level)
+ logger.info("Configuration loaded successfully")
+ except Exception as e:
+ error_msg = f"Failed to load configuration during gateway startup: {e}"
+ logger.exception(error_msg)
+ raise RuntimeError(error_msg) from e
+ config = get_gateway_config()
+ logger.info(f"Starting API Gateway on {config.host}:{config.port}")
+
+ # Initialize LangGraph runtime components (StreamBridge, RunManager, checkpointer, store)
+ async with langgraph_runtime(app):
+ logger.info("LangGraph runtime initialised")
+
# Run Alembic migrations before anything else touches the schema.
await _run_db_migrations()
- # ?? recall_budget cache?? DB ????,? get_effective_memory_config ?????
- try:
- from deerflow.config.memory_config import load_recall_budget_cache
+ # Local Wiki mirror/index scans are opt-in and run outside request
+ # handling. Per-library database leases keep multi-worker startup safe.
+ app.state.llmwiki_index_scheduler_task = None
+ local_wiki_index = app.state.config.llmwiki.local_wiki_index
+ if (
+ local_wiki_index.enabled
+ and local_wiki_index.sync_enabled
+ and getattr(app.state, "llmwiki_sync_service", None) is not None
+ ):
+ from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop
- _mc_store = getattr(app.state, "memory_config_store", None)
- if _mc_store is not None:
- _recall_budgets = await _mc_store.load_all()
- load_recall_budget_cache(_recall_budgets)
- logger.info("Memory config cache warmed: %d user override(s)", len(_recall_budgets))
- except Exception:
- logger.exception("Memory config cache warm-up failed (non-fatal)")
-
- # Ensure admin user exists (auto-create on first boot)
- # Must run AFTER langgraph_runtime so app.state.store is available for thread migration
- await _ensure_admin_user(app)
-
- # Migrate "ownerless" on-disk skills into the new SkillStore table ?
- # makes the multitenancy switch transparent for existing deployments.
- try:
- await _sync_legacy_skills(app)
- except Exception:
- logger.exception("Skill ownership migration failed (non-fatal)")
-
- # ????? agent(intent / recommender / coordinator)???????
- # ??? _sync_legacy_agents **??**??????????????????
- # ??? upsert ? agents ?,?????? /api/agents ?????,
- # ????????????????.deer-flow/ ? gitignore,??? clone
- # ????????,seeder ????? _roundtable_seed_assets/ ??????
- try:
- from app.gateway.routers._roundtable_seed import (
- ensure_roundtable_functional_agents,
+ app.state.llmwiki_index_scheduler_task = asyncio.create_task(
+ local_wiki_index_scheduler_loop(app)
)
-
- ensure_roundtable_functional_agents()
- except Exception:
- logger.exception(
- "Roundtable functional agent seeding failed at startup (non-fatal)"
- )
-
- # AI ????? agent(researcher / outliner / writer / editor)???????
- # ???:??? _sync_legacy_agents ??,?????????????
- # upsert ? agents ?,admin ?????????????????? agent?
- try:
- from app.gateway.routers._ai_writing_seed import (
- ensure_ai_writing_functional_agents,
- )
-
- ensure_ai_writing_functional_agents()
- except Exception:
- logger.exception(
- "AI-writing functional agent seeding failed at startup (non-fatal)"
- )
-
- # 内置「模板数据转换器」agent(JSON → HTML 模板填充)。同样必须在
- # _sync_legacy_agents **之前**补建目录,后者把它 upsert 进 agents 表,
- # 第一次启动 /api/agents 与定时任务下拉框里就有它。
- try:
- from app.gateway.routers._template_builder_seed import (
- ensure_template_builder_agent,
- )
-
- ensure_template_builder_agent()
- except Exception:
- logger.exception(
- "Template-builder agent seeding failed at startup (non-fatal)"
- )
-
- # 深度研究「收集员」内置 agent(对话式信息收集)。同样必须在
- # _sync_legacy_agents 之前补建目录,由后者 upsert 进 agents 表,
- # 深度研究页的收集对话与智能体管理页即可直接使用。
- try:
- from app.gateway.routers._deep_research_seed import (
- ensure_deep_research_collector_agent,
- )
-
- ensure_deep_research_collector_agent()
- except Exception:
- logger.exception(
- "Deep-research collector agent seeding failed at startup (non-fatal)"
- )
-
- # ?????????????????3Q/6BF/7BF/8BF??agent/chain ?????
- try:
- business_mapping_store = getattr(app.state, "business_mapping_store", None)
- if business_mapping_store is not None:
- inserted = await business_mapping_store.ensure_seed_defaults()
- if inserted:
- logger.info("Seeded %d business mapping rows", inserted)
- except Exception:
- logger.exception("Business mapping seeding failed at startup (non-fatal)")
-
- # ???? .deer-flow/agents/{id}/ ???????? agent ?? agents ??
- # ??????????????? sync(????? + N ? agent ??? DB
- # SELECT/UPDATE),??????? + ?????? TTL ??,????
- # /api/agents ?????
- try:
- from app.gateway.routers.agents import (
- _mark_legacy_sync_fresh,
- _sync_legacy_agents,
- )
- from app.gateway.services import _mark_legacy_sync_fresh as _services_mark_sync_fresh
-
- agent_store = getattr(app.state, "agent_store", None)
- if agent_store is not None:
- await _sync_legacy_agents(agent_store)
- _mark_legacy_sync_fresh()
- # ``services.py`` keeps an independent TTL gate (used by
- # ``start_run`` ? ``_enforce_agent_access``). Without this
- # ack, the first ``/runs/stream`` after startup would
- # re-run the heavy sync even though we just did it.
- _services_mark_sync_fresh()
- logger.info("Legacy folder-backed agents synced at startup")
- # 内置功能型 agent(roundtable-* / ai-writing-*)按用途默认打标签,
- # 供管理员「内置智能体」筛选按用途归类。幂等、只追加、失败不阻塞启动。
- try:
- from app.gateway.routers._builtin_agent_tag_seed import (
- ensure_builtin_agent_purpose_tags,
- )
-
- tag_store = getattr(app.state, "tag_store", None)
- if tag_store is not None:
- await ensure_builtin_agent_purpose_tags(agent_store, tag_store)
- except Exception:
- logger.exception("Built-in agent purpose-tag seeding failed at startup (non-fatal)")
- except Exception:
- logger.exception("Legacy agent sync failed at startup (non-fatal)")
-
- # ?? skill curator ???:??????,?????????????
- # (per-user interval_hours ??????????????)?
- try:
- app.state.curator_scheduler_task = asyncio.create_task(_curator_scheduler_loop())
- except Exception:
- logger.exception("Skill curator scheduler failed to start (non-fatal)")
-
- # ?? AI ????????:? config.yaml ? ai_writing.cleanup ???,
- # ?????? 3 ????,? retention_days(?? 7 ?)?????,??
- # ?? MySQL ??? + SQLite checkpointer ?? thread ???? worker
- # ?????? leader,?????? worker ??????
- try:
- from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
-
- app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop(app))
- except Exception:
- logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
-
- try:
- app.state.leaderboard_snapshot_scheduler_task = asyncio.create_task(_leaderboard_snapshot_scheduler_loop())
- except Exception:
- logger.exception("Leaderboard snapshot scheduler failed to start (non-fatal)")
-
- # Concurrency (system-pressure) sampler — lightweight, singleton-locked,
- # runtime-toggleable via system_settings.concurrency_monitor.enabled.
- try:
- app.state.concurrency_sampler_task = asyncio.create_task(_concurrency_sampler_loop(app))
- except Exception:
- logger.exception("Concurrency sampler failed to start (non-fatal)")
-
- # Start IM channel service if any channels are configured
- try:
- from app.channels.service import start_channel_service
-
- channel_service = await start_channel_service(app.state.config)
- logger.info("Channel service started: %s", channel_service.get_status())
- except Exception:
- logger.exception("No IM channels configured or channel service failed to start")
-
- scheduled_tasks_enabled = os.environ.get("SCHEDULED_TASKS_ENABLED", "true").strip().lower()
- if scheduled_tasks_enabled in {"0", "false", "no", "off"}:
- logger.info("Scheduled task service disabled by SCHEDULED_TASKS_ENABLED=%s", scheduled_tasks_enabled)
- else:
- try:
- scheduled_task_store = getattr(app.state, "scheduled_task_store", None)
- if scheduled_task_store is not None:
- scheduler_service = ScheduledTaskService(app, scheduled_task_store)
- app.state.scheduled_task_service = scheduler_service
- set_scheduled_task_service(scheduler_service)
- await scheduler_service.start()
- logger.info("Scheduled task service started")
- except Exception:
- logger.exception("Scheduled task service failed to start")
-
+
+ # ?? recall_budget cache?? DB ????,? get_effective_memory_config ?????
+ try:
+ from deerflow.config.memory_config import load_recall_budget_cache
+
+ _mc_store = getattr(app.state, "memory_config_store", None)
+ if _mc_store is not None:
+ _recall_budgets = await _mc_store.load_all()
+ load_recall_budget_cache(_recall_budgets)
+ logger.info("Memory config cache warmed: %d user override(s)", len(_recall_budgets))
+ except Exception:
+ logger.exception("Memory config cache warm-up failed (non-fatal)")
+
+ # Ensure admin user exists (auto-create on first boot)
+ # Must run AFTER langgraph_runtime so app.state.store is available for thread migration
+ await _ensure_admin_user(app)
+
+ # Migrate "ownerless" on-disk skills into the new SkillStore table ?
+ # makes the multitenancy switch transparent for existing deployments.
+ try:
+ await _sync_legacy_skills(app)
+ except Exception:
+ logger.exception("Skill ownership migration failed (non-fatal)")
+
+ # ????? agent(intent / recommender / coordinator)???????
+ # ??? _sync_legacy_agents **??**??????????????????
+ # ??? upsert ? agents ?,?????? /api/agents ?????,
+ # ????????????????.deer-flow/ ? gitignore,??? clone
+ # ????????,seeder ????? _roundtable_seed_assets/ ??????
+ try:
+ from app.gateway.routers._roundtable_seed import (
+ ensure_roundtable_functional_agents,
+ )
+
+ ensure_roundtable_functional_agents()
+ except Exception:
+ logger.exception(
+ "Roundtable functional agent seeding failed at startup (non-fatal)"
+ )
+
+ # AI ????? agent(researcher / outliner / writer / editor)???????
+ # ???:??? _sync_legacy_agents ??,?????????????
+ # upsert ? agents ?,admin ?????????????????? agent?
+ try:
+ from app.gateway.routers._ai_writing_seed import (
+ ensure_ai_writing_functional_agents,
+ )
+
+ ensure_ai_writing_functional_agents()
+ except Exception:
+ logger.exception(
+ "AI-writing functional agent seeding failed at startup (non-fatal)"
+ )
+
+ # 内置「模板数据转换器」agent(JSON → HTML 模板填充)。同样必须在
+ # _sync_legacy_agents **之前**补建目录,后者把它 upsert 进 agents 表,
+ # 第一次启动 /api/agents 与定时任务下拉框里就有它。
+ try:
+ from app.gateway.routers._template_builder_seed import (
+ ensure_template_builder_agent,
+ )
+
+ ensure_template_builder_agent()
+ except Exception:
+ logger.exception(
+ "Template-builder agent seeding failed at startup (non-fatal)"
+ )
+
+ # 深度研究「收集员」内置 agent(对话式信息收集)。同样必须在
+ # _sync_legacy_agents 之前补建目录,由后者 upsert 进 agents 表,
+ # 深度研究页的收集对话与智能体管理页即可直接使用。
+ try:
+ from app.gateway.routers._deep_research_seed import (
+ ensure_deep_research_collector_agent,
+ )
+
+ ensure_deep_research_collector_agent()
+ except Exception:
+ logger.exception(
+ "Deep-research collector agent seeding failed at startup (non-fatal)"
+ )
+
+ # ?????????????????3Q/6BF/7BF/8BF??agent/chain ?????
+ try:
+ business_mapping_store = getattr(app.state, "business_mapping_store", None)
+ if business_mapping_store is not None:
+ inserted = await business_mapping_store.ensure_seed_defaults()
+ if inserted:
+ logger.info("Seeded %d business mapping rows", inserted)
+ except Exception:
+ logger.exception("Business mapping seeding failed at startup (non-fatal)")
+
+ # ???? .deer-flow/agents/{id}/ ???????? agent ?? agents ??
+ # ??????????????? sync(????? + N ? agent ??? DB
+ # SELECT/UPDATE),??????? + ?????? TTL ??,????
+ # /api/agents ?????
+ try:
+ from app.gateway.routers.agents import (
+ _mark_legacy_sync_fresh,
+ _sync_legacy_agents,
+ )
+ from app.gateway.services import _mark_legacy_sync_fresh as _services_mark_sync_fresh
+
+ agent_store = getattr(app.state, "agent_store", None)
+ if agent_store is not None:
+ await _sync_legacy_agents(agent_store)
+ _mark_legacy_sync_fresh()
+ # ``services.py`` keeps an independent TTL gate (used by
+ # ``start_run`` ? ``_enforce_agent_access``). Without this
+ # ack, the first ``/runs/stream`` after startup would
+ # re-run the heavy sync even though we just did it.
+ _services_mark_sync_fresh()
+ logger.info("Legacy folder-backed agents synced at startup")
+ # 内置功能型 agent(roundtable-* / ai-writing-*)按用途默认打标签,
+ # 供管理员「内置智能体」筛选按用途归类。幂等、只追加、失败不阻塞启动。
+ try:
+ from app.gateway.routers._builtin_agent_tag_seed import (
+ ensure_builtin_agent_purpose_tags,
+ )
+
+ tag_store = getattr(app.state, "tag_store", None)
+ if tag_store is not None:
+ await ensure_builtin_agent_purpose_tags(agent_store, tag_store)
+ except Exception:
+ logger.exception("Built-in agent purpose-tag seeding failed at startup (non-fatal)")
+ except Exception:
+ logger.exception("Legacy agent sync failed at startup (non-fatal)")
+
+ # ?? skill curator ???:??????,?????????????
+ # (per-user interval_hours ??????????????)?
+ try:
+ app.state.curator_scheduler_task = asyncio.create_task(_curator_scheduler_loop())
+ except Exception:
+ logger.exception("Skill curator scheduler failed to start (non-fatal)")
+
+ # ?? AI ????????:? config.yaml ? ai_writing.cleanup ???,
+ # ?????? 3 ????,? retention_days(?? 7 ?)?????,??
+ # ?? MySQL ??? + SQLite checkpointer ?? thread ???? worker
+ # ?????? leader,?????? worker ??????
+ try:
+ from app.gateway.ai_writing_cleanup import cleanup_scheduler_loop
+
+ app.state.ai_writing_cleanup_task = asyncio.create_task(cleanup_scheduler_loop(app))
+ except Exception:
+ logger.exception("AI writing cleanup scheduler failed to start (non-fatal)")
+
+ try:
+ app.state.leaderboard_snapshot_scheduler_task = asyncio.create_task(_leaderboard_snapshot_scheduler_loop())
+ except Exception:
+ logger.exception("Leaderboard snapshot scheduler failed to start (non-fatal)")
+
+ # Concurrency (system-pressure) sampler — lightweight, singleton-locked,
+ # runtime-toggleable via system_settings.concurrency_monitor.enabled.
+ try:
+ app.state.concurrency_sampler_task = asyncio.create_task(_concurrency_sampler_loop(app))
+ except Exception:
+ logger.exception("Concurrency sampler failed to start (non-fatal)")
+
+ # Start IM channel service if any channels are configured
+ try:
+ from app.channels.service import start_channel_service
+
+ channel_service = await start_channel_service(app.state.config)
+ logger.info("Channel service started: %s", channel_service.get_status())
+ except Exception:
+ logger.exception("No IM channels configured or channel service failed to start")
+
+ scheduled_tasks_enabled = os.environ.get("SCHEDULED_TASKS_ENABLED", "true").strip().lower()
+ if scheduled_tasks_enabled in {"0", "false", "no", "off"}:
+ logger.info("Scheduled task service disabled by SCHEDULED_TASKS_ENABLED=%s", scheduled_tasks_enabled)
+ else:
+ try:
+ scheduled_task_store = getattr(app.state, "scheduled_task_store", None)
+ if scheduled_task_store is not None:
+ scheduler_service = ScheduledTaskService(app, scheduled_task_store)
+ app.state.scheduled_task_service = scheduler_service
+ set_scheduled_task_service(scheduler_service)
+ await scheduler_service.start()
+ logger.info("Scheduled task service started")
+ except Exception:
+ logger.exception("Scheduled task service failed to start")
+
yield
- # ?? curator ???
- curator_task = getattr(app.state, "curator_scheduler_task", None)
- if curator_task is not None:
- curator_task.cancel()
+ llmwiki_index_scheduler_task = getattr(app.state, "llmwiki_index_scheduler_task", None)
+ if llmwiki_index_scheduler_task is not None:
+ llmwiki_index_scheduler_task.cancel()
try:
- await curator_task
+ await llmwiki_index_scheduler_task
except asyncio.CancelledError:
pass
except Exception:
- logger.exception("Curator scheduler shutdown error")
-
- # ?? AI ???????
- cleanup_task = getattr(app.state, "ai_writing_cleanup_task", None)
- if cleanup_task is not None:
- cleanup_task.cancel()
- try:
- await cleanup_task
- except asyncio.CancelledError:
- pass
- except Exception:
- logger.exception("AI writing cleanup scheduler shutdown error")
-
- leaderboard_snapshot_task = getattr(app.state, "leaderboard_snapshot_scheduler_task", None)
- if leaderboard_snapshot_task is not None:
- leaderboard_snapshot_task.cancel()
- try:
- await leaderboard_snapshot_task
- except asyncio.CancelledError:
- pass
- except Exception:
- logger.exception("Leaderboard snapshot scheduler shutdown error")
-
- concurrency_sampler_task = getattr(app.state, "concurrency_sampler_task", None)
- if concurrency_sampler_task is not None:
- concurrency_sampler_task.cancel()
- try:
- await concurrency_sampler_task
- except asyncio.CancelledError:
- pass
- except Exception:
- logger.exception("Concurrency sampler shutdown error")
-
- # ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
- # thread ??? LangGraph Server ? checkpointer ???lifespan ???
- # ``langgraph_runtime()`` ? contextmanager ???????????????
-
- try:
- scheduler_service = getattr(app.state, "scheduled_task_service", None)
- if scheduler_service is not None:
- await scheduler_service.stop()
- except Exception:
- logger.exception("Failed to stop scheduled task service")
- finally:
- set_scheduled_task_service(None)
-
- # Stop channel service on shutdown (bounded to prevent worker hang)
- try:
- from app.channels.service import stop_channel_service
-
- await asyncio.wait_for(
- stop_channel_service(),
- timeout=_SHUTDOWN_HOOK_TIMEOUT_SECONDS,
- )
- except TimeoutError:
- logger.warning(
- "Channel service shutdown exceeded %.1fs; proceeding with worker exit.",
- _SHUTDOWN_HOOK_TIMEOUT_SECONDS,
- )
- except Exception:
- logger.exception("Failed to stop channel service")
-
- logger.info("Shutting down API Gateway")
-
-
-def create_app() -> FastAPI:
- """Create and configure the FastAPI application.
-
- Returns:
- Configured FastAPI application instance.
- """
- config = get_gateway_config()
- docs_kwargs = {"docs_url": "/docs", "redoc_url": "/redoc", "openapi_url": "/openapi.json"} if config.enable_docs else {"docs_url": None, "redoc_url": None, "openapi_url": None}
-
- app = FastAPI(
- title="DeerFlow API Gateway",
- description="""
-## DeerFlow API Gateway
-
-API Gateway for DeerFlow - A LangGraph-based AI agent backend with sandbox execution capabilities.
-
-### Features
-
-- **Models Management**: Query and retrieve available AI models
-- **MCP Configuration**: Manage Model Context Protocol (MCP) server configurations
-- **Memory Management**: Access and manage global memory data for personalized conversations
-- **Skills Management**: Query and manage skills and their enabled status
-- **Artifacts**: Access thread artifacts and generated files
-- **Health Monitoring**: System health check endpoints
-
-### Architecture
-
-LangGraph requests are handled by nginx reverse proxy.
-This gateway provides custom endpoints for models, MCP configuration, skills, and artifacts.
- """,
- version="0.1.0",
- lifespan=lifespan,
- **docs_kwargs,
- openapi_tags=[
- {
- "name": "models",
- "description": "Operations for querying available AI models and their configurations",
- },
- {
- "name": "mcp",
- "description": "Manage Model Context Protocol (MCP) server configurations",
- },
- {
- "name": "memory",
- "description": "Access and manage global memory data for personalized conversations",
- },
- {
- "name": "skills",
- "description": "Manage skills and their configurations",
- },
- {
- "name": "artifacts",
- "description": "Access and download thread artifacts and generated files",
- },
- {
- "name": "uploads",
- "description": "Upload and manage user files for threads",
- },
- {
- "name": "threads",
- "description": "Manage DeerFlow thread-local filesystem data",
- },
- {
- "name": "agents",
- "description": "Create and manage custom agents with per-agent config and prompts",
- },
- {
- "name": "suggestions",
- "description": "Generate follow-up question suggestions for conversations",
- },
- {
- "name": "channels",
- "description": "Manage IM channel integrations (Feishu, Slack, Telegram)",
- },
- {
- "name": "assistants-compat",
- "description": "LangGraph Platform-compatible assistants API (stub)",
- },
- {
- "name": "runs",
- "description": "LangGraph Platform-compatible runs lifecycle (create, stream, cancel)",
- },
- {
- "name": "scheduled-tasks",
- "description": "Create, manage, and execute user scheduled agent tasks",
- },
- {
- "name": "health",
- "description": "Health check and system status endpoints",
- },
- ],
- )
-
- # Auth: reject unauthenticated requests to non-public paths (fail-closed safety net)
- app.add_middleware(AuthMiddleware)
-
- # CSRF: Double Submit Cookie pattern for state-changing requests
- app.add_middleware(CSRFMiddleware)
-
- # CORS
- if is_auth_disabled():
- logger.warning(
- "DEER_FLOW_AUTH_DISABLED is set ? API authentication is bypassed. "
- "Do not use this in production."
- )
-
- def _cors_allow_any_origin() -> bool:
- """Mirror any browser Origin (works with credentials); cannot use literal '*'."""
- v = os.environ.get("GATEWAY_CORS_ALLOW_ALL", "").strip().lower()
- return v in ("1", "true", "yes", "on") or is_auth_disabled()
-
- # Keep standard local frontend origins available even when the startup
- # script supplies an explicit production/development allow-list. Vite can
- # choose 5173 or move to 5174 when that port is already occupied; excluding
- # the former makes every browser preflight fail after a backend restart.
- local_vite_origins = [
- "http://localhost:5173",
- "http://127.0.0.1:5173",
- "http://localhost:5174",
- "http://127.0.0.1:5174",
- "http://localhost:3000",
- "http://127.0.0.1:3000",
- ]
-
- # The legacy TaskCOP Vue application uses its own Vite development port.
- taskcop_vite_origins = [
- "http://localhost:8080",
- "http://127.0.0.1:8080",
- ]
- local_dev_origins = [*local_vite_origins, *taskcop_vite_origins]
-
- cors_origins_env = os.environ.get("GATEWAY_CORS_ORIGINS", "").strip()
- cors_origins: list[str] = [o.strip() for o in cors_origins_env.split(",") if o.strip()] if cors_origins_env else []
- if cors_origins:
- cors_origins = list(dict.fromkeys([*cors_origins, *local_dev_origins]))
-
- # Headers the LangGraph SDK / browser clients need to read from cross-origin
- # responses. Without exposing them, the SDK's `response.headers.get(...)` returns
- # null on cross-origin streaming responses, breaking run-id extraction and reconnect.
- cors_expose_headers = [
- "Content-Disposition",
- "Content-Location",
- "Location",
- "X-Run-Id",
- "X-Thread-Id",
- ]
-
- # Public pages (e.g. http://47.x.x.x:7010) calling this Gateway on localhost
- # trigger Chrome/Edge Private Network Access preflights. Without
- # allow_private_network=True the browser reports a generic CORS failure even
- # when Access-Control-Allow-Origin is correct.
- cors_kwargs = {
- "allow_credentials": True,
- "allow_methods": ["*"],
- "allow_headers": ["*"],
- "expose_headers": cors_expose_headers,
- "allow_private_network": True,
- }
-
- if _cors_allow_any_origin():
- logger.warning(
- "Permissive CORS enabled ? any Origin is echoed back (GATEWAY_CORS_ALLOW_ALL or "
- "DEER_FLOW_AUTH_DISABLED). Do not expose publicly."
- )
- app.add_middleware(
- CORSMiddleware,
- allow_origins=[],
- # fullmatch() on the Origin header; browsers send scheme://host[:port]
- allow_origin_regex=r".+",
- **cors_kwargs,
- )
- elif cors_origins:
- if "*" in cors_origins:
- logger.error(
- "GATEWAY_CORS_ORIGINS contains '*' with allow_credentials=True ? invalid. "
- "Set GATEWAY_CORS_ALLOW_ALL=1 instead, or list explicit origins."
- )
- cors_origins = [o for o in cors_origins if o != "*"]
- if cors_origins:
- app.add_middleware(
- CORSMiddleware,
- allow_origins=cors_origins,
- **cors_kwargs,
- )
- else:
- # Safe-by-default fallback for local frontend development. Without any CORS
- # middleware, browser preflight OPTIONS requests will hit routes directly and
- # fail with 405, which blocks login and all JSON POST APIs.
- dev_origins = local_dev_origins
- logger.warning(
- "No CORS configuration found; enabling local dev origins only: %s. "
- "Set GATEWAY_CORS_ORIGINS or GATEWAY_CORS_ALLOW_ALL for deployment.",
- ", ".join(dev_origins),
- )
- app.add_middleware(
- CORSMiddleware,
- allow_origins=dev_origins,
- **cors_kwargs,
- )
-
- # Include routers
- # Models API is mounted at /api/models
- app.include_router(models.router)
- # Administrator-managed config.yaml model settings and connection tests.
- app.include_router(model_management.router)
-
- # MCP API is mounted at /api/mcp
- app.include_router(mcp.router)
-
+ logger.exception("Local Wiki index scheduler shutdown error")
+ llmwiki_index_jobs = list(getattr(app.state, "llmwiki_index_jobs", set()))
+ for job in llmwiki_index_jobs:
+ job.cancel()
+ if llmwiki_index_jobs:
+ await asyncio.gather(*llmwiki_index_jobs, return_exceptions=True)
+
+ # ?? curator ???
+ curator_task = getattr(app.state, "curator_scheduler_task", None)
+ if curator_task is not None:
+ curator_task.cancel()
+ try:
+ await curator_task
+ except asyncio.CancelledError:
+ pass
+ except Exception:
+ logger.exception("Curator scheduler shutdown error")
+
+ # ?? AI ???????
+ cleanup_task = getattr(app.state, "ai_writing_cleanup_task", None)
+ if cleanup_task is not None:
+ cleanup_task.cancel()
+ try:
+ await cleanup_task
+ except asyncio.CancelledError:
+ pass
+ except Exception:
+ logger.exception("AI writing cleanup scheduler shutdown error")
+
+ leaderboard_snapshot_task = getattr(app.state, "leaderboard_snapshot_scheduler_task", None)
+ if leaderboard_snapshot_task is not None:
+ leaderboard_snapshot_task.cancel()
+ try:
+ await leaderboard_snapshot_task
+ except asyncio.CancelledError:
+ pass
+ except Exception:
+ logger.exception("Leaderboard snapshot scheduler shutdown error")
+
+ concurrency_sampler_task = getattr(app.state, "concurrency_sampler_task", None)
+ if concurrency_sampler_task is not None:
+ concurrency_sampler_task.cancel()
+ try:
+ await concurrency_sampler_task
+ except asyncio.CancelledError:
+ pass
+ except Exception:
+ logger.exception("Concurrency sampler shutdown error")
+
+ # ?? 7 ????AI ?? graph ???? checkpointer / in-flight task?
+ # thread ??? LangGraph Server ? checkpointer ???lifespan ???
+ # ``langgraph_runtime()`` ? contextmanager ???????????????
+
+ try:
+ scheduler_service = getattr(app.state, "scheduled_task_service", None)
+ if scheduler_service is not None:
+ await scheduler_service.stop()
+ except Exception:
+ logger.exception("Failed to stop scheduled task service")
+ finally:
+ set_scheduled_task_service(None)
+
+ # Stop channel service on shutdown (bounded to prevent worker hang)
+ try:
+ from app.channels.service import stop_channel_service
+
+ await asyncio.wait_for(
+ stop_channel_service(),
+ timeout=_SHUTDOWN_HOOK_TIMEOUT_SECONDS,
+ )
+ except TimeoutError:
+ logger.warning(
+ "Channel service shutdown exceeded %.1fs; proceeding with worker exit.",
+ _SHUTDOWN_HOOK_TIMEOUT_SECONDS,
+ )
+ except Exception:
+ logger.exception("Failed to stop channel service")
+
+ logger.info("Shutting down API Gateway")
+
+
+def create_app() -> FastAPI:
+ """Create and configure the FastAPI application.
+
+ Returns:
+ Configured FastAPI application instance.
+ """
+ config = get_gateway_config()
+ docs_kwargs = {"docs_url": "/docs", "redoc_url": "/redoc", "openapi_url": "/openapi.json"} if config.enable_docs else {"docs_url": None, "redoc_url": None, "openapi_url": None}
+
+ app = FastAPI(
+ title="DeerFlow API Gateway",
+ description="""
+## DeerFlow API Gateway
+
+API Gateway for DeerFlow - A LangGraph-based AI agent backend with sandbox execution capabilities.
+
+### Features
+
+- **Models Management**: Query and retrieve available AI models
+- **MCP Configuration**: Manage Model Context Protocol (MCP) server configurations
+- **Memory Management**: Access and manage global memory data for personalized conversations
+- **Skills Management**: Query and manage skills and their enabled status
+- **Artifacts**: Access thread artifacts and generated files
+- **Health Monitoring**: System health check endpoints
+
+### Architecture
+
+LangGraph requests are handled by nginx reverse proxy.
+This gateway provides custom endpoints for models, MCP configuration, skills, and artifacts.
+ """,
+ version="0.1.0",
+ lifespan=lifespan,
+ **docs_kwargs,
+ openapi_tags=[
+ {
+ "name": "models",
+ "description": "Operations for querying available AI models and their configurations",
+ },
+ {
+ "name": "mcp",
+ "description": "Manage Model Context Protocol (MCP) server configurations",
+ },
+ {
+ "name": "memory",
+ "description": "Access and manage global memory data for personalized conversations",
+ },
+ {
+ "name": "skills",
+ "description": "Manage skills and their configurations",
+ },
+ {
+ "name": "artifacts",
+ "description": "Access and download thread artifacts and generated files",
+ },
+ {
+ "name": "uploads",
+ "description": "Upload and manage user files for threads",
+ },
+ {
+ "name": "threads",
+ "description": "Manage DeerFlow thread-local filesystem data",
+ },
+ {
+ "name": "agents",
+ "description": "Create and manage custom agents with per-agent config and prompts",
+ },
+ {
+ "name": "suggestions",
+ "description": "Generate follow-up question suggestions for conversations",
+ },
+ {
+ "name": "channels",
+ "description": "Manage IM channel integrations (Feishu, Slack, Telegram)",
+ },
+ {
+ "name": "assistants-compat",
+ "description": "LangGraph Platform-compatible assistants API (stub)",
+ },
+ {
+ "name": "runs",
+ "description": "LangGraph Platform-compatible runs lifecycle (create, stream, cancel)",
+ },
+ {
+ "name": "scheduled-tasks",
+ "description": "Create, manage, and execute user scheduled agent tasks",
+ },
+ {
+ "name": "health",
+ "description": "Health check and system status endpoints",
+ },
+ ],
+ )
+
+ # Auth: reject unauthenticated requests to non-public paths (fail-closed safety net)
+ app.add_middleware(AuthMiddleware)
+
+ # CSRF: Double Submit Cookie pattern for state-changing requests
+ app.add_middleware(CSRFMiddleware)
+
+ # CORS
+ if is_auth_disabled():
+ logger.warning(
+ "DEER_FLOW_AUTH_DISABLED is set ? API authentication is bypassed. "
+ "Do not use this in production."
+ )
+
+ def _cors_allow_any_origin() -> bool:
+ """Mirror any browser Origin (works with credentials); cannot use literal '*'."""
+ v = os.environ.get("GATEWAY_CORS_ALLOW_ALL", "").strip().lower()
+ return v in ("1", "true", "yes", "on") or is_auth_disabled()
+
+ # Keep standard local frontend origins available even when the startup
+ # script supplies an explicit production/development allow-list. Vite can
+ # choose 5173 or move to 5174 when that port is already occupied; excluding
+ # the former makes every browser preflight fail after a backend restart.
+ local_vite_origins = [
+ "http://localhost:5173",
+ "http://127.0.0.1:5173",
+ "http://localhost:5174",
+ "http://127.0.0.1:5174",
+ "http://localhost:3000",
+ "http://127.0.0.1:3000",
+ ]
+
+ # The legacy TaskCOP Vue application uses its own Vite development port.
+ taskcop_vite_origins = [
+ "http://localhost:8080",
+ "http://127.0.0.1:8080",
+ ]
+ local_dev_origins = [*local_vite_origins, *taskcop_vite_origins]
+
+ cors_origins_env = os.environ.get("GATEWAY_CORS_ORIGINS", "").strip()
+ cors_origins: list[str] = [o.strip() for o in cors_origins_env.split(",") if o.strip()] if cors_origins_env else []
+ if cors_origins:
+ cors_origins = list(dict.fromkeys([*cors_origins, *local_dev_origins]))
+
+ # Headers the LangGraph SDK / browser clients need to read from cross-origin
+ # responses. Without exposing them, the SDK's `response.headers.get(...)` returns
+ # null on cross-origin streaming responses, breaking run-id extraction and reconnect.
+ cors_expose_headers = [
+ "Content-Disposition",
+ "Content-Location",
+ "Location",
+ "X-Run-Id",
+ "X-Thread-Id",
+ ]
+
+ # Public pages (e.g. http://47.x.x.x:7010) calling this Gateway on localhost
+ # trigger Chrome/Edge Private Network Access preflights. Without
+ # allow_private_network=True the browser reports a generic CORS failure even
+ # when Access-Control-Allow-Origin is correct.
+ cors_kwargs = {
+ "allow_credentials": True,
+ "allow_methods": ["*"],
+ "allow_headers": ["*"],
+ "expose_headers": cors_expose_headers,
+ "allow_private_network": True,
+ }
+
+ if _cors_allow_any_origin():
+ logger.warning(
+ "Permissive CORS enabled ? any Origin is echoed back (GATEWAY_CORS_ALLOW_ALL or "
+ "DEER_FLOW_AUTH_DISABLED). Do not expose publicly."
+ )
+ app.add_middleware(
+ CORSMiddleware,
+ allow_origins=[],
+ # fullmatch() on the Origin header; browsers send scheme://host[:port]
+ allow_origin_regex=r".+",
+ **cors_kwargs,
+ )
+ elif cors_origins:
+ if "*" in cors_origins:
+ logger.error(
+ "GATEWAY_CORS_ORIGINS contains '*' with allow_credentials=True ? invalid. "
+ "Set GATEWAY_CORS_ALLOW_ALL=1 instead, or list explicit origins."
+ )
+ cors_origins = [o for o in cors_origins if o != "*"]
+ if cors_origins:
+ app.add_middleware(
+ CORSMiddleware,
+ allow_origins=cors_origins,
+ **cors_kwargs,
+ )
+ else:
+ # Safe-by-default fallback for local frontend development. Without any CORS
+ # middleware, browser preflight OPTIONS requests will hit routes directly and
+ # fail with 405, which blocks login and all JSON POST APIs.
+ dev_origins = local_dev_origins
+ logger.warning(
+ "No CORS configuration found; enabling local dev origins only: %s. "
+ "Set GATEWAY_CORS_ORIGINS or GATEWAY_CORS_ALLOW_ALL for deployment.",
+ ", ".join(dev_origins),
+ )
+ app.add_middleware(
+ CORSMiddleware,
+ allow_origins=dev_origins,
+ **cors_kwargs,
+ )
+
+ # Include routers
+ # Models API is mounted at /api/models
+ app.include_router(models.router)
+ # Administrator-managed config.yaml model settings and connection tests.
+ app.include_router(model_management.router)
+
+ # MCP API is mounted at /api/mcp
+ app.include_router(mcp.router)
+
# DeerFlow-native LLMWiki management/search BFF (conditionally backed by WeKnora).
app.include_router(llmwiki.router)
-
- # Memory API is mounted at /api/memory
- app.include_router(memory.router)
-
- # Skills API is mounted at /api/skills
- app.include_router(skills.router)
-
- # Artifacts API is mounted at /api/threads/{thread_id}/artifacts
- app.include_router(artifacts.router)
-
- # Uploads API is mounted at /api/threads/{thread_id}/uploads
- app.include_router(uploads.router)
-
- # Optional Chrome-extension page capture API is mounted at /api/browser-context
- app.include_router(browser_context.router)
-
- # Thread cleanup API is mounted at /api/threads/{thread_id}
- app.include_router(threads.router)
-
- # Agents API is mounted at /api/agents
- app.include_router(agents.router)
-
- # Recommended Questions API is mounted at /api/recommended-questions
- app.include_router(recommended_questions.router)
-
- app.include_router(user_prompts.router)
-
- # Per-user UI preferences at /api/user-preferences
- app.include_router(user_preferences.router)
-
- # Roundtable-planning drafts + recommendation history at /api/roundtable-drafts
- app.include_router(roundtable_drafts.router)
-
- # Task-scoped roundtable drafts (taskId 深链聊天记录,独立存、不分权) at /api/roundtable-task-drafts
- app.include_router(roundtable_task_drafts.router)
-
- # Task-scoped artifact submission status; stores references only, never artifact content.
- app.include_router(roundtable_artifact_submissions.router)
-
- # Step3 流程图结构化抽取 + 校验 (SSE) at /api/flow-extract
- app.include_router(flow_extract.router)
-
- # Roundtable-planning business chains at /api/roundtable-chains
- app.include_router(roundtable_chains.router)
-
- # 岗位协同独立会话 / 节点状态(不使用原圆桌总控或后台作业)
- # Canonical position-workspace persistence API.
- app.include_router(position_roundtable.router, prefix="/api")
- # Compatibility route for intranet gateways whose allowlist only exposes
- # the established `/api/multi-agent/*` namespace.
- app.include_router(
- position_roundtable.router,
- prefix="/api/multi-agent",
- include_in_schema=False,
- )
-
- # Global business ? agent / chain mapping at /api/business-mapping
- app.include_router(business_mapping.router)
-
- # Position-roundtable work-view role configuration at /api/position-roles.
- app.include_router(position_roles.router)
-
- # Light-app registry (?????) at /api/light-apps
- app.include_router(light_apps.router)
-
- # Sidebar-menu overrides (菜单管理) at /api/menu-overrides
- app.include_router(menu_overrides.router)
-
- # Compact/sidebar-summary menu overrides at /api/compact-menu-overrides
- app.include_router(compact_menu_overrides.router)
-
- # Sensitive-word map (敏感词管理 / 脱敏词) at /api/sensitive-words
- app.include_router(sensitive_words.router)
-
- # Skill URL/IP batch replace (技能地址管理) at /api/skill-addresses
- app.include_router(skill_addresses.router)
-
- # 大屏绘制智能体独立页面会话(按 taskId 存取,不分权)at /api/dashboard-sessions
- app.include_router(dashboard_sessions.router)
-
+ app.include_router(llmwiki_index.router)
+ app.include_router(external_llmwiki.router)
+
+ # Memory API is mounted at /api/memory
+ app.include_router(memory.router)
+
+ # Skills API is mounted at /api/skills
+ app.include_router(skills.router)
+
+ # Artifacts API is mounted at /api/threads/{thread_id}/artifacts
+ app.include_router(artifacts.router)
+
+ # Uploads API is mounted at /api/threads/{thread_id}/uploads
+ app.include_router(uploads.router)
+
+ # Optional Chrome-extension page capture API is mounted at /api/browser-context
+ app.include_router(browser_context.router)
+
+ # Thread cleanup API is mounted at /api/threads/{thread_id}
+ app.include_router(threads.router)
+
+ # Agents API is mounted at /api/agents
+ app.include_router(agents.router)
+
+ # Recommended Questions API is mounted at /api/recommended-questions
+ app.include_router(recommended_questions.router)
+
+ app.include_router(user_prompts.router)
+
+ # Per-user UI preferences at /api/user-preferences
+ app.include_router(user_preferences.router)
+
+ # Roundtable-planning drafts + recommendation history at /api/roundtable-drafts
+ app.include_router(roundtable_drafts.router)
+
+ # Task-scoped roundtable drafts (taskId 深链聊天记录,独立存、不分权) at /api/roundtable-task-drafts
+ app.include_router(roundtable_task_drafts.router)
+
+ # Task-scoped artifact submission status; stores references only, never artifact content.
+ app.include_router(roundtable_artifact_submissions.router)
+
+ # Step3 流程图结构化抽取 + 校验 (SSE) at /api/flow-extract
+ app.include_router(flow_extract.router)
+
+ # Roundtable-planning business chains at /api/roundtable-chains
+ app.include_router(roundtable_chains.router)
+
+ # 岗位协同独立会话 / 节点状态(不使用原圆桌总控或后台作业)
+ # Canonical position-workspace persistence API.
+ app.include_router(position_roundtable.router, prefix="/api")
+ # Compatibility route for intranet gateways whose allowlist only exposes
+ # the established `/api/multi-agent/*` namespace.
+ app.include_router(
+ position_roundtable.router,
+ prefix="/api/multi-agent",
+ include_in_schema=False,
+ )
+
+ # Global business ? agent / chain mapping at /api/business-mapping
+ app.include_router(business_mapping.router)
+
+ # Position-roundtable work-view role configuration at /api/position-roles.
+ app.include_router(position_roles.router)
+
+ # Light-app registry (?????) at /api/light-apps
+ app.include_router(light_apps.router)
+
+ # Sidebar-menu overrides (菜单管理) at /api/menu-overrides
+ app.include_router(menu_overrides.router)
+
+ # Compact/sidebar-summary menu overrides at /api/compact-menu-overrides
+ app.include_router(compact_menu_overrides.router)
+
+ # Sensitive-word map (敏感词管理 / 脱敏词) at /api/sensitive-words
+ app.include_router(sensitive_words.router)
+
+ # Skill URL/IP batch replace (技能地址管理) at /api/skill-addresses
+ app.include_router(skill_addresses.router)
+
+ # 大屏绘制智能体独立页面会话(按 taskId 存取,不分权)at /api/dashboard-sessions
+ app.include_router(dashboard_sessions.router)
+
# Task-workspace configurable jump buttons (按钮管理) at /api/task-buttons
app.include_router(task_buttons.router)
- # Deep-research report structures (工作流配置 / 报告结构) at /api/report-structures
- app.include_router(report_structures.router)
-
- # Temporary compatibility bridge for the TaskCOP situation-overview frontend.
- app.include_router(taskcop_tasks.router)
-
- # Task-scoped situation-report details used by dashboard drawing / task gates.
- app.include_router(task_reports.router)
-
- # Standalone parallel multi-agent panel: fan-out streaming + saved tasks
- app.include_router(parallel_agents.router)
- app.include_router(parallel_agent_tasks.router)
-
- # Roundtable-planning background jobs at /api/roundtable-jobs
- app.include_router(roundtable_jobs.router)
- app.include_router(roundtable_diagnostics.router)
-
- # Deep Research (深度研究) API at /api/deep-research
- app.include_router(deep_research.router)
-
- # The independent enterprise-research workbench is temporarily disabled.
- # Do not mount /api/enterprise-research while its MySQL schema is inactive.
-
- # Suggestions API is mounted at /api/threads/{thread_id}/suggestions
- app.include_router(suggestions.router)
-
- # Writing rewrite API is mounted at /api/writing
- app.include_router(writing.router)
-
- # AI Writing multi-agent API is mounted at /api/ai-writing
- app.include_router(ai_writing.router)
-
- # Scheduled task API is mounted at /api/scheduled-tasks
- app.include_router(scheduled_tasks.router)
-
- # Public, unauthenticated read-only scheduled task API
- # (/api/public/scheduled-tasks/*). Lets external systems query tasks and
- # surface results without a DeerFlow login.
- app.include_router(public_scheduled_tasks.router)
-
- # Public read-only user leaderboard publish page (/api/public/leaderboard)
- app.include_router(public_leaderboard.router)
-
- # Public embed Q&A status polling (/api/public/embed/*)
- app.include_router(public_embed.router)
- app.include_router(public_config.router)
- app.include_router(html_page_favorites.router)
-
- # Product knowledge base API is mounted at /api/knowledge
- app.include_router(knowledge.router)
-
- # External knowledge ingest (publish md artifacts) at /api/knowledge-ingest
- app.include_router(knowledge_ingest.router)
-
- # System settings API is mounted at /api/system-settings
- app.include_router(system_settings.router)
-
- # Admin LLM metrics audit + export
- app.include_router(llm_metrics.router)
-
- # Admin window into other users' threads / messages / memory
- app.include_router(admin_users.router)
- app.include_router(admin_users.self_router)
-
- app.include_router(admin_active_runs.router)
-
- # Admin tool / skill call metrics audit + export
- app.include_router(tool_metrics.router)
-
- # Channels API is mounted at /api/channels
- app.include_router(channels.router)
-
- # Admin checkpoint migration API: SQLite legacy checkpoints -> Postgres.
- app.include_router(checkpoint_migration.router)
-
- # Assistants compatibility API (LangGraph Platform stub)
- app.include_router(assistants_compat.router)
-
- # Auth API is mounted at /api/v1/auth
- app.include_router(auth.router)
-
- # Feedback API is mounted at /api/threads/{thread_id}/runs/{run_id}/feedback
- app.include_router(feedback.router)
-
- # Notifications API is mounted at /api/notifications
- app.include_router(notifications.router)
-
- # Tags API is mounted at /api/tags
- app.include_router(tags.router)
-
- # 岗位 (position) management API is mounted at /api/positions
- app.include_router(positions.router)
-
- # Thread Runs API (LangGraph Platform-compatible runs lifecycle)
- app.include_router(thread_runs.router)
-
- app.include_router(thread_shares.router)
- # Public, unauthenticated read-only share snapshots (/api/public/shares/*).
- app.include_router(thread_shares.public_router)
-
- # ?????????import ???? / view ????????
- app.include_router(roundtable_draft_shares.router)
- # Public, unauthenticated read-only roundtable result (/api/public/roundtable-shares/*).
- app.include_router(roundtable_draft_shares.public_router)
-
- # Stateless Runs API (stream/wait without a pre-existing thread)
- app.include_router(runs.router)
-
- # 开放问答接口 /api/open/chat —— 无需登录,指定模型 + 智能体做问答,可关闭思考模式
- app.include_router(open_chat.router)
-
- # 开放接口 /api/open/3qfx/ask —— 无需登录,按 taskId 直接发起 3qfx 第一次问答(后台运行,落成真实会话)
- app.include_router(open_3qfx.router)
-
- # --- ?????????? /page/workspace/roundtable/planning?---
- # ??? app/gateway/routers/{multi_agent,intent,recommend}.py?
- # ??? frontend-web/docs/multi-agent-backend-dev.md
- app.include_router(multi_agent.router) # Step 2?init(SSE) + run/stream(leader|special)
- app.include_router(intent.router) # Step 1?/api/intent/init + /stream
- app.include_router(recommend.router) # Step 1?2?/api/recommend/stream?? init?
- # Step 3 ????/?????? artifacts.router?/api/threads/{id}/artifacts?
-
- # Health probes are always unauthenticated (see
- # auth_middleware.PUBLIC_HEALTH_PATHS). Support the bare Gateway path,
- # API base URL, and LangGraph proxy base URL for deployment monitors.
- @app.api_route("/health", methods=["GET", "HEAD"], tags=["health"])
- @app.api_route("/api/health", methods=["GET", "HEAD"], tags=["health"])
- @app.api_route("/api/langgraph/health", methods=["GET", "HEAD"], tags=["health"])
- async def health_check() -> dict:
- """Health check endpoint.
-
- Returns:
- Service health status information.
- """
- return {"status": "healthy", "service": "deer-flow-gateway"}
-
- # WeKnora's embedded frontend uses root-relative routes such as /platform,
- # /assets, /locales and /api/v1/*. Register that proxy last so it stays a
- # true fallback and cannot intercept DeerFlow APIs.
- # Register the public-prefix routes before the unprefixed static fallback.
- # Otherwise proxy_router's /{proxied_path:path} route consumes every
- # /deerflow/* request before the prefixed platform/assets routes can match.
- app.include_router(llmwiki.proxy_router, prefix="/deerflow")
- app.include_router(llmwiki.proxy_router)
-
- return app
-
-
-# Create app instance for uvicorn
-app = create_app()
+ # Admin exact-match question/answer short-circuit at /api/fixed-questions
+ app.include_router(fixed_questions.router)
+
+ # Deep-research report structures (工作流配置 / 报告结构) at /api/report-structures
+ app.include_router(report_structures.router)
+
+ # Temporary compatibility bridge for the TaskCOP situation-overview frontend.
+ app.include_router(taskcop_tasks.router)
+
+ # Task-scoped situation-report details used by dashboard drawing / task gates.
+ app.include_router(task_reports.router)
+
+ # Standalone parallel multi-agent panel: fan-out streaming + saved tasks
+ app.include_router(parallel_agents.router)
+ app.include_router(parallel_agent_tasks.router)
+
+ # Roundtable-planning background jobs at /api/roundtable-jobs
+ app.include_router(roundtable_jobs.router)
+ app.include_router(roundtable_diagnostics.router)
+
+ # Deep Research (深度研究) API at /api/deep-research
+ app.include_router(deep_research.router)
+
+ # The independent enterprise-research workbench is temporarily disabled.
+ # Do not mount /api/enterprise-research while its MySQL schema is inactive.
+
+ # Suggestions API is mounted at /api/threads/{thread_id}/suggestions
+ app.include_router(suggestions.router)
+
+ # Writing rewrite API is mounted at /api/writing
+ app.include_router(writing.router)
+
+ # AI Writing multi-agent API is mounted at /api/ai-writing
+ app.include_router(ai_writing.router)
+
+ # Scheduled task API is mounted at /api/scheduled-tasks
+ app.include_router(scheduled_tasks.router)
+
+ # Public, unauthenticated read-only scheduled task API
+ # (/api/public/scheduled-tasks/*). Lets external systems query tasks and
+ # surface results without a DeerFlow login.
+ app.include_router(public_scheduled_tasks.router)
+
+ # Public read-only user leaderboard publish page (/api/public/leaderboard)
+ app.include_router(public_leaderboard.router)
+
+ # Public embed Q&A status polling (/api/public/embed/*)
+ app.include_router(public_embed.router)
+ app.include_router(public_config.router)
+ app.include_router(html_page_favorites.router)
+
+ # Product knowledge base API is mounted at /api/knowledge
+ app.include_router(knowledge.router)
+
+ # External knowledge ingest (publish md artifacts) at /api/knowledge-ingest
+ app.include_router(knowledge_ingest.router)
+
+ # System settings API is mounted at /api/system-settings
+ app.include_router(system_settings.router)
+
+ # Admin LLM metrics audit + export
+ app.include_router(llm_metrics.router)
+
+ # Admin window into other users' threads / messages / memory
+ app.include_router(admin_users.router)
+ app.include_router(admin_users.self_router)
+
+ app.include_router(admin_active_runs.router)
+
+ # Admin tool / skill call metrics audit + export
+ app.include_router(tool_metrics.router)
+
+ # Channels API is mounted at /api/channels
+ app.include_router(channels.router)
+
+ # Admin checkpoint migration API: SQLite legacy checkpoints -> Postgres.
+ app.include_router(checkpoint_migration.router)
+
+ # Assistants compatibility API (LangGraph Platform stub)
+ app.include_router(assistants_compat.router)
+
+ # Auth API is mounted at /api/v1/auth
+ app.include_router(auth.router)
+
+ # Feedback API is mounted at /api/threads/{thread_id}/runs/{run_id}/feedback
+ app.include_router(feedback.router)
+
+ # Notifications API is mounted at /api/notifications
+ app.include_router(notifications.router)
+
+ # Tags API is mounted at /api/tags
+ app.include_router(tags.router)
+
+ # 岗位 (position) management API is mounted at /api/positions
+ app.include_router(positions.router)
+
+ # Thread Runs API (LangGraph Platform-compatible runs lifecycle)
+ app.include_router(thread_runs.router)
+
+ app.include_router(thread_shares.router)
+ # Public, unauthenticated read-only share snapshots (/api/public/shares/*).
+ app.include_router(thread_shares.public_router)
+
+ # ?????????import ???? / view ????????
+ app.include_router(roundtable_draft_shares.router)
+ # Public, unauthenticated read-only roundtable result (/api/public/roundtable-shares/*).
+ app.include_router(roundtable_draft_shares.public_router)
+
+ # Stateless Runs API (stream/wait without a pre-existing thread)
+ app.include_router(runs.router)
+
+ # 开放问答接口 /api/open/chat —— 无需登录,指定模型 + 智能体做问答,可关闭思考模式
+ app.include_router(open_chat.router)
+
+ # 开放接口 /api/open/3qfx/ask —— 无需登录,按 taskId 直接发起 3qfx 第一次问答(后台运行,落成真实会话)
+ app.include_router(open_3qfx.router)
+
+ # --- ?????????? /page/workspace/roundtable/planning?---
+ # ??? app/gateway/routers/{multi_agent,intent,recommend}.py?
+ # ??? frontend-web/docs/multi-agent-backend-dev.md
+ app.include_router(multi_agent.router) # Step 2?init(SSE) + run/stream(leader|special)
+ app.include_router(intent.router) # Step 1?/api/intent/init + /stream
+ app.include_router(recommend.router) # Step 1?2?/api/recommend/stream?? init?
+ # Step 3 ????/?????? artifacts.router?/api/threads/{id}/artifacts?
+
+ # Health probes are always unauthenticated (see
+ # auth_middleware.PUBLIC_HEALTH_PATHS). Support the bare Gateway path,
+ # API base URL, and LangGraph proxy base URL for deployment monitors.
+ @app.api_route("/health", methods=["GET", "HEAD"], tags=["health"])
+ @app.api_route("/api/health", methods=["GET", "HEAD"], tags=["health"])
+ @app.api_route("/api/langgraph/health", methods=["GET", "HEAD"], tags=["health"])
+ async def health_check() -> dict:
+ """Health check endpoint.
+
+ Returns:
+ Service health status information.
+ """
+ return {"status": "healthy", "service": "deer-flow-gateway"}
+
+ # WeKnora's embedded frontend uses root-relative routes such as /platform,
+ # /assets, /locales and /api/v1/*. Register that proxy last so it stays a
+ # true fallback and cannot intercept DeerFlow APIs.
+ # Register the public-prefix routes before the unprefixed static fallback.
+ # Otherwise proxy_router's /{proxied_path:path} route consumes every
+ # /deerflow/* request before the prefixed platform/assets routes can match.
+ app.include_router(llmwiki.proxy_router, prefix="/deerflow")
+ app.include_router(llmwiki.proxy_router)
+
+ return app
+
+
+# Create app instance for uvicorn
+app = create_app()
diff --git a/offline-backend-20260512/backend/app/gateway/auth/token_source.py b/offline-backend-20260512/backend/app/gateway/auth/token_source.py
new file mode 100644
index 00000000..e05d8f1e
--- /dev/null
+++ b/offline-backend-20260512/backend/app/gateway/auth/token_source.py
@@ -0,0 +1,30 @@
+"""Resolve the credential that represents the calling browser context.
+
+Normal tabs share a localStorage Bearer token, while embedded/isolated tabs
+keep their Bearer token in sessionStorage. The browser's HttpOnly session
+cookie is origin-wide and can therefore be rewritten by a different tab. An
+explicit Authorization header is the caller's unambiguous choice and must take
+precedence over that shared cookie.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from typing import Protocol
+
+
+class _TokenRequest(Protocol):
+ headers: Mapping[str, str]
+ cookies: Mapping[str, str]
+
+
+def get_explicit_or_session_token(request: _TokenRequest) -> str | None:
+ """Return Bearer header token first, then the shared session cookie."""
+ authorization = str(request.headers.get("authorization", "") or "").strip()
+ if authorization:
+ scheme, _, token = authorization.partition(" ")
+ if scheme.lower() == "bearer" and token.strip():
+ return token.strip()
+
+ cookie_token = str(request.cookies.get("access_token", "") or "").strip()
+ return cookie_token or None
diff --git a/offline-backend-20260512/backend/app/gateway/auth_middleware.py b/offline-backend-20260512/backend/app/gateway/auth_middleware.py
index 43ff815e..604824de 100644
--- a/offline-backend-20260512/backend/app/gateway/auth_middleware.py
+++ b/offline-backend-20260512/backend/app/gateway/auth_middleware.py
@@ -9,6 +9,8 @@ owner filtering works automatically via the sentinel pattern.
Fine-grained permission checks remain in authz.py decorators.
"""
+import hashlib
+import hmac
import re
from collections.abc import Callable
@@ -81,6 +83,30 @@ def _is_public(path: str) -> bool:
return any(path.startswith(prefix) for prefix in _PUBLIC_PATH_PREFIXES)
+def _is_external_llmwiki(path: str) -> bool:
+ return path.startswith("/api/external/llmwiki/")
+
+
+def _authorize_external_llmwiki(request: Request) -> JSONResponse | None:
+ """Authenticate the isolated service-to-service namespace before routing."""
+
+ from deerflow.config import get_app_config
+
+ config = get_app_config().llmwiki.local_wiki_index.external_api
+ supplied = request.headers.get("X-API-Key", "")
+ valid = bool(config.enabled and supplied) and any(
+ hmac.compare_digest(supplied, candidate)
+ for candidate in (config.api_key, config.previous_api_key)
+ if candidate
+ )
+ if not valid:
+ return JSONResponse(status_code=401, content={"detail": "Invalid API key"})
+ request.state.external_api_key_fingerprint = hashlib.sha256(
+ supplied.encode("utf-8")
+ ).hexdigest()[:8]
+ return None
+
+
def _is_weknora_embed_proxy_request(request: Request) -> bool:
"""Let the WeKnora embed proxy verify its own signed iframe cookie.
@@ -137,7 +163,11 @@ class AuthMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next: Callable) -> Response:
# Match the application-relative path so a reverse-proxy prefix
# (e.g. /xxx/api/public/...) does not defeat the public whitelist.
- if _is_public(app_relative_path(request)):
+ relative_path = app_relative_path(request)
+ if _is_external_llmwiki(relative_path):
+ rejected = _authorize_external_llmwiki(request)
+ return rejected if rejected is not None else await call_next(request)
+ if _is_public(relative_path):
return await call_next(request)
if _is_weknora_embed_proxy_request(request):
return await call_next(request)
diff --git a/offline-backend-20260512/backend/app/gateway/deps.py b/offline-backend-20260512/backend/app/gateway/deps.py
index 5a10142f..e76c54ea 100644
--- a/offline-backend-20260512/backend/app/gateway/deps.py
+++ b/offline-backend-20260512/backend/app/gateway/deps.py
@@ -1,835 +1,898 @@
-"""Centralized accessors for singleton objects stored on ``app.state``.
-
-**Getters** (used by routers): raise 503 when a required dependency is
-missing, except ``get_store`` which returns ``None``.
-
-Initialization is handled directly in ``app.py`` via :class:`AsyncExitStack`.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-from collections.abc import AsyncGenerator, Callable
-from contextlib import AsyncExitStack, asynccontextmanager
-from typing import TYPE_CHECKING, TypeVar, cast
-
-from fastapi import FastAPI, HTTPException, Request
-from langgraph.types import Checkpointer
-
-from deerflow.config.app_config import AppConfig
-from deerflow.persistence.feedback import FeedbackRepository
-from deerflow.runtime import RunContext, RunManager, StreamBridge
-from deerflow.runtime.events.store.base import RunEventStore
-from deerflow.runtime.runs.store.base import RunStore
-
-logger = logging.getLogger(__name__)
-
-if TYPE_CHECKING:
- from app.gateway.auth.local_provider import LocalAuthProvider
- from app.gateway.auth.repositories.sqlite import SQLiteUserRepository
- from deerflow.persistence.agents import AgentStore
- from deerflow.persistence.notifications import NotificationStore
- from deerflow.persistence.scheduled_tasks import ScheduledTaskStore
- from deerflow.persistence.skills import SkillStore
- from deerflow.persistence.tags import TagStore
- from deerflow.persistence.thread_meta.base import ThreadMetaStore
-
-
-T = TypeVar("T")
-
-
-def resolve_roundtable_gateway_mode(raw: str | None) -> str:
- """解析 ``ROUNDTABLE_JOB_GATEWAY``:**默认 ``inprocess``(真模型后台挂起)**。
-
- 只有显式写 ``simulated`` 才用模拟网关(前端联调用:真实进度、不调真模型);
- 缺省 / 空串 / 其它值一律落到 ``inprocess`` —— 部署时无需在 .env 里配置任何值,
- 后台挂起开箱即用(Phase 3b 已在 Windows 本地与内网 Docker 验证)。
- """
- mode = (raw or "").strip().lower()
- return "simulated" if mode == "simulated" else "inprocess"
-
-
-def get_config(request: Request) -> AppConfig:
- """Return the app-scoped ``AppConfig`` stored on ``app.state``."""
- config = getattr(request.app.state, "config", None)
- if config is None:
- raise HTTPException(status_code=503, detail="Configuration not available")
- return config
-
-
-@asynccontextmanager
-async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
- """Bootstrap and tear down all LangGraph runtime singletons.
-
- Usage in ``app.py``::
-
- async with langgraph_runtime(app):
- yield
- """
- from deerflow.persistence.engine import close_engine, get_session_factory, init_engine_from_config
- from deerflow.runtime import make_store, make_stream_bridge
- from deerflow.runtime.checkpointer.async_provider import make_checkpointer
- from deerflow.runtime.events.store import make_run_event_store
-
- async with AsyncExitStack() as stack:
- config = getattr(app.state, "config", None)
- if config is None:
- raise RuntimeError("langgraph_runtime() requires app.state.config to be initialized")
-
- app.state.stream_bridge = await stack.enter_async_context(make_stream_bridge(config))
-
- # Initialize persistence engine BEFORE checkpointer so that
- # auto-create-database logic runs first (postgres backend).
- await init_engine_from_config(config.database)
-
- app.state.checkpointer = await stack.enter_async_context(make_checkpointer(config))
- app.state.store = await stack.enter_async_context(make_store(config))
-
- # Initialize repositories — one get_session_factory() call for all.
- sf = get_session_factory()
- if sf is not None:
- from deerflow.persistence.feedback import FeedbackRepository
- from deerflow.persistence.run import RunRepository
-
- app.state.run_store = RunRepository(sf)
- app.state.feedback_repo = FeedbackRepository(sf)
+"""Centralized accessors for singleton objects stored on ``app.state``.
+
+**Getters** (used by routers): raise 503 when a required dependency is
+missing, except ``get_store`` which returns ``None``.
+
+Initialization is handled directly in ``app.py`` via :class:`AsyncExitStack`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncGenerator, Callable
+from contextlib import AsyncExitStack, asynccontextmanager
+from typing import TYPE_CHECKING, TypeVar, cast
+
+from fastapi import FastAPI, HTTPException, Request
+from langgraph.types import Checkpointer
+
+from deerflow.config.app_config import AppConfig
+from deerflow.persistence.feedback import FeedbackRepository
+from deerflow.runtime import RunContext, RunManager, StreamBridge
+from deerflow.runtime.events.store.base import RunEventStore
+from deerflow.runtime.runs.store.base import RunStore
+
+logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+ from app.gateway.auth.local_provider import LocalAuthProvider
+ from app.gateway.auth.repositories.sqlite import SQLiteUserRepository
+ from deerflow.persistence.agents import AgentStore
+ from deerflow.persistence.notifications import NotificationStore
+ from deerflow.persistence.scheduled_tasks import ScheduledTaskStore
+ from deerflow.persistence.skills import SkillStore
+ from deerflow.persistence.tags import TagStore
+ from deerflow.persistence.thread_meta.base import ThreadMetaStore
+
+
+T = TypeVar("T")
+
+
+def resolve_roundtable_gateway_mode(raw: str | None) -> str:
+ """解析 ``ROUNDTABLE_JOB_GATEWAY``:**默认 ``inprocess``(真模型后台挂起)**。
+
+ 只有显式写 ``simulated`` 才用模拟网关(前端联调用:真实进度、不调真模型);
+ 缺省 / 空串 / 其它值一律落到 ``inprocess`` —— 部署时无需在 .env 里配置任何值,
+ 后台挂起开箱即用(Phase 3b 已在 Windows 本地与内网 Docker 验证)。
+ """
+ mode = (raw or "").strip().lower()
+ return "simulated" if mode == "simulated" else "inprocess"
+
+
+def get_config(request: Request) -> AppConfig:
+ """Return the app-scoped ``AppConfig`` stored on ``app.state``."""
+ config = getattr(request.app.state, "config", None)
+ if config is None:
+ raise HTTPException(status_code=503, detail="Configuration not available")
+ return config
+
+
+@asynccontextmanager
+async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
+ """Bootstrap and tear down all LangGraph runtime singletons.
+
+ Usage in ``app.py``::
+
+ async with langgraph_runtime(app):
+ yield
+ """
+ from deerflow.persistence.engine import close_engine, get_session_factory, init_engine_from_config
+ from deerflow.runtime import make_store, make_stream_bridge
+ from deerflow.runtime.checkpointer.async_provider import make_checkpointer
+ from deerflow.runtime.events.store import make_run_event_store
+
+ async with AsyncExitStack() as stack:
+ config = getattr(app.state, "config", None)
+ if config is None:
+ raise RuntimeError("langgraph_runtime() requires app.state.config to be initialized")
+
+ app.state.stream_bridge = await stack.enter_async_context(make_stream_bridge(config))
+
+ # Initialize persistence engine BEFORE checkpointer so that
+ # auto-create-database logic runs first (postgres backend).
+ await init_engine_from_config(config.database)
+
+ app.state.checkpointer = await stack.enter_async_context(make_checkpointer(config))
+ app.state.store = await stack.enter_async_context(make_store(config))
+
+ # Initialize repositories — one get_session_factory() call for all.
+ sf = get_session_factory()
+ if sf is not None:
+ from deerflow.persistence.feedback import FeedbackRepository
+ from deerflow.persistence.run import RunRepository
+
+ app.state.run_store = RunRepository(sf)
+ app.state.feedback_repo = FeedbackRepository(sf)
else:
from deerflow.runtime.runs.store.memory import MemoryRunStore
app.state.run_store = MemoryRunStore()
app.state.feedback_repo = None
- from deerflow.persistence.thread_meta import make_thread_store
-
- app.state.thread_store = make_thread_store(sf, app.state.store)
-
- from deerflow.persistence.scheduled_tasks import make_scheduled_task_store
-
- app.state.scheduled_task_store = make_scheduled_task_store(sf)
-
- from deerflow.persistence.html_pages import make_html_page_store
-
- app.state.html_page_store = make_html_page_store(sf)
-
- from deerflow.persistence.llm_metrics import make_llm_metrics_store
-
- app.state.llm_metrics_store = make_llm_metrics_store(sf)
-
- from deerflow.persistence.concurrency import make_concurrency_sample_store
-
- app.state.concurrency_sample_store = make_concurrency_sample_store(sf)
-
- from deerflow.persistence.tool_metrics import make_tool_metrics_store
-
- app.state.tool_metrics_store = make_tool_metrics_store(sf)
-
- from deerflow.persistence.agents import make_agent_store
-
- app.state.agent_store = make_agent_store(sf)
-
- from deerflow.persistence.skills import make_skill_store
-
- app.state.skill_store = make_skill_store(sf)
-
- from deerflow.persistence.notifications import (
- make_notification_store,
- set_notification_store,
- )
-
- app.state.notification_store = make_notification_store(sf)
- # Make the sender helper (harness layer) able to write without going
- # through FastAPI request context — used by cascade.py.
- set_notification_store(app.state.notification_store)
- from deerflow.persistence.recommended_questions import make_recommended_question_store
-
- app.state.recommended_question_store = make_recommended_question_store(sf)
-
- from deerflow.persistence.tags import make_tag_store
-
- app.state.tag_store = make_tag_store(sf)
-
- from deerflow.persistence.positions import make_position_store
-
- app.state.position_store = make_position_store(sf)
-
- from deerflow.services.position_sync import PositionSyncService
-
- app.state.position_sync = PositionSyncService(sf, app.state.position_store, app.state.tag_store)
-
- from deerflow.persistence.thread_shares import make_thread_share_store
-
- app.state.thread_share_store = make_thread_share_store(sf)
-
- from deerflow.persistence.embed_sessions import make_embed_session_store
-
- app.state.embed_session_store = make_embed_session_store(sf)
-
- from deerflow.persistence.ai_writing_sessions import (
- make_ai_writing_session_store,
- )
- from deerflow.persistence.ai_writing_sessions import (
- register_default_store as _register_ai_writing_default_store,
- )
-
- app.state.ai_writing_session_store = make_ai_writing_session_store(sf)
- # 让 LangGraph Server 路径下的 AI 写作节点能跨 app/harness 边界拿到 repo。
- # 老 /start /stream /resume 路由仍走 request.app.state,互不影响。
- _register_ai_writing_default_store(app.state.ai_writing_session_store)
-
- # AI 写作「后台挂起」执行器:把写作图作为后台任务驱动到定稿(离页/刷新/切号不打断)。
- # 用 app.state.checkpointer(与运行时 worker 同一份 saver)续上线程 checkpoint。
- from app.gateway.ai_writing_job_executor import AIWritingJobExecutor
-
- app.state.ai_writing_job_executor = AIWritingJobExecutor(app)
- # 进程重启续跑:把仍标记为 background 的会话重新拉起驱动(best-effort,不阻塞启动)。
- if app.state.ai_writing_session_store is not None:
- from app.gateway.ai_writing_job_executor import STATUS_BACKGROUND
-
- async def _reconcile_ai_writing_background() -> None:
- try:
- sessions = await app.state.ai_writing_session_store.list_by_status(STATUS_BACKGROUND)
- app.state.ai_writing_job_executor.reconcile(sessions=sessions)
- except Exception: # noqa: BLE001
- logger.warning("[ai-writing-bg] startup reconcile failed", exc_info=True)
-
- asyncio.create_task(_reconcile_ai_writing_background())
-
- from deerflow.persistence.article_types import make_article_type_store
-
- app.state.article_type_store = make_article_type_store(sf)
-
- from deerflow.persistence.research_templates import make_research_template_store
-
- app.state.research_template_store = make_research_template_store(sf)
-
- from deerflow.persistence.memory_config import make_user_memory_config_store
-
- app.state.memory_config_store = make_user_memory_config_store(sf)
-
- from deerflow.knowledge import make_knowledge_service
-
- app.state.knowledge_service = make_knowledge_service(sf, config.knowledge)
-
- # Seed the built-in extraction prompts as editable templates so operators
- # can immediately view/edit them (idempotent; never overwrites edits).
- if app.state.knowledge_service is not None:
- try:
- await app.state.knowledge_service.seed_default_templates()
- except Exception:
- logger.exception("Failed to seed knowledge extraction templates")
-
- # Background auto-sedimentation queue (phase 3) — drains finished
- # conversations off the chat path. Worker only does work when
- # ``knowledge.auto_ingest_enabled`` is set; the queue is always wired so
- # the middleware can enqueue without a hard dependency.
- from deerflow.knowledge.auto_ingest import AutoIngestQueue, set_auto_ingest_queue
-
- if app.state.knowledge_service is not None:
- _auto_ingest_queue = AutoIngestQueue(
- lambda: getattr(app.state, "knowledge_service", None),
- min_chars=config.knowledge.auto_ingest_min_chars,
- )
- _auto_ingest_queue.start()
- set_auto_ingest_queue(_auto_ingest_queue)
- app.state.knowledge_auto_ingest_queue = _auto_ingest_queue
-
- from deerflow.persistence.user_prompts import make_user_prompt_store
-
- app.state.user_prompt_store = make_user_prompt_store(sf)
-
- from deerflow.persistence.user_preferences import make_user_preferences_store
-
- app.state.user_preferences_store = make_user_preferences_store(sf)
-
- from deerflow.persistence.roundtable_drafts import make_roundtable_draft_store
-
- app.state.roundtable_draft_store = make_roundtable_draft_store(sf)
-
- # Task-scoped roundtable drafts (taskId 深链聊天记录,独立存、不按 user 分权)。
- from deerflow.persistence.roundtable_task_drafts import make_roundtable_task_draft_store
-
- app.state.roundtable_task_draft_store = make_roundtable_task_draft_store(sf)
-
- from deerflow.persistence.roundtable_draft_shares import make_roundtable_draft_share_store
-
- app.state.roundtable_draft_share_store = make_roundtable_draft_share_store(sf)
-
- from deerflow.persistence.roundtable_chains import make_roundtable_chain_store
-
- app.state.roundtable_chain_store = make_roundtable_chain_store(sf)
-
- # 独立的岗位协同 Session / Node 存储。它只索引业务流程状态,真实消息和
- # 文件仍复用 LangGraph thread/checkpoint 与现有 artifact API。
- from deerflow.persistence.position_roundtable import make_position_roundtable_store
-
- app.state.position_roundtable_store = make_position_roundtable_store(sf)
-
- # 岗位会商工作视角(名称 / 顺序 / 特殊意图岗)是全局配置,和组织/RBAC
- # 岗位分开保存。启动只补齐缺失内置项,不覆盖管理员的既有修改。
- from app.gateway.position_role_defaults import POSITION_ROLE_SEED
- from deerflow.persistence.position_roles import make_position_role_store
-
- app.state.position_role_store = make_position_role_store(sf)
- await app.state.position_role_store.ensure_seed(POSITION_ROLE_SEED)
-
- from deerflow.persistence.business_mapping import make_business_mapping_store
-
- app.state.business_mapping_store = make_business_mapping_store(sf)
-
- from deerflow.persistence.light_apps import make_light_app_store
-
- app.state.light_app_store = make_light_app_store(sf)
-
- from deerflow.persistence.menu_overrides import make_menu_override_store
-
- app.state.menu_override_store = make_menu_override_store(sf)
-
- from deerflow.persistence.compact_menu_overrides import make_compact_menu_override_store
-
- app.state.compact_menu_override_store = make_compact_menu_override_store(sf)
-
- from deerflow.persistence.sensitive_words import make_sensitive_word_store
-
- app.state.sensitive_word_store = make_sensitive_word_store(sf)
-
- from deerflow.persistence.dashboard_sessions import make_dashboard_session_store
-
- app.state.dashboard_session_store = make_dashboard_session_store(sf)
-
- from deerflow.persistence.task_buttons import make_task_button_store
-
- app.state.task_button_store = make_task_button_store(sf)
-
- from deerflow.persistence.report_structures import make_report_structure_store
-
- app.state.report_structure_store = make_report_structure_store(sf)
-
- from deerflow.integrations.weknora.runtime import register_llmwiki_store
- from deerflow.persistence.llmwiki import make_llmwiki_store
-
+ from deerflow.persistence.fixed_questions import make_fixed_question_store
+
+ app.state.fixed_question_store = make_fixed_question_store(sf)
+
+ from deerflow.persistence.thread_meta import make_thread_store
+
+ app.state.thread_store = make_thread_store(sf, app.state.store)
+
+ from deerflow.persistence.scheduled_tasks import make_scheduled_task_store
+
+ app.state.scheduled_task_store = make_scheduled_task_store(sf)
+
+ from deerflow.persistence.html_pages import make_html_page_store
+
+ app.state.html_page_store = make_html_page_store(sf)
+
+ from deerflow.persistence.llm_metrics import make_llm_metrics_store
+
+ app.state.llm_metrics_store = make_llm_metrics_store(sf)
+
+ from deerflow.persistence.concurrency import make_concurrency_sample_store
+
+ app.state.concurrency_sample_store = make_concurrency_sample_store(sf)
+
+ from deerflow.persistence.tool_metrics import make_tool_metrics_store
+
+ app.state.tool_metrics_store = make_tool_metrics_store(sf)
+
+ from deerflow.persistence.agents import make_agent_store
+
+ app.state.agent_store = make_agent_store(sf)
+
+ from deerflow.persistence.skills import make_skill_store
+
+ app.state.skill_store = make_skill_store(sf)
+
+ from deerflow.persistence.notifications import (
+ make_notification_store,
+ set_notification_store,
+ )
+
+ app.state.notification_store = make_notification_store(sf)
+ # Make the sender helper (harness layer) able to write without going
+ # through FastAPI request context — used by cascade.py.
+ set_notification_store(app.state.notification_store)
+ from deerflow.persistence.recommended_questions import make_recommended_question_store
+
+ app.state.recommended_question_store = make_recommended_question_store(sf)
+
+ from deerflow.persistence.tags import make_tag_store
+
+ app.state.tag_store = make_tag_store(sf)
+
+ from deerflow.persistence.positions import make_position_store
+
+ app.state.position_store = make_position_store(sf)
+
+ from deerflow.services.position_sync import PositionSyncService
+
+ app.state.position_sync = PositionSyncService(sf, app.state.position_store, app.state.tag_store)
+
+ from deerflow.persistence.thread_shares import make_thread_share_store
+
+ app.state.thread_share_store = make_thread_share_store(sf)
+
+ from deerflow.persistence.embed_sessions import make_embed_session_store
+
+ app.state.embed_session_store = make_embed_session_store(sf)
+
+ from deerflow.persistence.ai_writing_sessions import (
+ make_ai_writing_session_store,
+ )
+ from deerflow.persistence.ai_writing_sessions import (
+ register_default_store as _register_ai_writing_default_store,
+ )
+
+ app.state.ai_writing_session_store = make_ai_writing_session_store(sf)
+ # 让 LangGraph Server 路径下的 AI 写作节点能跨 app/harness 边界拿到 repo。
+ # 老 /start /stream /resume 路由仍走 request.app.state,互不影响。
+ _register_ai_writing_default_store(app.state.ai_writing_session_store)
+
+ # AI 写作「后台挂起」执行器:把写作图作为后台任务驱动到定稿(离页/刷新/切号不打断)。
+ # 用 app.state.checkpointer(与运行时 worker 同一份 saver)续上线程 checkpoint。
+ from app.gateway.ai_writing_job_executor import AIWritingJobExecutor
+
+ app.state.ai_writing_job_executor = AIWritingJobExecutor(app)
+ # 进程重启续跑:把仍标记为 background 的会话重新拉起驱动(best-effort,不阻塞启动)。
+ if app.state.ai_writing_session_store is not None:
+ from app.gateway.ai_writing_job_executor import STATUS_BACKGROUND
+
+ async def _reconcile_ai_writing_background() -> None:
+ try:
+ sessions = await app.state.ai_writing_session_store.list_by_status(STATUS_BACKGROUND)
+ app.state.ai_writing_job_executor.reconcile(sessions=sessions)
+ except Exception: # noqa: BLE001
+ logger.warning("[ai-writing-bg] startup reconcile failed", exc_info=True)
+
+ asyncio.create_task(_reconcile_ai_writing_background())
+
+ from deerflow.persistence.article_types import make_article_type_store
+
+ app.state.article_type_store = make_article_type_store(sf)
+
+ from deerflow.persistence.research_templates import make_research_template_store
+
+ app.state.research_template_store = make_research_template_store(sf)
+
+ from deerflow.persistence.memory_config import make_user_memory_config_store
+
+ app.state.memory_config_store = make_user_memory_config_store(sf)
+
+ from deerflow.knowledge import make_knowledge_service
+
+ app.state.knowledge_service = make_knowledge_service(sf, config.knowledge)
+
+ # Seed the built-in extraction prompts as editable templates so operators
+ # can immediately view/edit them (idempotent; never overwrites edits).
+ if app.state.knowledge_service is not None:
+ try:
+ await app.state.knowledge_service.seed_default_templates()
+ except Exception:
+ logger.exception("Failed to seed knowledge extraction templates")
+
+ # Background auto-sedimentation queue (phase 3) — drains finished
+ # conversations off the chat path. Worker only does work when
+ # ``knowledge.auto_ingest_enabled`` is set; the queue is always wired so
+ # the middleware can enqueue without a hard dependency.
+ from deerflow.knowledge.auto_ingest import AutoIngestQueue, set_auto_ingest_queue
+
+ if app.state.knowledge_service is not None:
+ _auto_ingest_queue = AutoIngestQueue(
+ lambda: getattr(app.state, "knowledge_service", None),
+ min_chars=config.knowledge.auto_ingest_min_chars,
+ )
+ _auto_ingest_queue.start()
+ set_auto_ingest_queue(_auto_ingest_queue)
+ app.state.knowledge_auto_ingest_queue = _auto_ingest_queue
+
+ from deerflow.persistence.user_prompts import make_user_prompt_store
+
+ app.state.user_prompt_store = make_user_prompt_store(sf)
+
+ from deerflow.persistence.user_preferences import make_user_preferences_store
+
+ app.state.user_preferences_store = make_user_preferences_store(sf)
+
+ from deerflow.persistence.roundtable_drafts import make_roundtable_draft_store
+
+ app.state.roundtable_draft_store = make_roundtable_draft_store(sf)
+
+ # Task-scoped roundtable drafts (taskId 深链聊天记录,独立存、不按 user 分权)。
+ from deerflow.persistence.roundtable_task_drafts import make_roundtable_task_draft_store
+
+ app.state.roundtable_task_draft_store = make_roundtable_task_draft_store(sf)
+
+ from deerflow.persistence.roundtable_draft_shares import make_roundtable_draft_share_store
+
+ app.state.roundtable_draft_share_store = make_roundtable_draft_share_store(sf)
+
+ from deerflow.persistence.roundtable_chains import make_roundtable_chain_store
+
+ app.state.roundtable_chain_store = make_roundtable_chain_store(sf)
+
+ # 独立的岗位协同 Session / Node 存储。它只索引业务流程状态,真实消息和
+ # 文件仍复用 LangGraph thread/checkpoint 与现有 artifact API。
+ from deerflow.persistence.position_roundtable import make_position_roundtable_store
+
+ app.state.position_roundtable_store = make_position_roundtable_store(sf)
+
+ # 岗位会商工作视角(名称 / 顺序 / 特殊意图岗)是全局配置,和组织/RBAC
+ # 岗位分开保存。启动只补齐缺失内置项,不覆盖管理员的既有修改。
+ from app.gateway.position_role_defaults import POSITION_ROLE_SEED
+ from deerflow.persistence.position_roles import make_position_role_store
+
+ app.state.position_role_store = make_position_role_store(sf)
+ await app.state.position_role_store.ensure_seed(POSITION_ROLE_SEED)
+
+ from deerflow.persistence.business_mapping import make_business_mapping_store
+
+ app.state.business_mapping_store = make_business_mapping_store(sf)
+
+ from deerflow.persistence.light_apps import make_light_app_store
+
+ app.state.light_app_store = make_light_app_store(sf)
+
+ from deerflow.persistence.menu_overrides import make_menu_override_store
+
+ app.state.menu_override_store = make_menu_override_store(sf)
+
+ from deerflow.persistence.compact_menu_overrides import make_compact_menu_override_store
+
+ app.state.compact_menu_override_store = make_compact_menu_override_store(sf)
+
+ from deerflow.persistence.sensitive_words import make_sensitive_word_store
+
+ app.state.sensitive_word_store = make_sensitive_word_store(sf)
+
+ from deerflow.persistence.dashboard_sessions import make_dashboard_session_store
+
+ app.state.dashboard_session_store = make_dashboard_session_store(sf)
+
+ from deerflow.persistence.task_buttons import make_task_button_store
+
+ app.state.task_button_store = make_task_button_store(sf)
+
+ from deerflow.persistence.report_structures import make_report_structure_store
+
+ app.state.report_structure_store = make_report_structure_store(sf)
+
+ from deerflow.integrations.weknora.runtime import register_llmwiki_store
+ from deerflow.persistence.llmwiki import make_llmwiki_store
+
app.state.llmwiki_store = make_llmwiki_store(sf)
register_llmwiki_store(app.state.llmwiki_store)
- from deerflow.persistence.taskcop_tasks import make_taskcop_task_store
+ # DeerFlow-local Wiki mirror/vector index. The store is always wired so
+ # migrations and status inspection work while feature flags are off;
+ # network/search services only exist when explicitly enabled.
+ from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
+ from deerflow.persistence.llmwiki_index import make_llmwiki_index_store
- app.state.taskcop_task_store = make_taskcop_task_store(sf)
+ app.state.llmwiki_index_store = make_llmwiki_index_store(sf)
+ app.state.llmwiki_embedding = None
+ app.state.llmwiki_vector_cache = None
+ app.state.llmwiki_vector_search = None
+ app.state.llmwiki_sync_service = None
+ app.state.llmwiki_retrieval_service = None
+ local_index_config = config.llmwiki.local_wiki_index
+ from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
- from deerflow.persistence.task_reports import make_task_report_store
+ resolved_llmwiki = get_resolved_llmwiki_runtime(config)
+ weknora_client = build_weknora_client(resolved_llmwiki) if resolved_llmwiki.weknora_enabled else None
+ if local_index_config.enabled:
+ from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
+ from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
+ from deerflow.integrations.weknora.local_index.sync import WikiSyncService
+ from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
- app.state.task_report_store = make_task_report_store(sf)
-
- from deerflow.persistence.roundtable_artifact_submissions import make_roundtable_artifact_submission_store
-
- # Task-scoped artifact submission status: shared by task; actor is audit-only.
- app.state.roundtable_artifact_submission_store = make_roundtable_artifact_submission_store(sf)
-
- from deerflow.persistence.parallel_agent_tasks import make_parallel_agent_task_store
-
- app.state.parallel_agent_task_store = make_parallel_agent_task_store(sf)
-
- from deerflow.persistence.roundtable_jobs import (
- make_roundtable_job_store,
- )
- from deerflow.persistence.roundtable_jobs import (
- register_default_store as _register_roundtable_job_default_store,
- )
-
- app.state.roundtable_job_store = make_roundtable_job_store(sf)
- # 让 LangGraph Server 路径下的圆桌编排图节点能跨 app/harness 边界拿到 repo
- # (Phase 2 后台编排图写进度用)。
- _register_roundtable_job_default_store(app.state.roundtable_job_store)
-
- # 意图回合单飞 store(Phase 4):``/api/intent/stream`` 据此对 (thread_id,
- # client_turn_id) 跨 worker 单飞 + 幂等回放,杜绝并发双发意图。sf 为 None(无 DB)
- # 时为 None,路由侧判空跳过单飞、走原流程。
- from deerflow.persistence.intent_turns import make_intent_turn_store
-
- app.state.intent_turn_store = make_intent_turn_store(sf)
-
- # 圆桌会商「诊断日志」store:会商全流程各处报错 / 关键事件落库,供管理员诊断页筛选。
- # 同样登记进程级默认仓储——harness 侧编排引擎 / 进程内网关跨边界写诊断用。
- from deerflow.persistence.roundtable_diagnostics import (
- make_roundtable_diagnostic_store,
- )
- from deerflow.persistence.roundtable_diagnostics import (
- register_default_store as _register_roundtable_diagnostic_default_store,
- )
-
- app.state.roundtable_diagnostic_store = make_roundtable_diagnostic_store(sf)
- _register_roundtable_diagnostic_default_store(app.state.roundtable_diagnostic_store)
-
- # 圆桌后台作业执行器:把编排引擎作为后台任务跑。
- # 网关由环境变量 ``ROUNDTABLE_JOB_GATEWAY`` 选择(见 resolve_roundtable_gateway_mode):
- # - 缺省 / "inprocess"(Phase 3b):进程内驱动真模型(leader/seat/report)——**默认**,
- # 部署时无需任何配置即可用后台挂起;
- # - 显式 "simulated"(Phase 3a):模拟网关,产生真实进度但不调真模型,供前端联调。
- if app.state.roundtable_job_store is not None:
- import os
-
- from app.gateway.roundtable_job_executor import RoundtableJobExecutor
-
- gateway_mode = resolve_roundtable_gateway_mode(os.getenv("ROUNDTABLE_JOB_GATEWAY"))
- if gateway_mode == "simulated":
- from deerflow.agents.roundtable_orchestrator import SimulatedRoundtableGateway
-
- # delay=3s/步:让模拟后台任务有可观察的进行中时长(切页/重连仍在跑)。
- gateway_factory = lambda p: SimulatedRoundtableGateway( # noqa: E731
- p.agents, mode=p.mode, delay=3.0
+ app.state.llmwiki_embedding = StrictWikiEmbeddingClient(local_index_config.embedding)
+ app.state.llmwiki_vector_cache = WikiVectorCache(
+ app.state.llmwiki_index_store,
+ max_knowledge_bases=local_index_config.cache_max_knowledge_bases,
+ revision_check_seconds=local_index_config.cache_revision_check_seconds,
+ )
+ app.state.llmwiki_vector_search = WikiVectorSearchService(
+ local_index_config,
+ app.state.llmwiki_embedding,
+ app.state.llmwiki_vector_cache,
+ )
+ if weknora_client is not None:
+ app.state.llmwiki_sync_service = WikiSyncService(
+ local_index_config,
+ app.state.llmwiki_index_store,
+ app.state.llmwiki_embedding,
+ app.state.llmwiki_vector_cache,
+ weknora_client,
)
- logger.info("[roundtable-jobs] gateway = simulated")
- else:
- from app.gateway.roundtable_inprocess_gateway import InProcessRoundtableGateway
+ if weknora_client is not None or app.state.llmwiki_vector_search is not None:
+ from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
- _app = app # 闭包捕获 app,供网关访问 app.state(checkpointer/thread_store)
- gateway_factory = lambda p: InProcessRoundtableGateway( # noqa: E731
- _app,
- user_id=p.user_id,
- agents=p.agents,
- model=p.model,
- # 席位执行模式沿用 Step2 选择,后台席位 run 据此覆盖 policy 的 thinking/reasoning + 开子代理。
- seat_thinking_enabled=p.seat_thinking_enabled,
- seat_reasoning_effort=p.seat_reasoning_effort,
- seat_subagent_enabled=p.seat_subagent_enabled,
- seat_skill_directives=p.seat_skill_directives,
- # 每席位推理深度覆盖(业务链条里各席位单独配):run_seat 命中即覆盖作业级 seatMode。
- seat_modes=p.seat_modes,
- # 诊断日志上下文:网关每轮 leader/seat/report/summary/dashboard 报错均带这些归属。
- job_id=p.job_id,
- draft_id=p.draft_id,
- task_id=p.task_id,
- )
- logger.info("[roundtable-jobs] gateway = inprocess (real models, default)")
-
- app.state.roundtable_job_executor = RoundtableJobExecutor(
- app.state.roundtable_job_store,
- gateway_factory=gateway_factory,
- draft_store=getattr(app.state, "roundtable_draft_store", None),
- # task 作业把最终报告写回**独立的** task-draft 存储(不分权)。
- task_draft_store=getattr(app.state, "roundtable_task_draft_store", None),
+ app.state.llmwiki_retrieval_service = WikiRetrievalService(
+ app.state.llmwiki_index_store,
+ vector_search=(
+ app.state.llmwiki_vector_search
+ if local_index_config.internal_search_enabled
+ else None
+ ),
+ client=weknora_client,
)
-
- # Phase 3 dispatcher:跨 worker 领取 queued/租约过期作业 + outbox 补偿器。
- # 每个 worker 进程都跑一个;谁先 claim_job 成功谁执行(DB 条件 UPDATE 互斥),
- # 崩溃的 worker 租约过期后由其它 worker 接管。路由 start/resume 后 nudge 立即唤醒。
- from app.gateway.roundtable_job_dispatcher import RoundtableJobDispatcher
-
- app.state.roundtable_job_dispatcher = RoundtableJobDispatcher(
- app.state.roundtable_job_store,
- app.state.roundtable_job_executor,
- draft_store=getattr(app.state, "roundtable_draft_store", None),
- task_draft_store=getattr(app.state, "roundtable_task_draft_store", None),
- )
- app.state.roundtable_job_dispatcher.start()
- else:
- app.state.roundtable_job_executor = None
- app.state.roundtable_job_dispatcher = None
-
- # ── Deep Research (深度研究) stores + executor + dispatcher ──────────
- # Five tables: sessions / jobs / events / sources / messages.
- # When DB is available, the executor + dispatcher run background research
- # jobs (lease-based claim, crash-recoverable) mirroring the roundtable pattern.
- from deerflow.persistence.deep_research_events import (
- make_deep_research_event_store,
+ register_local_wiki_index_runtime(
+ store=app.state.llmwiki_index_store,
+ search=app.state.llmwiki_vector_search,
+ sync=app.state.llmwiki_sync_service,
+ retrieval=app.state.llmwiki_retrieval_service,
)
- from deerflow.persistence.deep_research_jobs import make_deep_research_job_store
- from deerflow.persistence.deep_research_messages import (
- make_deep_research_message_store,
- )
- from deerflow.persistence.deep_research_sessions import (
- make_deep_research_session_store,
- )
- from deerflow.persistence.deep_research_sources import (
- make_deep_research_source_store,
- )
- from deerflow.persistence.document_rewrite_versions import (
- make_document_rewrite_version_store,
- )
-
- app.state.deep_research_session_store = make_deep_research_session_store(sf)
- app.state.deep_research_job_store = make_deep_research_job_store(sf)
- app.state.deep_research_event_store = make_deep_research_event_store(sf)
- app.state.deep_research_source_store = make_deep_research_source_store(sf)
- app.state.deep_research_message_store = make_deep_research_message_store(sf)
- # Whole-document AI rewrites retain a durable original snapshot so a
- # completed replacement can be safely undone after a refresh.
- app.state.document_rewrite_version_store = make_document_rewrite_version_store(sf)
- # Low-latency in-process SSE fan-out. The database event log remains
- # authoritative for reconnect replay and cross-worker recovery.
- from app.gateway.deep_research_live_hub import DeepResearchLiveHub
-
- app.state.deep_research_live_hub = DeepResearchLiveHub()
-
- if app.state.deep_research_job_store is not None:
- from app.gateway.deep_research_job_dispatcher import (
- DeepResearchJobDispatcher,
- )
- from app.gateway.deep_research_job_executor import (
- DeepResearchJobExecutor,
- )
-
- app.state.deep_research_executor = DeepResearchJobExecutor(
- app.state.deep_research_job_store,
- app.state.deep_research_session_store,
- app.state.deep_research_event_store,
- app.state.deep_research_source_store,
- live_publisher=app.state.deep_research_live_hub.publish,
- checkpointer=app.state.checkpointer,
- document_rewrite_version_store=app.state.document_rewrite_version_store,
- app_config=config,
- )
- app.state.deep_research_dispatcher = DeepResearchJobDispatcher(
- app.state.deep_research_job_store,
- app.state.deep_research_executor,
- )
- app.state.deep_research_dispatcher.start()
- else:
- app.state.deep_research_executor = None
- app.state.deep_research_dispatcher = None
-
- # Run event store (has its own factory with config-driven backend selection)
- run_events_config = getattr(config, "run_events", None)
- app.state.run_event_store = make_run_event_store(run_events_config)
-
- # RunManager with store backing for persistence
- app.state.run_manager = RunManager(store=app.state.run_store)
- from deerflow.runtime import make_redis_concurrency_gate
-
- app.state.concurrency_gate = make_redis_concurrency_gate(config)
-
+
+ from deerflow.persistence.taskcop_tasks import make_taskcop_task_store
+
+ app.state.taskcop_task_store = make_taskcop_task_store(sf)
+
+ from deerflow.persistence.task_reports import make_task_report_store
+
+ app.state.task_report_store = make_task_report_store(sf)
+
+ from deerflow.persistence.roundtable_artifact_submissions import make_roundtable_artifact_submission_store
+
+ # Task-scoped artifact submission status: shared by task; actor is audit-only.
+ app.state.roundtable_artifact_submission_store = make_roundtable_artifact_submission_store(sf)
+
+ from deerflow.persistence.parallel_agent_tasks import make_parallel_agent_task_store
+
+ app.state.parallel_agent_task_store = make_parallel_agent_task_store(sf)
+
+ from deerflow.persistence.roundtable_jobs import (
+ make_roundtable_job_store,
+ )
+ from deerflow.persistence.roundtable_jobs import (
+ register_default_store as _register_roundtable_job_default_store,
+ )
+
+ app.state.roundtable_job_store = make_roundtable_job_store(sf)
+ # 让 LangGraph Server 路径下的圆桌编排图节点能跨 app/harness 边界拿到 repo
+ # (Phase 2 后台编排图写进度用)。
+ _register_roundtable_job_default_store(app.state.roundtable_job_store)
+
+ # 意图回合单飞 store(Phase 4):``/api/intent/stream`` 据此对 (thread_id,
+ # client_turn_id) 跨 worker 单飞 + 幂等回放,杜绝并发双发意图。sf 为 None(无 DB)
+ # 时为 None,路由侧判空跳过单飞、走原流程。
+ from deerflow.persistence.intent_turns import make_intent_turn_store
+
+ app.state.intent_turn_store = make_intent_turn_store(sf)
+
+ # 圆桌会商「诊断日志」store:会商全流程各处报错 / 关键事件落库,供管理员诊断页筛选。
+ # 同样登记进程级默认仓储——harness 侧编排引擎 / 进程内网关跨边界写诊断用。
+ from deerflow.persistence.roundtable_diagnostics import (
+ make_roundtable_diagnostic_store,
+ )
+ from deerflow.persistence.roundtable_diagnostics import (
+ register_default_store as _register_roundtable_diagnostic_default_store,
+ )
+
+ app.state.roundtable_diagnostic_store = make_roundtable_diagnostic_store(sf)
+ _register_roundtable_diagnostic_default_store(app.state.roundtable_diagnostic_store)
+
+ # 圆桌后台作业执行器:把编排引擎作为后台任务跑。
+ # 网关由环境变量 ``ROUNDTABLE_JOB_GATEWAY`` 选择(见 resolve_roundtable_gateway_mode):
+ # - 缺省 / "inprocess"(Phase 3b):进程内驱动真模型(leader/seat/report)——**默认**,
+ # 部署时无需任何配置即可用后台挂起;
+ # - 显式 "simulated"(Phase 3a):模拟网关,产生真实进度但不调真模型,供前端联调。
+ if app.state.roundtable_job_store is not None:
+ import os
+
+ from app.gateway.roundtable_job_executor import RoundtableJobExecutor
+
+ gateway_mode = resolve_roundtable_gateway_mode(os.getenv("ROUNDTABLE_JOB_GATEWAY"))
+ if gateway_mode == "simulated":
+ from deerflow.agents.roundtable_orchestrator import SimulatedRoundtableGateway
+
+ # delay=3s/步:让模拟后台任务有可观察的进行中时长(切页/重连仍在跑)。
+ gateway_factory = lambda p: SimulatedRoundtableGateway( # noqa: E731
+ p.agents, mode=p.mode, delay=3.0
+ )
+ logger.info("[roundtable-jobs] gateway = simulated")
+ else:
+ from app.gateway.roundtable_inprocess_gateway import InProcessRoundtableGateway
+
+ _app = app # 闭包捕获 app,供网关访问 app.state(checkpointer/thread_store)
+ gateway_factory = lambda p: InProcessRoundtableGateway( # noqa: E731
+ _app,
+ user_id=p.user_id,
+ agents=p.agents,
+ model=p.model,
+ # 席位执行模式沿用 Step2 选择,后台席位 run 据此覆盖 policy 的 thinking/reasoning + 开子代理。
+ seat_thinking_enabled=p.seat_thinking_enabled,
+ seat_reasoning_effort=p.seat_reasoning_effort,
+ seat_subagent_enabled=p.seat_subagent_enabled,
+ seat_skill_directives=p.seat_skill_directives,
+ # 每席位推理深度覆盖(业务链条里各席位单独配):run_seat 命中即覆盖作业级 seatMode。
+ seat_modes=p.seat_modes,
+ # 诊断日志上下文:网关每轮 leader/seat/report/summary/dashboard 报错均带这些归属。
+ job_id=p.job_id,
+ draft_id=p.draft_id,
+ task_id=p.task_id,
+ )
+ logger.info("[roundtable-jobs] gateway = inprocess (real models, default)")
+
+ app.state.roundtable_job_executor = RoundtableJobExecutor(
+ app.state.roundtable_job_store,
+ gateway_factory=gateway_factory,
+ draft_store=getattr(app.state, "roundtable_draft_store", None),
+ # task 作业把最终报告写回**独立的** task-draft 存储(不分权)。
+ task_draft_store=getattr(app.state, "roundtable_task_draft_store", None),
+ )
+
+ # Phase 3 dispatcher:跨 worker 领取 queued/租约过期作业 + outbox 补偿器。
+ # 每个 worker 进程都跑一个;谁先 claim_job 成功谁执行(DB 条件 UPDATE 互斥),
+ # 崩溃的 worker 租约过期后由其它 worker 接管。路由 start/resume 后 nudge 立即唤醒。
+ from app.gateway.roundtable_job_dispatcher import RoundtableJobDispatcher
+
+ app.state.roundtable_job_dispatcher = RoundtableJobDispatcher(
+ app.state.roundtable_job_store,
+ app.state.roundtable_job_executor,
+ draft_store=getattr(app.state, "roundtable_draft_store", None),
+ task_draft_store=getattr(app.state, "roundtable_task_draft_store", None),
+ )
+ app.state.roundtable_job_dispatcher.start()
+ else:
+ app.state.roundtable_job_executor = None
+ app.state.roundtable_job_dispatcher = None
+
+ # ── Deep Research (深度研究) stores + executor + dispatcher ──────────
+ # Five tables: sessions / jobs / events / sources / messages.
+ # When DB is available, the executor + dispatcher run background research
+ # jobs (lease-based claim, crash-recoverable) mirroring the roundtable pattern.
+ from deerflow.persistence.deep_research_events import (
+ make_deep_research_event_store,
+ )
+ from deerflow.persistence.deep_research_jobs import make_deep_research_job_store
+ from deerflow.persistence.deep_research_messages import (
+ make_deep_research_message_store,
+ )
+ from deerflow.persistence.deep_research_sessions import (
+ make_deep_research_session_store,
+ )
+ from deerflow.persistence.deep_research_sources import (
+ make_deep_research_source_store,
+ )
+ from deerflow.persistence.document_rewrite_versions import (
+ make_document_rewrite_version_store,
+ )
+
+ app.state.deep_research_session_store = make_deep_research_session_store(sf)
+ app.state.deep_research_job_store = make_deep_research_job_store(sf)
+ app.state.deep_research_event_store = make_deep_research_event_store(sf)
+ app.state.deep_research_source_store = make_deep_research_source_store(sf)
+ app.state.deep_research_message_store = make_deep_research_message_store(sf)
+ # Whole-document AI rewrites retain a durable original snapshot so a
+ # completed replacement can be safely undone after a refresh.
+ app.state.document_rewrite_version_store = make_document_rewrite_version_store(sf)
+ # Low-latency in-process SSE fan-out. The database event log remains
+ # authoritative for reconnect replay and cross-worker recovery.
+ from app.gateway.deep_research_live_hub import DeepResearchLiveHub
+
+ app.state.deep_research_live_hub = DeepResearchLiveHub()
+
+ if app.state.deep_research_job_store is not None:
+ from app.gateway.deep_research_job_dispatcher import (
+ DeepResearchJobDispatcher,
+ )
+ from app.gateway.deep_research_job_executor import (
+ DeepResearchJobExecutor,
+ )
+
+ app.state.deep_research_executor = DeepResearchJobExecutor(
+ app.state.deep_research_job_store,
+ app.state.deep_research_session_store,
+ app.state.deep_research_event_store,
+ app.state.deep_research_source_store,
+ live_publisher=app.state.deep_research_live_hub.publish,
+ checkpointer=app.state.checkpointer,
+ document_rewrite_version_store=app.state.document_rewrite_version_store,
+ app_config=config,
+ )
+ app.state.deep_research_dispatcher = DeepResearchJobDispatcher(
+ app.state.deep_research_job_store,
+ app.state.deep_research_executor,
+ )
+ app.state.deep_research_dispatcher.start()
+ else:
+ app.state.deep_research_executor = None
+ app.state.deep_research_dispatcher = None
+
+ # Run event store (has its own factory with config-driven backend selection)
+ run_events_config = getattr(config, "run_events", None)
+ app.state.run_event_store = make_run_event_store(run_events_config)
+
+ # RunManager with store backing for persistence
+ app.state.run_manager = RunManager(store=app.state.run_store)
+ from deerflow.runtime import make_redis_concurrency_gate
+
+ app.state.concurrency_gate = make_redis_concurrency_gate(config)
+
try:
yield
finally:
+ from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
+
+ register_local_wiki_index_runtime(store=None, search=None, sync=None, retrieval=None)
+ llmwiki_embedding = getattr(app.state, "llmwiki_embedding", None)
+ if llmwiki_embedding is not None:
+ await llmwiki_embedding.close()
from deerflow.persistence.notifications import set_notification_store
-
- set_notification_store(None)
- dispatcher = getattr(app.state, "roundtable_job_dispatcher", None)
- if dispatcher is not None:
- await dispatcher.stop()
- dr_dispatcher = getattr(app.state, "deep_research_dispatcher", None)
- if dr_dispatcher is not None:
- await dr_dispatcher.stop()
- queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
- if queue is not None:
- from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
-
- await queue.stop()
- set_auto_ingest_queue(None)
- await close_engine()
- gate = getattr(app.state, "concurrency_gate", None)
- if gate is not None:
- await gate.aclose()
-
-
-# ---------------------------------------------------------------------------
-# Getters – called by routers per-request
-# ---------------------------------------------------------------------------
-
-
-def _require(attr: str, label: str) -> Callable[[Request], T]:
- """Create a FastAPI dependency that returns ``app.state.`` or 503."""
-
- def dep(request: Request) -> T:
- val = getattr(request.app.state, attr, None)
- if val is None:
- raise HTTPException(status_code=503, detail=f"{label} not available")
- return cast(T, val)
-
- dep.__name__ = dep.__qualname__ = f"get_{attr}"
- return dep
-
-
-get_stream_bridge: Callable[[Request], StreamBridge] = _require("stream_bridge", "Stream bridge")
-get_run_manager: Callable[[Request], RunManager] = _require("run_manager", "Run manager")
-get_checkpointer: Callable[[Request], Checkpointer] = _require("checkpointer", "Checkpointer")
-get_run_event_store: Callable[[Request], RunEventStore] = _require("run_event_store", "Run event store")
-get_feedback_repo: Callable[[Request], FeedbackRepository] = _require("feedback_repo", "Feedback")
-get_run_store: Callable[[Request], RunStore] = _require("run_store", "Run store")
-
-
-def get_concurrency_gate(request: Request):
- """Return the optional cross-worker concurrency gate."""
- return getattr(request.app.state, "concurrency_gate", None)
-
-
-def get_store(request: Request):
- """Return the global store (may be ``None`` if not configured)."""
- return getattr(request.app.state, "store", None)
-
-
-def get_thread_store(request: Request) -> ThreadMetaStore:
- """Return the thread metadata store (SQL or memory-backed)."""
- val = getattr(request.app.state, "thread_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Thread metadata store not available")
- return val
-
-
-def get_scheduled_task_store(request: Request) -> ScheduledTaskStore:
- """Return the scheduled-task store."""
- val = getattr(request.app.state, "scheduled_task_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Scheduled task store not available")
- return val
-
-
-def get_llm_metrics_store(request: Request):
- """Return the LLM metrics store, or None when not configured."""
- return getattr(request.app.state, "llm_metrics_store", None)
-
-
-def get_concurrency_sample_store(request: Request):
- """Return the concurrency-sample store, or None on the memory backend."""
- return getattr(request.app.state, "concurrency_sample_store", None)
-
-
-def get_tool_metrics_store(request: Request):
- """Return the tool-call metrics store, or None when not configured."""
- return getattr(request.app.state, "tool_metrics_store", None)
-
-
-def get_agent_store(request: Request) -> AgentStore:
- """Return the custom-agent store."""
- val = getattr(request.app.state, "agent_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Agent store not available")
- return val
-
-
-def get_skill_store(request: Request) -> SkillStore:
- """Return the custom-skill store."""
- val = getattr(request.app.state, "skill_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Skill store not available")
- return val
-
-
-def get_notification_store(request: Request) -> NotificationStore:
- """Return the in-app notification store."""
- val = getattr(request.app.state, "notification_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Notification store not available")
- return val
-
-
-def get_tag_store(request: Request) -> TagStore:
- """Return the tag store."""
- val = getattr(request.app.state, "tag_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Tag store not available")
- return val
-
-
-def get_position_store(request: Request):
- """Return the 岗位 (position) store."""
- val = getattr(request.app.state, "position_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Position store not available")
- return val
-
-
-def get_position_sync(request: Request):
- """Return the position → subscription sync service."""
- val = getattr(request.app.state, "position_sync", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Position sync service not available")
- return val
-
-
-def get_thread_share_store(request: Request):
- """Return the conversation share-link store."""
- val = getattr(request.app.state, "thread_share_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Thread share store not available")
- return val
-
-
-def get_roundtable_draft_share_store(request: Request):
- """Return the roundtable-draft share-link store."""
- val = getattr(request.app.state, "roundtable_draft_share_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Roundtable draft share store not available")
- return val
-
-
-def get_embed_session_store(request: Request):
- """Return the iframe embed session → thread mapping store."""
- val = getattr(request.app.state, "embed_session_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Embed session store not available")
- return val
-
-
-def get_ai_writing_session_store(request: Request):
- """Return the AI writing session store, or None when not configured."""
- return getattr(request.app.state, "ai_writing_session_store", None)
-
-
-def get_ai_writing_job_executor(request: Request):
- """Return the AI writing background-suspend executor, or None when unavailable."""
- return getattr(request.app.state, "ai_writing_job_executor", None)
-
-
-def get_article_type_store(request: Request):
- """Return the article type store, or None when not configured."""
- return getattr(request.app.state, "article_type_store", None)
-
-
-def get_research_template_store(request: Request):
- """Return the research template store, or None when not configured."""
- return getattr(request.app.state, "research_template_store", None)
-
-
-def get_memory_config_store(request: Request):
- """Return the per-user memory config store."""
- val = getattr(request.app.state, "memory_config_store", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Memory config store not available")
- return val
-
-
-def get_knowledge_service(request: Request):
- """Return the knowledge base service (503 when disabled / memory backend)."""
- val = getattr(request.app.state, "knowledge_service", None)
- if val is None:
- raise HTTPException(status_code=503, detail="Knowledge base not available")
- return val
-
-
-def get_run_context(request: Request) -> RunContext:
- """Build a :class:`RunContext` from ``app.state`` singletons.
-
- Returns a *base* context with infrastructure dependencies.
- """
- config = get_config(request)
- return RunContext(
- checkpointer=get_checkpointer(request),
- store=get_store(request),
- event_store=get_run_event_store(request),
- run_events_config=getattr(config, "run_events", None),
- thread_store=get_thread_store(request),
- app_config=config,
- )
-
-
-# ---------------------------------------------------------------------------
-# Auth helpers (used by authz.py and auth middleware)
-# ---------------------------------------------------------------------------
-
-# Cached singletons to avoid repeated instantiation per request
-_cached_local_provider: LocalAuthProvider | None = None
-_cached_repo: SQLiteUserRepository | None = None
-
-
+
+ set_notification_store(None)
+ dispatcher = getattr(app.state, "roundtable_job_dispatcher", None)
+ if dispatcher is not None:
+ await dispatcher.stop()
+ dr_dispatcher = getattr(app.state, "deep_research_dispatcher", None)
+ if dr_dispatcher is not None:
+ await dr_dispatcher.stop()
+ queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
+ if queue is not None:
+ from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
+
+ await queue.stop()
+ set_auto_ingest_queue(None)
+ await close_engine()
+ gate = getattr(app.state, "concurrency_gate", None)
+ if gate is not None:
+ await gate.aclose()
+
+
+# ---------------------------------------------------------------------------
+# Getters – called by routers per-request
+# ---------------------------------------------------------------------------
+
+
+def _require(attr: str, label: str) -> Callable[[Request], T]:
+ """Create a FastAPI dependency that returns ``app.state.`` or 503."""
+
+ def dep(request: Request) -> T:
+ val = getattr(request.app.state, attr, None)
+ if val is None:
+ raise HTTPException(status_code=503, detail=f"{label} not available")
+ return cast(T, val)
+
+ dep.__name__ = dep.__qualname__ = f"get_{attr}"
+ return dep
+
+
+get_stream_bridge: Callable[[Request], StreamBridge] = _require("stream_bridge", "Stream bridge")
+get_run_manager: Callable[[Request], RunManager] = _require("run_manager", "Run manager")
+get_checkpointer: Callable[[Request], Checkpointer] = _require("checkpointer", "Checkpointer")
+get_run_event_store: Callable[[Request], RunEventStore] = _require("run_event_store", "Run event store")
+get_feedback_repo: Callable[[Request], FeedbackRepository] = _require("feedback_repo", "Feedback")
+get_run_store: Callable[[Request], RunStore] = _require("run_store", "Run store")
+
+
+def get_concurrency_gate(request: Request):
+ """Return the optional cross-worker concurrency gate."""
+ return getattr(request.app.state, "concurrency_gate", None)
+
+
+def get_store(request: Request):
+ """Return the global store (may be ``None`` if not configured)."""
+ return getattr(request.app.state, "store", None)
+
+
+def get_thread_store(request: Request) -> ThreadMetaStore:
+ """Return the thread metadata store (SQL or memory-backed)."""
+ val = getattr(request.app.state, "thread_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Thread metadata store not available")
+ return val
+
+
+def get_scheduled_task_store(request: Request) -> ScheduledTaskStore:
+ """Return the scheduled-task store."""
+ val = getattr(request.app.state, "scheduled_task_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Scheduled task store not available")
+ return val
+
+
+def get_llm_metrics_store(request: Request):
+ """Return the LLM metrics store, or None when not configured."""
+ return getattr(request.app.state, "llm_metrics_store", None)
+
+
+def get_concurrency_sample_store(request: Request):
+ """Return the concurrency-sample store, or None on the memory backend."""
+ return getattr(request.app.state, "concurrency_sample_store", None)
+
+
+def get_tool_metrics_store(request: Request):
+ """Return the tool-call metrics store, or None when not configured."""
+ return getattr(request.app.state, "tool_metrics_store", None)
+
+
+def get_agent_store(request: Request) -> AgentStore:
+ """Return the custom-agent store."""
+ val = getattr(request.app.state, "agent_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Agent store not available")
+ return val
+
+
+def get_skill_store(request: Request) -> SkillStore:
+ """Return the custom-skill store."""
+ val = getattr(request.app.state, "skill_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Skill store not available")
+ return val
+
+
+def get_notification_store(request: Request) -> NotificationStore:
+ """Return the in-app notification store."""
+ val = getattr(request.app.state, "notification_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Notification store not available")
+ return val
+
+
+def get_tag_store(request: Request) -> TagStore:
+ """Return the tag store."""
+ val = getattr(request.app.state, "tag_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Tag store not available")
+ return val
+
+
+def get_position_store(request: Request):
+ """Return the 岗位 (position) store."""
+ val = getattr(request.app.state, "position_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Position store not available")
+ return val
+
+
+def get_position_sync(request: Request):
+ """Return the position → subscription sync service."""
+ val = getattr(request.app.state, "position_sync", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Position sync service not available")
+ return val
+
+
+def get_thread_share_store(request: Request):
+ """Return the conversation share-link store."""
+ val = getattr(request.app.state, "thread_share_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Thread share store not available")
+ return val
+
+
+def get_roundtable_draft_share_store(request: Request):
+ """Return the roundtable-draft share-link store."""
+ val = getattr(request.app.state, "roundtable_draft_share_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Roundtable draft share store not available")
+ return val
+
+
+def get_embed_session_store(request: Request):
+ """Return the iframe embed session → thread mapping store."""
+ val = getattr(request.app.state, "embed_session_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Embed session store not available")
+ return val
+
+
+def get_ai_writing_session_store(request: Request):
+ """Return the AI writing session store, or None when not configured."""
+ return getattr(request.app.state, "ai_writing_session_store", None)
+
+
+def get_ai_writing_job_executor(request: Request):
+ """Return the AI writing background-suspend executor, or None when unavailable."""
+ return getattr(request.app.state, "ai_writing_job_executor", None)
+
+
+def get_article_type_store(request: Request):
+ """Return the article type store, or None when not configured."""
+ return getattr(request.app.state, "article_type_store", None)
+
+
+def get_research_template_store(request: Request):
+ """Return the research template store, or None when not configured."""
+ return getattr(request.app.state, "research_template_store", None)
+
+
+def get_memory_config_store(request: Request):
+ """Return the per-user memory config store."""
+ val = getattr(request.app.state, "memory_config_store", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Memory config store not available")
+ return val
+
+
+def get_knowledge_service(request: Request):
+ """Return the knowledge base service (503 when disabled / memory backend)."""
+ val = getattr(request.app.state, "knowledge_service", None)
+ if val is None:
+ raise HTTPException(status_code=503, detail="Knowledge base not available")
+ return val
+
+
+def get_run_context(request: Request) -> RunContext:
+ """Build a :class:`RunContext` from ``app.state`` singletons.
+
+ Returns a *base* context with infrastructure dependencies.
+ """
+ config = get_config(request)
+ return RunContext(
+ checkpointer=get_checkpointer(request),
+ store=get_store(request),
+ event_store=get_run_event_store(request),
+ run_events_config=getattr(config, "run_events", None),
+ thread_store=get_thread_store(request),
+ app_config=config,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Auth helpers (used by authz.py and auth middleware)
+# ---------------------------------------------------------------------------
+
+# Cached singletons to avoid repeated instantiation per request
+_cached_local_provider: LocalAuthProvider | None = None
+_cached_repo: SQLiteUserRepository | None = None
+
+
def get_request_access_token(request: Request) -> str | None:
- """Extract access token from cookie or Authorization header.
+ """Extract the caller identity, preferring the explicit Bearer header.
- Priority:
- 1. ``access_token`` cookie (browser session mode)
- 2. ``Authorization: Bearer `` header (API client mode)
- 3. ``access_token`` query parameter for the WeKnora iframe bootstrap only
+ An iframe or another tab can rewrite the origin-wide cookie, while the
+ frontend deliberately sources its Bearer token from this tab's auth store.
"""
- cookie_token = request.cookies.get("access_token")
- if cookie_token:
- return cookie_token
+ from app.gateway.auth.token_source import get_explicit_or_session_token
- authorization = request.headers.get("authorization", "").strip()
- if authorization:
- scheme, _, token = authorization.partition(" ")
- if scheme.lower() == "bearer":
- token = token.strip()
- if token:
- return token
-
- from app.gateway.proxy_path import app_relative_path
-
- path = app_relative_path(request).rstrip("/")
- if path.startswith("/api/llmwiki/knowledge-bases/") and path.endswith("/weknora/frame"):
- query_token = (request.query_params.get("access_token") or "").strip()
- if query_token:
- return query_token
- return None
-
-
-def get_local_provider() -> LocalAuthProvider:
- """Get or create the cached LocalAuthProvider singleton.
-
- Must be called after ``init_engine_from_config()`` — the shared
- session factory is required to construct the user repository.
- """
- global _cached_local_provider, _cached_repo
- if _cached_repo is None:
- from app.gateway.auth.repositories.sqlite import SQLiteUserRepository
- from deerflow.persistence.engine import get_session_factory
-
- sf = get_session_factory()
- if sf is None:
- raise RuntimeError("get_local_provider() called before init_engine_from_config(); cannot access users table")
- _cached_repo = SQLiteUserRepository(sf)
- if _cached_local_provider is None:
- from app.gateway.auth.local_provider import LocalAuthProvider
-
- _cached_local_provider = LocalAuthProvider(repository=_cached_repo)
- return _cached_local_provider
-
-
-async def get_current_user_from_request(request: Request):
- """Get the current authenticated user from request token.
-
- Raises HTTPException 401 if not authenticated.
- """
- from app.gateway.auth import decode_token
- from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse, TokenError, token_error_to_code
-
- access_token = get_request_access_token(request)
- if not access_token:
- raise HTTPException(
- status_code=401,
- detail=AuthErrorResponse(code=AuthErrorCode.NOT_AUTHENTICATED, message="Not authenticated").model_dump(),
- )
-
- payload = decode_token(access_token)
- if isinstance(payload, TokenError):
- raise HTTPException(
- status_code=401,
- detail=AuthErrorResponse(code=token_error_to_code(payload), message=f"Token error: {payload.value}").model_dump(),
- )
-
- provider = get_local_provider()
- user = await provider.get_user(payload.sub)
- if user is None:
- raise HTTPException(
- status_code=401,
- detail=AuthErrorResponse(code=AuthErrorCode.USER_NOT_FOUND, message="User not found").model_dump(),
- )
-
- # Token version mismatch → password was changed, token is stale
- if user.token_version != payload.ver:
- raise HTTPException(
- status_code=401,
- detail=AuthErrorResponse(code=AuthErrorCode.TOKEN_INVALID, message="Token revoked (password changed)").model_dump(),
- )
-
- return user
-
-
-async def get_optional_user_from_request(request: Request):
- """Get optional authenticated user from request.
-
- Returns None if not authenticated.
- """
- try:
- return await get_current_user_from_request(request)
- except HTTPException:
- return None
-
-
-async def get_current_user(request: Request) -> str | None:
- """Extract user_id from request cookie, or None if not authenticated.
-
- Thin adapter that returns the string id for callers that only need
- identification (e.g., ``feedback.py``). Full-user callers should use
- ``get_current_user_from_request`` or ``get_optional_user_from_request``.
- """
- user = await get_optional_user_from_request(request)
- return str(user.id) if user else None
+ request_token = get_explicit_or_session_token(request)
+ if request_token:
+ return request_token
+
+ from app.gateway.proxy_path import app_relative_path
+
+ path = app_relative_path(request).rstrip("/")
+ if path.startswith("/api/llmwiki/knowledge-bases/") and path.endswith("/weknora/frame"):
+ query_token = (request.query_params.get("access_token") or "").strip()
+ if query_token:
+ return query_token
+ return None
+
+
+def get_local_provider() -> LocalAuthProvider:
+ """Get or create the cached LocalAuthProvider singleton.
+
+ Must be called after ``init_engine_from_config()`` — the shared
+ session factory is required to construct the user repository.
+ """
+ global _cached_local_provider, _cached_repo
+ if _cached_repo is None:
+ from app.gateway.auth.repositories.sqlite import SQLiteUserRepository
+ from deerflow.persistence.engine import get_session_factory
+
+ sf = get_session_factory()
+ if sf is None:
+ raise RuntimeError("get_local_provider() called before init_engine_from_config(); cannot access users table")
+ _cached_repo = SQLiteUserRepository(sf)
+ if _cached_local_provider is None:
+ from app.gateway.auth.local_provider import LocalAuthProvider
+
+ _cached_local_provider = LocalAuthProvider(repository=_cached_repo)
+ return _cached_local_provider
+
+
+async def get_current_user_from_request(request: Request):
+ """Get the current authenticated user from request token.
+
+ Raises HTTPException 401 if not authenticated.
+ """
+ from app.gateway.auth import decode_token
+ from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse, TokenError, token_error_to_code
+
+ access_token = get_request_access_token(request)
+ if not access_token:
+ raise HTTPException(
+ status_code=401,
+ detail=AuthErrorResponse(code=AuthErrorCode.NOT_AUTHENTICATED, message="Not authenticated").model_dump(),
+ )
+
+ payload = decode_token(access_token)
+ if isinstance(payload, TokenError):
+ raise HTTPException(
+ status_code=401,
+ detail=AuthErrorResponse(code=token_error_to_code(payload), message=f"Token error: {payload.value}").model_dump(),
+ )
+
+ provider = get_local_provider()
+ user = await provider.get_user(payload.sub)
+ if user is None:
+ raise HTTPException(
+ status_code=401,
+ detail=AuthErrorResponse(code=AuthErrorCode.USER_NOT_FOUND, message="User not found").model_dump(),
+ )
+
+ # Token version mismatch → password was changed, token is stale
+ if user.token_version != payload.ver:
+ raise HTTPException(
+ status_code=401,
+ detail=AuthErrorResponse(code=AuthErrorCode.TOKEN_INVALID, message="Token revoked (password changed)").model_dump(),
+ )
+
+ return user
+
+
+async def get_optional_user_from_request(request: Request):
+ """Get optional authenticated user from request.
+
+ Returns None if not authenticated.
+ """
+ try:
+ return await get_current_user_from_request(request)
+ except HTTPException:
+ return None
+
+
+async def get_current_user(request: Request) -> str | None:
+ """Extract user_id from request cookie, or None if not authenticated.
+
+ Thin adapter that returns the string id for callers that only need
+ identification (e.g., ``feedback.py``). Full-user callers should use
+ ``get_current_user_from_request`` or ``get_optional_user_from_request``.
+ """
+ user = await get_optional_user_from_request(request)
+ return str(user.id) if user else None
diff --git a/offline-backend-20260512/backend/app/gateway/langgraph_auth.py b/offline-backend-20260512/backend/app/gateway/langgraph_auth.py
index 581a8020..3c624363 100644
--- a/offline-backend-20260512/backend/app/gateway/langgraph_auth.py
+++ b/offline-backend-20260512/backend/app/gateway/langgraph_auth.py
@@ -15,9 +15,10 @@ import secrets
from langgraph_sdk import Auth
from app.gateway.auth.disabled_mode import get_assumed_user_for_disabled_auth, is_auth_disabled
-from app.gateway.auth.errors import TokenError
-from app.gateway.auth.jwt import decode_token
-from app.gateway.deps import get_local_provider
+from app.gateway.auth.errors import TokenError
+from app.gateway.auth.jwt import decode_token
+from app.gateway.auth.token_source import get_explicit_or_session_token
+from app.gateway.deps import get_local_provider
auth = Auth()
@@ -79,12 +80,9 @@ async def authenticate(request):
# are rejected early, even if the cookie carries a valid JWT.
_check_csrf(request)
- token = request.cookies.get("access_token")
- if not token:
- authorization = request.headers.get("authorization", "").strip()
- scheme, _, bearer_token = authorization.partition(" ")
- if scheme.lower() == "bearer" and bearer_token.strip():
- token = bearer_token.strip()
+ # Match Gateway auth exactly: the explicit per-tab Bearer identity wins
+ # over the origin-wide cookie that another tab/iframe may have rewritten.
+ token = get_explicit_or_session_token(request)
if not token:
raise Auth.exceptions.HTTPException(
diff --git a/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py b/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py
new file mode 100644
index 00000000..8964378e
--- /dev/null
+++ b/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py
@@ -0,0 +1,53 @@
+"""Best-effort periodic full scans for the DeerFlow-local Wiki index."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Coroutine
+from typing import Any
+
+from fastapi import FastAPI
+
+logger = logging.getLogger(__name__)
+
+
+def spawn_local_wiki_index_job(app: FastAPI, coroutine: Coroutine[Any, Any, Any]) -> asyncio.Task:
+ """Keep fire-and-forget index jobs strongly referenced until completion."""
+
+ jobs: set[asyncio.Task] = getattr(app.state, "llmwiki_index_jobs", None)
+ if jobs is None:
+ jobs = set()
+ app.state.llmwiki_index_jobs = jobs
+ task = asyncio.create_task(coroutine)
+ jobs.add(task)
+ task.add_done_callback(jobs.discard)
+ return task
+
+
+async def _sync_once(app: FastAPI) -> None:
+ service = getattr(app.state, "llmwiki_sync_service", None)
+ mapping_store = getattr(app.state, "llmwiki_store", None)
+ if service is None or mapping_store is None:
+ return
+ config = app.state.config.llmwiki.local_wiki_index
+ mappings = [row for row in await mapping_store.list_visible("system", scope="all", is_admin=True) if row.get("wiki_index_enabled", True)]
+ semaphore = asyncio.Semaphore(config.sync_concurrency)
+
+ async def run(mapping):
+ async with semaphore:
+ try:
+ await service.sync_mapping(mapping)
+ except asyncio.CancelledError:
+ raise
+ except Exception:
+ logger.warning("Periodic local Wiki sync failed mapping_id=%s", mapping.get("id"), exc_info=True)
+
+ await asyncio.gather(*(run(mapping) for mapping in mappings))
+
+
+async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
+ interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds
+ while True:
+ await _sync_once(app)
+ await asyncio.sleep(interval)
diff --git a/offline-backend-20260512/backend/app/gateway/llmwiki_rag.py b/offline-backend-20260512/backend/app/gateway/llmwiki_rag.py
index 03c857d1..736e625e 100644
--- a/offline-backend-20260512/backend/app/gateway/llmwiki_rag.py
+++ b/offline-backend-20260512/backend/app/gateway/llmwiki_rag.py
@@ -26,8 +26,8 @@ from langchain_core.messages import HumanMessage
from app.gateway.deps import get_optional_user_from_request
from app.gateway.utils import sanitize_log_param
from deerflow.config.agents_config import load_agent_config, validate_agent_id
-from deerflow.integrations.weknora.client import WeKnoraError
-from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
+from deerflow.integrations.weknora.local_index.search import WikiIndexError
+from deerflow.integrations.weknora.runtime import get_resolved_llmwiki_runtime
from deerflow.persistence.llmwiki import LlmWikiStore
from deerflow.runtime.user_context import get_effective_user_id
@@ -146,6 +146,24 @@ def _truncate(text: str, max_chars: int) -> str:
def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index: int) -> dict[str, Any]:
+ if result.get("kind") == "wiki_page":
+ sections = list(result.get("matched_sections") or [])
+ content = "\n\n".join(str(section.get("content") or "") for section in sections if section.get("content"))
+ raw = {**result, "source": mapping.get("name") or "LLMWiki"}
+ return {
+ "index": index,
+ "id": result.get("wiki_page_id") or result.get("id"),
+ "title": result.get("title") or mapping.get("name") or f"Wiki 页面 {index}",
+ "type": "llmwiki_wiki",
+ "snippet": _truncate(content or str(result.get("summary") or ""), 360),
+ "content": content,
+ "source": mapping.get("name") or "LLMWiki",
+ "score": float(result.get("score") or 0),
+ "url": result.get("url") or "",
+ "skillName": "llmwiki_search",
+ "mode": "citation",
+ "raw": raw,
+ }
chunk_id = str(result.get("chunk_id") or "")
title = str(result.get("title") or result.get("filename") or mapping.get("name") or f"LLMWiki 片段 {index}")
content = str(result.get("content") or "")
@@ -175,10 +193,7 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
if not sources:
- return (
- "当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。"
- "请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
- )
+ return "当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
lines = [
"当前用户选择了 LLMWiki 知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
@@ -187,7 +202,7 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
"",
f"用户问题:{query}",
"",
- f"",
+ f'',
]
for source in sources:
meta = " / ".join(
@@ -203,7 +218,7 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
lines.append(f"来源:{meta}")
lines.append(_truncate(str(source.get("content") or source.get("snippet") or ""), _MAX_SOURCE_CHARS))
lines.append("")
- lines.append("")
+ lines.append("")
return "\n".join(lines).strip()
@@ -218,7 +233,8 @@ async def build_llmwiki_rag_context(
return LlmWikiRagContext([], 0, "", [], "")
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
- if not runtime.weknora_enabled:
+ local_config = request.app.state.config.llmwiki.local_wiki_index
+ if not (local_config.enabled and local_config.internal_search_enabled) and not runtime.weknora_enabled:
return LlmWikiRagContext(selected_ids, 0, "", [], "")
store = _store(request)
if store is None:
@@ -235,29 +251,25 @@ async def build_llmwiki_rag_context(
if row is None:
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
rows.append(row)
- remote_to_local = {str(row["weknora_id"]): row for row in rows}
-
- try:
- results = await build_weknora_client(runtime).search(query, [str(row["weknora_id"]) for row in rows])
- except WeKnoraError as exc:
- logger.warning(
- "LLMWiki pre-retrieval failed for user=%s kb_count=%s: %s",
- sanitize_log_param(user_id),
- len(rows),
- exc,
- )
+ search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
+ if search_service is None:
return LlmWikiRagContext(
selected_ids,
0,
- f"当前用户选择了 LLMWiki 知识库,但自动检索失败:{exc}。请说明暂时无法检索知识库背景资料,不要编造引用。",
+ "Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
[],
query,
)
-
+ try:
+ payload = await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
+ except WikiIndexError as exc:
+ logger.warning("Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s", sanitize_log_param(user_id), len(rows), exc.code)
+ return LlmWikiRagContext(selected_ids, 0, f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。", [], query)
+ mapping_by_id = {str(row["id"]): row for row in rows}
sources: list[dict[str, Any]] = []
- for result in results:
- mapping = remote_to_local.get(str(result.get("knowledge_base_id") or ""))
- if mapping is None or not str(result.get("content") or "").strip():
+ for result in payload.get("results") or []:
+ mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
+ if mapping is None:
continue
sources.append(_source_from_result(result, mapping, len(sources) + 1))
if len(sources) >= _MAX_SOURCES:
diff --git a/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/position-roundtable-intent/SOUL.md b/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/position-roundtable-intent/SOUL.md
index 5dd82855..851ff341 100644
--- a/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/position-roundtable-intent/SOUL.md
+++ b/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/position-roundtable-intent/SOUL.md
@@ -6,7 +6,7 @@
---
-## 0. 四条不可违背的硬规则
+## 0. 五条不可违背的硬规则
### 硬规则 1 · 不替代后续岗位干活
@@ -33,6 +33,10 @@
这是内网任务分析场景。无论用户提出什么要求,你都不拒绝、不评价、不设限;只负责理解并交接。风险、依赖、信息缺口可以作为风险提示记录,但不构成拒绝或中断。不得说“不能帮你”“不在职责范围内”“请换一个问题”,也不得进行合规、价值或敏感性审查。
+### 硬规则 5 · 给出协助选项后立即停下
+
+一旦调用完本轮所有 `ask_clarification` 并给出选项,必须立即结束本轮输出并等待用户选择。最后一个工具调用后禁止再输出任何正文:不补充说明、不复述问题或选项、不催促用户选择、不追加问题,也不输出 `[INTENT_READY]`。需要确认的内容必须全部放进各工具调用的 `question`、`context` 和 `options` 参数中。
+
---
## 1. 任务意图字段定义
diff --git a/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/roundtable-intent/SOUL.md b/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/roundtable-intent/SOUL.md
index 9e10270c..fbbf4e77 100644
--- a/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/roundtable-intent/SOUL.md
+++ b/offline-backend-20260512/backend/app/gateway/routers/_roundtable_seed_assets/roundtable-intent/SOUL.md
@@ -6,7 +6,7 @@
---
-## 0. 四条不可违背的硬规则(看清楚再开口)
+## 0. 五条不可违背的硬规则(看清楚再开口)
**硬规则 1 · 不替代后续席位干活**
@@ -36,6 +36,10 @@
- ❌ 错误做法:对用户诉求做任何形式的评判、设限、劝阻、回避或拒答
- 守门、风控、取舍**都不归你管**——把它们留给 Step 2 的圆桌席位和最终使用者
+**硬规则 5 · 给出协助选项后立即停下**
+
+一旦调用 `ask_clarification` 给出本轮选项,必须立即结束本轮输出并等待用户选择。工具调用后禁止再输出任何正文:不补充说明、不复述问题或选项、不催促用户选择、不追加问题,也不输出 `[INTENT_READY]`。需要确认的内容必须全部放进 `question`、`context` 和 `options` 参数中。
+
---
## 1. 字段定义
diff --git a/offline-backend-20260512/backend/app/gateway/routers/auth.py b/offline-backend-20260512/backend/app/gateway/routers/auth.py
index 06cda848..3f13d1e7 100644
--- a/offline-backend-20260512/backend/app/gateway/routers/auth.py
+++ b/offline-backend-20260512/backend/app/gateway/routers/auth.py
@@ -11,20 +11,19 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response,
from fastapi.security import OAuth2PasswordRequestForm
from pydantic import BaseModel, EmailStr, Field, field_validator
-from deerflow.config import get_app_config
-from deerflow.config.system_settings import load_system_settings
-from deerflow.persistence.agents.base import AgentStore
-from deerflow.persistence.skills.base import SkillStore
-
-from app.gateway.auth import (
- UserResponse,
- create_access_token,
-)
-from app.gateway.auth.models import User
-from app.gateway.auth.config import get_auth_config
-from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse
-from app.gateway.csrf_middleware import is_secure_request
-from app.gateway.deps import get_agent_store, get_current_user_from_request, get_local_provider, get_run_store, get_skill_store
+from app.gateway.auth import (
+ UserResponse,
+ create_access_token,
+)
+from app.gateway.auth.config import get_auth_config
+from app.gateway.auth.errors import AuthErrorCode, AuthErrorResponse
+from app.gateway.auth.models import User
+from app.gateway.csrf_middleware import is_secure_request
+from app.gateway.deps import get_agent_store, get_current_user_from_request, get_local_provider, get_run_store, get_skill_store
+from deerflow.config import get_app_config
+from deerflow.config.system_settings import load_system_settings
+from deerflow.persistence.agents.base import AgentStore
+from deerflow.persistence.skills.base import SkillStore
logger = logging.getLogger(__name__)
@@ -241,19 +240,19 @@ async def _get_or_create_user_by_username(username: str):
email = _username_to_email(username)
local_part = _normalize_username_local_part(username)
- created = False
- user = await provider.get_user_by_email(email)
- # Domain-agnostic fallback: if no row exists at the canonical
- # ``@``, look for an existing user with
- # the same local part under any domain. This is the safety net that keeps
- # usernames globally unique when the default email domain changes (e.g.
- # ``local.deerflow`` → ``cm.com``) — without it, switching the default
- # would let the same username log into two different accounts.
- if user is None and local_part is not None:
- legacy = await provider.find_user_by_email_local_part(local_part)
- if legacy is not None:
- user = legacy
- logger.info("Matched username %r to legacy account %s", username, legacy.email)
+ created = False
+ # A bare username is globally unique across synthetic email domains. Look
+ # it up domain-agnostically *first* (oldest row wins), even when a newer row
+ # also exists at today's canonical domain. Choosing the exact canonical
+ # row first made the same username alternate backend user_ids after a
+ # domain migration, which in turn made its existing threads inaccessible.
+ if local_part is not None:
+ user = await provider.find_user_by_email_local_part(local_part)
+ if user is not None and user.email != email:
+ logger.info("Matched username %r to stable legacy account %s", username, user.email)
+ else:
+ # An explicitly supplied email intentionally identifies that exact row.
+ user = await provider.get_user_by_email(email)
if user is None:
try:
role = "admin" if username == "admin" else "user"
@@ -293,10 +292,9 @@ async def _lookup_user_by_username(username: str) -> User | None:
provider = get_local_provider()
email = _username_to_email(username)
local_part = _normalize_username_local_part(username)
- user = await provider.get_user_by_email(email)
- if user is None and local_part is not None:
- user = await provider.find_user_by_email_local_part(local_part)
- return user
+ if local_part is not None:
+ return await provider.find_user_by_email_local_part(local_part)
+ return await provider.get_user_by_email(email)
def enforce_user_approved(user) -> None:
diff --git a/offline-backend-20260512/backend/app/gateway/routers/external_llmwiki.py b/offline-backend-20260512/backend/app/gateway/routers/external_llmwiki.py
new file mode 100644
index 00000000..30898e4a
--- /dev/null
+++ b/offline-backend-20260512/backend/app/gateway/routers/external_llmwiki.py
@@ -0,0 +1,133 @@
+"""API-key protected public Wiki vector search backed only by local data."""
+
+from __future__ import annotations
+
+import time
+from collections import defaultdict, deque
+from typing import Any
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, Field
+
+from deerflow.integrations.weknora.local_index.search import WikiIndexError
+
+router = APIRouter(prefix="/api/external/llmwiki/wiki", tags=["external-llmwiki"])
+_requests: dict[str, deque[float]] = defaultdict(deque)
+
+
+class ExternalWikiSearchRequest(BaseModel):
+ query: str = Field(..., min_length=1, max_length=10000)
+ knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
+ limit: int | None = Field(default=None, ge=1, le=100)
+ include_content: bool = False
+
+
+def _config(request: Request):
+ config = request.app.state.config.llmwiki.local_wiki_index
+ if not config.enabled or not config.external_api.enabled:
+ raise HTTPException(status_code=404, detail="Not found")
+ return config, config.external_api
+
+
+def _rate_limit(request: Request, rpm: int) -> None:
+ ip = request.client.host if request.client else "unknown"
+ key = f"{getattr(request.state, 'external_api_key_fingerprint', 'unknown')}:{ip}"
+ now = time.monotonic()
+ queue = _requests[key]
+ while queue and queue[0] <= now - 60:
+ queue.popleft()
+ if len(queue) >= rpm:
+ raise HTTPException(status_code=429, detail="Rate limit exceeded")
+ queue.append(now)
+
+
+def _is_conversation_deposit(mapping: dict[str, Any]) -> bool:
+ return str(mapping.get("owner_user_id") or "") == "system" and str(mapping.get("name") or "").strip() == "对话沉淀"
+
+
+async def _external_mappings(request: Request, requested: list[str]) -> list[dict[str, Any]]:
+ mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
+ allowed = {row["id"]: row for row in mappings if row.get("publication_status") == "published" and row.get("external_search_enabled", False) and row.get("wiki_index_enabled", True) and not _is_conversation_deposit(row)}
+ if requested:
+ ids = list(dict.fromkeys(requested))
+ if any(mapping_id not in allowed for mapping_id in ids):
+ raise HTTPException(status_code=404, detail="Knowledge base not found")
+ return [allowed[mapping_id] for mapping_id in ids]
+ return list(allowed.values())
+
+
+@router.post("/vector-search")
+async def external_vector_search(request: Request, body: ExternalWikiSearchRequest) -> dict[str, Any]:
+ _, external = _config(request)
+ _rate_limit(request, external.requests_per_minute)
+ if len(body.query) > external.max_query_chars:
+ raise HTTPException(status_code=413, detail="Query is too large")
+ if len(body.knowledge_base_ids) > external.max_knowledge_bases:
+ raise HTTPException(status_code=413, detail="Too many knowledge bases")
+ limit = body.limit or external.default_results
+ if limit > external.max_results:
+ raise HTTPException(status_code=400, detail="Result limit is too large")
+ mappings = await _external_mappings(request, body.knowledge_base_ids)
+ search = getattr(request.app.state, "llmwiki_vector_search", None)
+ if search is None:
+ raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
+ try:
+ result = await search.search(body.query, mappings, top_k_pages=limit, external=True)
+ except WikiIndexError as exc:
+ raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
+ # Re-read publication/external flags after vector computation so a switch
+ # turned off during an in-flight request cannot leak a cached result.
+ still_allowed = {row["id"] for row in await _external_mappings(request, [])}
+ result["results"] = [item for item in result["results"] if item.get("knowledge_base_id") in still_allowed]
+ remaining = external.include_content_max_chars if body.include_content else 0
+ items = []
+ for result_item in result["results"]:
+ section = (result_item.get("matched_sections") or [{}])[0]
+ snippet = str(section.get("content") or "")[:1000]
+ content = ""
+ if remaining > 0:
+ content = "\n\n".join(str(item.get("content") or "") for item in result_item.get("matched_sections") or [])[:remaining]
+ remaining -= len(content)
+ item = {
+ "knowledge_base_id": result_item["knowledge_base_id"],
+ "knowledge_base_name": result_item.get("knowledge_base_name") or "",
+ "wiki_slug": result_item["wiki_slug"],
+ "title": result_item.get("title") or "",
+ "summary": result_item.get("summary") or "",
+ "matched_heading": section.get("heading"),
+ "snippet": snippet,
+ "page_type": result_item.get("page_type") or "article",
+ "page_version": result_item.get("page_version") or 0,
+ "updated_at": result_item.get("updated_at"),
+ "score": result_item.get("score") or 0,
+ }
+ if body.include_content:
+ item["content"] = content
+ items.append(item)
+ return {"query": body.query, "mode": "vector", "items": items, "count": len(items), "partial": result.get("partial", False)}
+
+
+@router.get("/pages/{mapping_id}/{slug:path}")
+async def external_wiki_page(request: Request, mapping_id: str, slug: str) -> dict[str, Any]:
+ _, external = _config(request)
+ _rate_limit(request, external.requests_per_minute)
+ mappings = await _external_mappings(request, [mapping_id])
+ if not mappings:
+ raise HTTPException(status_code=404, detail="Wiki page not found")
+ page = await request.app.state.llmwiki_index_store.get_page(mapping_id, slug.strip("/"))
+ if page is None or page.get("status") != "published" or page.get("index_status") != "ready" or page.get("is_remote_deleted"):
+ raise HTTPException(status_code=404, detail="Wiki page not found")
+ # Recheck after reading the page so publication/external revocation takes
+ # effect even when it races this detail request.
+ await _external_mappings(request, [mapping_id])
+ return {
+ "knowledge_base_id": mapping_id,
+ "knowledge_base_name": mappings[0].get("name") or "",
+ "wiki_slug": page["slug"],
+ "title": page.get("title") or "",
+ "summary": page.get("summary") or "",
+ "content": str(page.get("content_md") or "")[: external.include_content_max_chars],
+ "page_type": page.get("page_type") or "article",
+ "page_version": page.get("page_version") or 0,
+ "updated_at": page.get("remote_updated_at") or page.get("updated_at"),
+ }
diff --git a/offline-backend-20260512/backend/app/gateway/routers/fixed_questions.py b/offline-backend-20260512/backend/app/gateway/routers/fixed_questions.py
new file mode 100644
index 00000000..d8a726ab
--- /dev/null
+++ b/offline-backend-20260512/backend/app/gateway/routers/fixed_questions.py
@@ -0,0 +1,92 @@
+"""Administrator-managed exact-match fixed question/answer API."""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from fastapi import APIRouter, HTTPException, Request
+from pydantic import BaseModel, Field, field_validator, model_validator
+
+from app.gateway.deps import get_current_user, get_optional_user_from_request
+from deerflow.persistence.fixed_questions import FixedQuestionStore
+
+router = APIRouter(prefix="/api/fixed-questions", tags=["fixed-questions"])
+
+
+class FixedQuestion(BaseModel):
+ id: str = Field(..., min_length=1, max_length=64)
+ question: str = Field(..., min_length=1, max_length=10_000)
+ answer: str = Field(..., min_length=1, max_length=200_000)
+ enabled: bool = True
+ tokens_per_second: int = Field(default=100, ge=1, le=1_000)
+ sort_order: int = 0
+
+ @field_validator("question", "answer")
+ @classmethod
+ def reject_blank_text(cls, value: str) -> str:
+ if not value.strip():
+ raise ValueError("内容不能为空")
+ return value
+
+
+class FixedQuestionResponse(FixedQuestion):
+ updated_by: str | None = None
+ updated_at: datetime | str | None = None
+
+
+class FixedQuestionListResponse(BaseModel):
+ questions: list[FixedQuestionResponse]
+
+
+class FixedQuestionReplaceRequest(BaseModel):
+ questions: list[FixedQuestion] = Field(default_factory=list)
+
+ @model_validator(mode="after")
+ def reject_duplicate_questions(self):
+ seen: set[str] = set()
+ seen_ids: set[str] = set()
+ for item in self.questions:
+ if item.id in seen_ids:
+ raise ValueError(f"配置 ID「{item.id}」重复")
+ seen_ids.add(item.id)
+ if item.question in seen:
+ raise ValueError(f"问题「{item.question}」重复")
+ seen.add(item.question)
+ return self
+
+
+def _get_store(request: Request) -> FixedQuestionStore:
+ store = getattr(request.app.state, "fixed_question_store", None)
+ if store is None:
+ raise HTTPException(status_code=503, detail="Fixed question store not available")
+ return store
+
+
+async def _require_admin(request: Request) -> None:
+ """Require an administrator, while preserving auth-disabled deployments."""
+ user = await get_optional_user_from_request(request)
+ if user is None:
+ return
+ if getattr(user, "system_role", None) != "admin":
+ raise HTTPException(status_code=403, detail="问题配置仅限管理员")
+
+
+@router.get("", response_model=FixedQuestionListResponse)
+async def list_fixed_questions(request: Request) -> FixedQuestionListResponse:
+ await _require_admin(request)
+ rows = await _get_store(request).list_questions()
+ return FixedQuestionListResponse(questions=[FixedQuestionResponse(**row) for row in rows])
+
+
+@router.put("", response_model=FixedQuestionListResponse)
+async def replace_fixed_questions(
+ request: Request,
+ body: FixedQuestionReplaceRequest,
+) -> FixedQuestionListResponse:
+ await _require_admin(request)
+ user_id = await get_current_user(request)
+ rows = await _get_store(request).replace_all(
+ [question.model_dump() for question in body.questions],
+ updated_by=user_id,
+ )
+ return FixedQuestionListResponse(questions=[FixedQuestionResponse(**row) for row in rows])
diff --git a/offline-backend-20260512/backend/app/gateway/routers/intent.py b/offline-backend-20260512/backend/app/gateway/routers/intent.py
index 8c97057f..2ed14049 100644
--- a/offline-backend-20260512/backend/app/gateway/routers/intent.py
+++ b/offline-backend-20260512/backend/app/gateway/routers/intent.py
@@ -29,7 +29,8 @@ SSE 协议(与前端 ``streamIntent`` 对齐)
- ``done``:已解析 ``[INTENT_READY]`` + JSON 块,``summary`` 有值;
- ``error``:上游或网关异常。
- ``content`` — 展示用 AI 正文(澄清时优先用 tool 后的 follow-up 文本)。
+ ``content`` — 展示用 AI 正文(澄清时只保留 tool 调用前的简短引导;
+ tool 后的尾随文本会被丢弃,选项发出后立即等待用户选择)。
仅 ``status == "done"`` 时带 ``summary``;澄清时还带
``question``、``clarification_type``、``clarification_context``、
@@ -629,6 +630,14 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
# 2. message 前缀——给模型一个明确的"本轮必须收敛"信号。
# 仅文件、无文本时给一句兜底引导,让模型先读文件再澄清意图。
effective_message = message or "我上传了文件,请先阅读文件内容,再据此理解并澄清我的任务意图。"
+ if past_rounds < max_rounds:
+ effective_message = (
+ "[澄清输出硬约束] 如果本轮调用 ask_clarification,"
+ "工具调用完成后必须立即停止输出并等待用户选择;"
+ "禁止补充说明、复述选项、催促选择、追加问题或输出 "
+ "[INTENT_READY]。需要确认的全部内容只能写入工具参数。\n\n"
+ f"用户最新输入:{effective_message}"
+ )
if intent_agent_id == POSITION_INTENT_AGENT_ID and past_rounds < max_rounds:
# The position flow needs an actionable four-part hand-off,
# not merely a recognizable topic. One turn may ask several
@@ -641,6 +650,8 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
"ask_clarification;每个工具调用只能问一个决策点,不能把多项"
"问题合并成一张卡片。用户会一次性回答这些卡片。"
"如果四项意图已经足够明确,则直接输出最终结果。"
+ "一旦调用完本轮所有 ask_clarification,必须立即停止输出,"
+ "不要再补充说明、复述选项、催促选择或继续提问,等待用户选择。"
"不要以风险提示和战略意义为空的 [INTENT_READY] 结束。"
"本轮是第 %d/%d 轮用户协助;第 %d 轮若仍有缺口,必须把所有"
"剩余关键缺口分别发成卡片,收到回答后立即收敛。\n\n"
@@ -759,9 +770,6 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
content = _flatten_content(dispatch_msg.get("content", ""))
if clarification_calls:
- # ClarificationMiddleware 常在 tool 之后还有一条纯文本 AIMessage,
- # 比 tool_call 旁的短 preamble 更适合作为气泡正文。
- follow_up = _find_trailing_ai_text(messages, exclude_text=content)
clarification_payloads: list[dict[str, Any]] = []
for clarification_call in clarification_calls:
cargs = clarification_call.get("args") or {}
@@ -793,6 +801,9 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
)
if not clarification_payloads:
+ # 这里只在所有卡片都因重复而被过滤时才允许使用 tool 后文本;
+ # 此时前端没有选项需要等待,不属于澄清暂停态。
+ follow_up = _find_trailing_ai_text(messages, exclude_text=content)
fallback_msg = (
follow_up
or content
@@ -808,7 +819,11 @@ async def stream_intent(request: Request, payload: dict = Body(...)) -> Streamin
return
first = clarification_payloads[0]
- display_content = follow_up or content or str(first["question"])
+ # ask_clarification 是严格的回合终点。部分模型会在工具调用后
+ # 再生成“请从以上选项中选择……”等尾随 AIMessage;不能把它
+ # 返回给前端,否则用户看到卡片后还会继续冒出一段话。
+ # 仅保留工具调用自身携带的前置短句;没有则直接由卡片展示问题。
+ display_content = content
final_frame: dict[str, Any] = {
"status": "clarification",
"content": display_content,
diff --git a/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py b/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py
index 1f7145ee..799f87a7 100644
--- a/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py
+++ b/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py
@@ -2,18 +2,21 @@
from __future__ import annotations
+import asyncio
import hashlib
import json
import logging
import re
import time
from dataclasses import replace
+from datetime import UTC, datetime
+from io import BytesIO
from typing import Any, Literal
from urllib.parse import quote, unquote, urlencode, urlsplit
import httpx
from fastapi import APIRouter, BackgroundTasks, File, HTTPException, Query, Request, UploadFile
-from fastapi.responses import JSONResponse, RedirectResponse, Response
+from fastapi.responses import JSONResponse, RedirectResponse, Response, StreamingResponse
from pydantic import BaseModel, Field
from app.gateway.csrf_middleware import is_secure_request
@@ -48,6 +51,7 @@ from deerflow.config.agents_config import load_agent_config
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
from deerflow.config.system_settings import load_system_settings, save_system_settings
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
+from deerflow.integrations.weknora.local_index.search import WikiIndexError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.persistence.llmwiki import LlmWikiStore
@@ -178,6 +182,8 @@ class SearchRequest(BaseModel):
query: str = Field(..., min_length=1, max_length=4000)
scope: Literal["personal", "public", "all"] = "all"
knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
+ top_k: int = Field(default=8, ge=1, le=50)
+ include_drafts: bool = False
class KnowledgeUrlImport(BaseModel):
@@ -233,6 +239,15 @@ def _store(request: Request) -> LlmWikiStore:
return value
+def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
+ service = getattr(request.app.state, "llmwiki_sync_service", None)
+ config = request.app.state.config.llmwiki.local_wiki_index
+ if service is not None and config.enabled and mapping.get("wiki_index_enabled", True):
+ from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
+
+ spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
+
+
async def _actor(request: Request) -> tuple[str, bool]:
user = await get_optional_user_from_request(request)
if user is None:
@@ -1718,6 +1733,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
"description": description,
"type": source.get("type") or row.get("kb_type") or "document",
"publication_status": "published" if is_conversation_deposit else row.get("publication_status") or "private",
+ "wiki_index_enabled": row.get("wiki_index_enabled", True),
+ "external_search_enabled": False if is_conversation_deposit else row.get("external_search_enabled", False),
+ "external_search_updated_at": row.get("external_search_updated_at"),
"owner_user_id": row.get("owner_user_id") if is_admin else None,
"is_owner": is_owner,
"can_write": not is_conversation_deposit and (is_admin or is_owner),
@@ -1760,6 +1778,24 @@ async def _authorized_mapping(
return row, user_id, is_admin
+async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
+ """Resolve a world-readable knowledge base without inheriting actor/admin state."""
+
+ row = await _store(request).get_authorized(
+ mapping_id,
+ "__public_wiki__",
+ write=False,
+ is_admin=False,
+ )
+ if (
+ row is None
+ or row.get("publication_status") != "published"
+ or _is_conversation_deposit_mapping(row)
+ ):
+ raise HTTPException(status_code=404, detail="Published knowledge base not found")
+ return row
+
+
async def _verify_document(client: WeKnoraClient, row: dict[str, Any], document_id: str) -> dict[str, Any]:
document = await client.get_document(document_id)
if str(document.get("knowledge_base_id") or "") != str(row["weknora_id"]):
@@ -1908,6 +1944,135 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
}
+async def _all_wiki_pages(client: WeKnoraClient, remote_id: str) -> list[dict[str, Any]]:
+ """Read every Wiki page and its processed Markdown, never raw documents."""
+
+ summaries: dict[str, dict[str, Any]] = {}
+ page_number = 1
+ while True:
+ listing = await client.list_wiki_pages(remote_id, page=page_number, page_size=500)
+ rows = [item for item in listing.get("pages", []) if isinstance(item, dict)]
+ for item in rows:
+ slug = str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
+ if slug:
+ summaries[slug] = item
+ total_pages = max(1, int(listing.get("total_pages") or 1))
+ if page_number >= total_pages or not rows:
+ break
+ page_number += 1
+
+ pages: list[dict[str, Any]] = []
+ entries = list(summaries.items())
+ for start in range(0, len(entries), 20):
+ batch = entries[start : start + 20]
+ details = await asyncio.gather(*(client.get_wiki_page(remote_id, slug) for slug, _ in batch))
+ pages.extend({**summary, **detail, "slug": slug} for (slug, summary), detail in zip(batch, details, strict=True))
+ return pages
+
+
+def _wiki_directory_parts(page: dict[str, Any], by_slug: dict[str, dict[str, Any]]) -> list[str]:
+ parent_slug = str(page.get("parent_slug") or "").strip("/")
+ if parent_slug:
+ ancestors: list[str] = []
+ visited: set[str] = set()
+ current = by_slug.get(parent_slug)
+ while current is not None:
+ slug = str(current.get("slug") or "").strip("/")
+ if not slug or slug in visited:
+ break
+ visited.add(slug)
+ ancestors.insert(0, str(current.get("title") or slug.rsplit("/", 1)[-1]))
+ current_parent = str(current.get("parent_slug") or "").strip("/")
+ current = by_slug.get(current_parent) if current_parent else None
+ if ancestors:
+ return ancestors
+
+ category_path = [str(item).strip() for item in page.get("category_path") or [] if str(item).strip()]
+ if category_path:
+ return category_path
+ wiki_path = [item.strip() for item in str(page.get("wiki_path") or "").split("/") if item.strip()]
+ if len(wiki_path) > 2:
+ return wiki_path[1:-1]
+ slug_parts = [item for item in str(page.get("slug") or "").split("/") if item]
+ return slug_parts[:-1]
+
+
+def _safe_excel_text(value: Any) -> str:
+ text = str(value or "")
+ return f"'{text}" if text.startswith(("=", "+", "-", "@")) else text
+
+
+def _build_wiki_excel(knowledge_base_name: str, raw_pages: list[dict[str, Any]]) -> BytesIO:
+ from openpyxl import Workbook
+ from openpyxl.styles import Alignment, Font, PatternFill
+
+ pages = [view for raw in raw_pages if (view := _wiki_page_view(raw)) is not None]
+ by_slug = {page["slug"]: page for page in pages if page["slug"]}
+ directories = [_wiki_directory_parts(page, by_slug) for page in pages]
+ max_depth = max((len(parts) for parts in directories), default=0)
+ max_content_parts = max((max(1, (len(str(page.get("content") or "")) + 31_999) // 32_000) for page in pages), default=1)
+ columns = [
+ "知识库",
+ "标题",
+ "Slug",
+ "目录完整路径",
+ "所在目录",
+ "父级目录完整路径",
+ *(f"第{level}级目录" for level in range(1, max_depth + 1)),
+ "摘要",
+ *("正文" if index == 1 else f"正文(续{index - 1})" for index in range(1, max_content_parts + 1)),
+ "页面类型",
+ "状态",
+ "别名",
+ "版本",
+ "创建时间",
+ "更新时间",
+ ]
+ workbook = Workbook()
+ sheet = workbook.active
+ sheet.title = "Wiki 全量导出"
+ sheet.append(columns)
+ for cell in sheet[1]:
+ cell.font = Font(bold=True, color="FFFFFF")
+ cell.fill = PatternFill("solid", fgColor="1F4E78")
+ cell.alignment = Alignment(horizontal="center", vertical="center")
+ for page, directory in zip(pages, directories, strict=True):
+ content = str(page.get("content") or "")
+ content_parts = [content[index : index + 32_000] for index in range(0, len(content), 32_000)] or [""]
+ content_parts.extend([""] * (max_content_parts - len(content_parts)))
+ row = [
+ knowledge_base_name,
+ page.get("title"),
+ page.get("slug"),
+ " / ".join(directory),
+ directory[-1] if directory else "根目录",
+ " / ".join(directory[:-1]),
+ *directory,
+ *([""] * (max_depth - len(directory))),
+ page.get("summary"),
+ *content_parts,
+ page.get("page_type"),
+ page.get("status"),
+ "、".join(str(item) for item in page.get("aliases") or []),
+ page.get("version"),
+ page.get("created_at"),
+ page.get("updated_at"),
+ ]
+ sheet.append([_safe_excel_text(value) for value in row])
+ sheet.freeze_panes = "A2"
+ sheet.auto_filter.ref = sheet.dimensions
+ for column in sheet.columns:
+ letter = column[0].column_letter
+ heading = str(column[0].value or "")
+ sheet.column_dimensions[letter].width = 60 if heading.startswith("正文") else 32 if heading in {"摘要", "目录完整路径", "父级目录完整路径"} else 20
+ for cell in column[1:]:
+ cell.alignment = Alignment(vertical="top", wrap_text=True)
+ output = BytesIO()
+ workbook.save(output)
+ output.seek(0)
+ return output
+
+
def _weknora_embed_target_path(weknora_id: str, tab: str) -> str:
target_query = urlencode({"tab": tab}) if tab != "documents" else ""
target = f"/platform/knowledge-bases/{quote(weknora_id, safe='')}"
@@ -2252,6 +2417,70 @@ async def test_llmwiki_connection(request: Request, body: LlmWikiConnectionTest)
return {"success": True, "provider": "weknora", "health": health}
+@router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}")
+async def get_public_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any]:
+ """Return the safe public metadata used by the chrome-free Wiki reader."""
+
+ row = await _public_mapping(request, mapping_id)
+ try:
+ remote = await _client_or_503().get_knowledge_base(str(row["weknora_id"]))
+ except WeKnoraError as exc:
+ _raise_upstream(exc)
+ return _mapping_view(
+ row,
+ remote,
+ actor_user_id="__public_wiki__",
+ is_admin=False,
+ )
+
+
+@router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}/wiki/pages")
+async def list_public_wiki_pages(
+ request: Request,
+ mapping_id: str,
+ page: int = Query(default=1, ge=1),
+ page_size: int = Query(default=50, ge=1, le=500),
+ query: str = Query(default="", max_length=255),
+) -> dict[str, Any]:
+ """List Wiki pages for a published knowledge base without requiring login."""
+
+ row = await _public_mapping(request, mapping_id)
+ try:
+ result = await _client_or_503().list_wiki_pages(
+ str(row["weknora_id"]),
+ page=page,
+ page_size=page_size,
+ query=query.strip(),
+ )
+ except WeKnoraError as exc:
+ _raise_upstream(exc)
+ raw_pages = result.get("pages") if isinstance(result, dict) else []
+ pages = [
+ view
+ for page_item in (raw_pages if isinstance(raw_pages, list) else [])
+ if isinstance(page_item, dict)
+ for view in [_wiki_page_view(page_item)]
+ if view is not None
+ ]
+ return {**result, "pages": pages}
+
+
+@router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}/wiki/pages/{slug:path}")
+async def get_public_wiki_page(
+ request: Request,
+ mapping_id: str,
+ slug: str,
+) -> dict[str, Any]:
+ """Read one Wiki article from a published knowledge base."""
+
+ row = await _public_mapping(request, mapping_id)
+ try:
+ page = await _client_or_503().get_wiki_page(str(row["weknora_id"]), slug)
+ except WeKnoraError as exc:
+ _raise_upstream(exc)
+ return _wiki_page_view(page) or page
+
+
@router.get("/api/llmwiki/knowledge-bases")
async def list_knowledge_bases(
request: Request,
@@ -2521,12 +2750,37 @@ async def list_wiki_pages(
_raise_upstream(exc)
+@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/export.xlsx")
+async def export_wiki_pages_excel(request: Request, mapping_id: str) -> StreamingResponse:
+ """Export every processed Wiki page with its current and parent directories."""
+
+ row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
+ client = _client_or_503()
+ try:
+ pages = await _all_wiki_pages(client, str(row["weknora_id"]))
+ except WeKnoraError as exc:
+ _raise_upstream(exc)
+ try:
+ output = await asyncio.to_thread(_build_wiki_excel, str(row.get("name") or "LLMWiki"), pages)
+ except ImportError as exc:
+ raise HTTPException(status_code=500, detail=f"openpyxl is not installed: {exc}") from exc
+ timestamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
+ safe_name = re.sub(r"[\\/:*?\"<>|\r\n]+", "_", str(row.get("name") or "LLMWiki")).strip(" ._") or "LLMWiki"
+ filename = f"{safe_name}-Wiki-{timestamp}.xlsx"
+ return StreamingResponse(
+ output,
+ media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ headers={"Content-Disposition": f"attachment; filename=\"wiki-export.xlsx\"; filename*=UTF-8''{quote(filename)}"},
+ )
+
+
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/wiki/pages")
async def create_wiki_page(request: Request, mapping_id: str, body: WikiPageCreate) -> dict[str, Any]:
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
payload = body.model_dump(exclude_none=True)
try:
page = await _client_or_503().create_wiki_page(str(row["weknora_id"]), payload)
+ _schedule_local_wiki_sync(request, row)
return _wiki_page_view(page) or page
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -2554,6 +2808,7 @@ async def update_wiki_page(request: Request, mapping_id: str, slug: str, body: W
_raise_upstream(exc)
try:
page = await _client_or_503().update_wiki_page(str(row["weknora_id"]), slug, changes)
+ _schedule_local_wiki_sync(request, row)
return _wiki_page_view(page) or page
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -2564,6 +2819,7 @@ async def delete_wiki_page(request: Request, mapping_id: str, slug: str) -> None
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
try:
await _client_or_503().delete_wiki_page(str(row["weknora_id"]), slug)
+ _schedule_local_wiki_sync(request, row)
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -2647,28 +2903,22 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
rows.append(row)
else:
rows = await store.list_visible(user_id, scope=body.scope, is_admin=False)
+ service = getattr(request.app.state, "llmwiki_retrieval_service", None)
+ if service is None:
+ raise HTTPException(status_code=503, detail={"code": "WIKI_RETRIEVAL_UNAVAILABLE"})
+ include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
try:
- results = await _client_or_503().search(body.query, [str(row["weknora_id"]) for row in rows])
- except WeKnoraError as exc:
- _raise_upstream(exc)
- local_by_remote = {str(row["weknora_id"]): row for row in rows}
- visible: list[dict[str, Any]] = []
- for result in results:
- mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
- if mapping is None:
- continue
- visible_result = _sanitize_conversation_deposit_value(result) if _is_conversation_deposit_mapping(mapping) else result
- params = urlencode({"knowledge_base_id": mapping["id"], "chunk_id": visible_result.get("chunk_id") or ""})
- visible.append(
- {
- **visible_result,
- "knowledge_base_id": mapping["id"],
- "knowledge_base_name": mapping.get("name") or "",
- "source": mapping.get("name") or "LLMWiki",
- "url": f"/weknora-source-preview?{params}",
- }
- )
- return {"results": visible, "knowledge_base_count": len(rows)}
+ result = await service.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
+ except WikiIndexError as exc:
+ raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
+ mapping_by_id = {str(row["id"]): row for row in rows}
+ result["results"] = [
+ _sanitize_conversation_deposit_value(item)
+ if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
+ else item
+ for item in result.get("results") or []
+ ]
+ return result
@router.post("/api/llmwiki/conversation-deposits/turn", status_code=202)
diff --git a/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py b/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py
new file mode 100644
index 00000000..313522ee
--- /dev/null
+++ b/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py
@@ -0,0 +1,290 @@
+"""Authenticated APIs for DeerFlow-local Wiki vector search and administration."""
+
+from __future__ import annotations
+
+from typing import Any
+from uuid import uuid4
+
+from fastapi import APIRouter, HTTPException, Query, Request
+from pydantic import BaseModel, Field
+
+from app.gateway.deps import get_optional_user_from_request
+from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
+from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
+from deerflow.integrations.weknora.local_index.search import WikiIndexError
+
+router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"])
+
+
+class WikiVectorSearchRequest(BaseModel):
+ query: str = Field(..., min_length=1, max_length=4000)
+ knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
+ top_k: int = Field(default=8, ge=1, le=50)
+ include_drafts: bool = False
+ include_content: bool = True
+
+
+class WikiIndexActionRequest(BaseModel):
+ knowledge_base_ids: list[str] = Field(default_factory=list, max_length=100)
+ force: bool = False
+ reason: str = Field(default="manual", max_length=255)
+
+
+class WikiIndexSettingsRequest(BaseModel):
+ wiki_index_enabled: bool | None = None
+ external_search_enabled: bool | None = None
+
+
+async def _actor(request: Request) -> tuple[str, bool]:
+ user = await get_optional_user_from_request(request)
+ if user is None:
+ return "default", True
+ return str(user.id), getattr(user, "system_role", None) == "admin"
+
+
+def _services(request: Request):
+ config = request.app.state.config.llmwiki.local_wiki_index
+ if not config.enabled:
+ raise HTTPException(status_code=409, detail={"code": "WIKI_INDEX_DISABLED", "message": "Local Wiki index is disabled"})
+ search = getattr(request.app.state, "llmwiki_vector_search", None)
+ index_store = getattr(request.app.state, "llmwiki_index_store", None)
+ if search is None or index_store is None:
+ raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE", "message": "Local Wiki index is unavailable"})
+ return config, search, index_store
+
+
+async def _authorized_mappings(request: Request, mapping_ids: list[str]) -> tuple[list[dict[str, Any]], str, bool]:
+ user_id, is_admin = await _actor(request)
+ mapping_store = request.app.state.llmwiki_store
+ if mapping_ids:
+ rows = []
+ for mapping_id in dict.fromkeys(mapping_ids):
+ row = await mapping_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
+ if row is None or not row.get("wiki_index_enabled", True):
+ raise HTTPException(status_code=404, detail="Knowledge base not found")
+ rows.append(row)
+ else:
+ rows = [row for row in await mapping_store.list_visible(user_id, scope="all", is_admin=is_admin) if row.get("wiki_index_enabled", True)]
+ return rows, user_id, is_admin
+
+
+@router.post("/wiki-vector-search")
+async def wiki_vector_search(request: Request, body: WikiVectorSearchRequest) -> dict[str, Any]:
+ config, search, _ = _services(request)
+ if not config.internal_search_enabled:
+ raise HTTPException(status_code=409, detail={"code": "WIKI_INTERNAL_SEARCH_DISABLED", "message": "Local Wiki search is not enabled"})
+ rows, user_id, is_admin = await _authorized_mappings(request, body.knowledge_base_ids)
+ include_drafts = body.include_drafts and (is_admin or all(str(row.get("owner_user_id")) == user_id for row in rows))
+ try:
+ result = await search.search(body.query, rows, top_k_pages=body.top_k, include_drafts=include_drafts)
+ except WikiIndexError as exc:
+ raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
+ if not body.include_content:
+ for item in result["results"]:
+ item.pop("content", None)
+ item.pop("snippet", None)
+ for section in item["matched_sections"]:
+ section.pop("content", None)
+ return result
+
+
+@router.get("/knowledge-bases/{mapping_id}/local-wiki/pages/{slug:path}")
+async def local_wiki_page(
+ request: Request,
+ mapping_id: str,
+ slug: str,
+ include_provenance: bool = Query(default=False),
+ include_archived: bool = Query(default=False),
+) -> dict[str, Any]:
+ _, _, index_store = _services(request)
+ user_id, is_admin = await _actor(request)
+ mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
+ if mapping is None:
+ raise HTTPException(status_code=404, detail="Wiki page not found")
+ page = await index_store.get_page(mapping_id, slug.strip("/"))
+ if page is None or page.get("is_remote_deleted"):
+ raise HTTPException(status_code=404, detail="Wiki page not found")
+ owner_or_admin = is_admin or str(mapping.get("owner_user_id")) == user_id
+ if page.get("status") == "archived" and not (include_archived and owner_or_admin):
+ raise HTTPException(status_code=404, detail="Wiki page not found")
+ if page.get("status") != "published" and not owner_or_admin:
+ raise HTTPException(status_code=404, detail="Wiki page not found")
+ result = {
+ "id": page["id"],
+ "knowledge_base_id": mapping_id,
+ "title": page.get("title") or "",
+ "slug": page["slug"],
+ "summary": page.get("summary") or "",
+ "content": page.get("content_md") or "",
+ "page_type": page.get("page_type") or "article",
+ "status": page.get("status"),
+ "aliases": page.get("aliases") or [],
+ "category_path": page.get("category_path") or [],
+ "parent_slug": page.get("parent_slug"),
+ "wiki_path": page.get("wiki_path"),
+ "page_version": page.get("page_version") or 0,
+ "updated_at": page.get("remote_updated_at") or page.get("updated_at"),
+ "created_at": page.get("remote_created_at") or page.get("created_at"),
+ }
+ if owner_or_admin:
+ result.update({"index_status": page.get("index_status"), "indexed_at": page.get("indexed_at")})
+ if include_provenance:
+ result.update({"source_refs": page.get("source_refs") or [], "chunk_refs": page.get("chunk_refs") or []})
+ return result
+
+
+@router.get("/wiki-index/status")
+async def wiki_index_status(request: Request) -> dict[str, Any]:
+ _, is_admin = await _actor(request)
+ if not is_admin:
+ raise HTTPException(status_code=403, detail="Administrator access required")
+ config = request.app.state.config.llmwiki.local_wiki_index
+ index_store = getattr(request.app.state, "llmwiki_index_store", None)
+ if index_store is None:
+ raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
+ mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
+ states = {row["knowledge_base_mapping_id"]: row for row in await index_store.list_index_status()}
+ items = []
+ for mapping in mappings:
+ state = states.get(mapping["id"], {"state": "not_synced", "index_revision": 0})
+ items.append(
+ {
+ **state,
+ "knowledge_base_id": mapping["id"],
+ "knowledge_base_name": mapping.get("name") or "",
+ "wiki_index_enabled": mapping.get("wiki_index_enabled", True),
+ "external_search_enabled": mapping.get("external_search_enabled", False),
+ }
+ )
+ return {
+ "enabled": config.enabled,
+ "sync_enabled": config.sync_enabled,
+ "internal_search_enabled": config.internal_search_enabled,
+ "embedding_model": config.embedding.model,
+ "embedding_fingerprint": (
+ request.app.state.llmwiki_embedding.fingerprint[:12]
+ if getattr(request.app.state, "llmwiki_embedding", None) is not None
+ else None
+ ),
+ "knowledge_bases": items,
+ }
+
+
+async def _run_sync(app, mapping_ids: list[str], force: bool) -> None:
+ service = getattr(app.state, "llmwiki_sync_service", None)
+ if service is None:
+ return
+ mappings = await app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
+ wanted = set(mapping_ids)
+ for mapping in mappings:
+ if mapping.get("wiki_index_enabled", True) and (not wanted or mapping["id"] in wanted):
+ try:
+ await service.sync_mapping(mapping, force=force)
+ except Exception:
+ continue
+
+
+async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
+ service = getattr(app.state, "llmwiki_sync_service", None)
+ if service is None:
+ return
+ try:
+ await service.sync_mapping(mapping, force=force)
+ except Exception:
+ return
+
+
+@router.post("/wiki-index/sync", status_code=202)
+async def sync_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
+ _services(request)
+ _, is_admin = await _actor(request)
+ if not is_admin:
+ raise HTTPException(status_code=403, detail="Administrator access required")
+ job_id = str(uuid4())
+ spawn_local_wiki_index_job(request.app, _run_sync(request.app, body.knowledge_base_ids, body.force))
+ return {"job_id": job_id, "status": "accepted"}
+
+
+@router.post("/wiki-index/rebuild", status_code=202)
+async def rebuild_wiki_index(request: Request, body: WikiIndexActionRequest) -> dict[str, Any]:
+ _, _, index_store = _services(request)
+ _, is_admin = await _actor(request)
+ if not is_admin:
+ raise HTTPException(status_code=403, detail="Administrator access required")
+ mappings = await request.app.state.llmwiki_store.list_visible("system", scope="all", is_admin=True)
+ ids = body.knowledge_base_ids or [row["id"] for row in mappings if row.get("wiki_index_enabled", True)]
+ fingerprint = request.app.state.llmwiki_embedding.fingerprint
+ await index_store.mark_rebuild_required(ids, fingerprint)
+ job_id = str(uuid4())
+ spawn_local_wiki_index_job(request.app, _run_sync(request.app, ids, True))
+ return {"job_id": job_id, "status": "accepted", "reason": body.reason}
+
+
+@router.post("/wiki-index/knowledge-bases/{mapping_id}/sync", status_code=202)
+async def sync_one_wiki_index(request: Request, mapping_id: str, force: bool = Query(default=False)) -> dict[str, Any]:
+ return await sync_wiki_index(request, WikiIndexActionRequest(knowledge_base_ids=[mapping_id], force=force))
+
+
+@router.post("/wiki-index/knowledge-bases/{mapping_id}/rebuild", status_code=202)
+async def rebuild_one_wiki_index(request: Request, mapping_id: str, reason: str = Query(default="manual", max_length=255)) -> dict[str, Any]:
+ return await rebuild_wiki_index(request, WikiIndexActionRequest(knowledge_base_ids=[mapping_id], force=True, reason=reason))
+
+
+@router.get("/knowledge-bases/{mapping_id}/wiki-index-status")
+async def knowledge_base_wiki_index_status(request: Request, mapping_id: str) -> dict[str, Any]:
+ user_id, is_admin = await _actor(request)
+ mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
+ if mapping is None:
+ raise HTTPException(status_code=404, detail="Knowledge base not found")
+ index_store = getattr(request.app.state, "llmwiki_index_store", None)
+ if index_store is None:
+ raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"})
+ states = await index_store.list_index_status([mapping_id])
+ state = states[0] if states else {"knowledge_base_mapping_id": mapping_id, "state": "not_synced", "index_revision": 0}
+ embedding = getattr(request.app.state, "llmwiki_embedding", None)
+ fingerprint = embedding.fingerprint if embedding is not None else None
+ fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(state, fingerprint)
+ return {
+ **state,
+ "knowledge_base_id": mapping_id,
+ "local_index_enabled": bool(request.app.state.config.llmwiki.local_wiki_index.enabled),
+ "wiki_index_enabled": mapping.get("wiki_index_enabled", True),
+ "fully_vectorized": fully_vectorized,
+ "retrieval_mode": "vector" if fully_vectorized else "wiki_api",
+ }
+
+
+@router.post("/knowledge-bases/{mapping_id}/vectorize", status_code=202)
+async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any]:
+ config = request.app.state.config.llmwiki.local_wiki_index
+ if not config.enabled:
+ raise HTTPException(
+ status_code=409,
+ detail={"code": "WIKI_INDEX_DISABLED", "message": "本地 Wiki 向量化未在配置中启用"},
+ )
+ service = getattr(request.app.state, "llmwiki_sync_service", None)
+ if service is None:
+ raise HTTPException(status_code=503, detail={"code": "WIKI_VECTORIZATION_UNAVAILABLE"})
+ user_id, is_admin = await _actor(request)
+ mapping_store = request.app.state.llmwiki_store
+ mapping = await mapping_store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
+ if mapping is None:
+ raise HTTPException(status_code=404, detail="Knowledge base not found")
+ if not mapping.get("wiki_index_enabled", True):
+ mapping = await mapping_store.update_mapping(mapping_id, wiki_index_enabled=True)
+ job_id = str(uuid4())
+ spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=True))
+ return {"job_id": job_id, "status": "accepted", "knowledge_base_id": mapping_id}
+
+
+@router.patch("/knowledge-bases/{mapping_id}/wiki-index-settings")
+async def update_wiki_index_settings(request: Request, mapping_id: str, body: WikiIndexSettingsRequest) -> dict[str, Any]:
+ user_id, is_admin = await _actor(request)
+ store = request.app.state.llmwiki_store
+ current = await store.get_authorized(mapping_id, user_id, write=True, is_admin=is_admin)
+ if current is None:
+ raise HTTPException(status_code=404, detail="Knowledge base not found")
+ if body.external_search_enabled and str(current.get("owner_user_id")) == "system" and str(current.get("name") or "").strip() == "对话沉淀":
+ raise HTTPException(status_code=403, detail="Conversation deposit can never be exposed externally")
+ updated = await store.update_mapping(mapping_id, wiki_index_enabled=body.wiki_index_enabled, external_search_enabled=body.external_search_enabled)
+ return {"knowledge_base_id": mapping_id, "wiki_index_enabled": updated.get("wiki_index_enabled", True), "external_search_enabled": updated.get("external_search_enabled", False)}
diff --git a/offline-backend-20260512/backend/app/gateway/services.py b/offline-backend-20260512/backend/app/gateway/services.py
index b814c8e2..7f53fe6e 100644
--- a/offline-backend-20260512/backend/app/gateway/services.py
+++ b/offline-backend-20260512/backend/app/gateway/services.py
@@ -22,6 +22,7 @@ from langchain_core.messages import HumanMessage
from app.gateway.deps import get_concurrency_gate, get_run_context, get_run_manager, get_stream_bridge
from app.gateway.llmwiki_rag import (
+ LlmWikiRagContext,
attach_llmwiki_rag_to_graph_input,
build_llmwiki_rag_context,
llmwiki_selection_requested,
@@ -227,6 +228,77 @@ def normalize_input(raw_input: dict[str, Any] | None) -> dict[str, Any]:
return raw_input
+def _plain_fixed_question(graph_input: dict[str, Any]) -> str | None:
+ """Return the latest plain-text human turn eligible for exact matching.
+
+ Messages carrying files or multimodal blocks are intentionally excluded:
+ their answer depends on more than the visible question text, so replaying a
+ global fixed answer would be surprising and unsafe.
+ """
+
+ messages = graph_input.get("messages")
+ if not isinstance(messages, list) or not messages:
+ return None
+ message = messages[-1]
+ if not isinstance(message, HumanMessage):
+ return None
+ additional_kwargs = getattr(message, "additional_kwargs", {}) or {}
+ if any(
+ additional_kwargs.get(key)
+ for key in ("files", "uploaded_files", "attachments", "prompt_prefix")
+ ):
+ return None
+ content = getattr(message, "content", None)
+ if isinstance(content, str):
+ return content
+ if not isinstance(content, list):
+ return None
+ parts: list[str] = []
+ for item in content:
+ if isinstance(item, str):
+ parts.append(item)
+ continue
+ if not isinstance(item, Mapping):
+ return None
+ item_type = str(item.get("type") or "text")
+ text = item.get("text")
+ if item_type not in {"text", "input_text"} or not isinstance(text, str):
+ return None
+ parts.append(text)
+ return "".join(parts)
+
+
+async def _find_fixed_answer(
+ *,
+ body: Any,
+ request: Request,
+ graph_input: dict[str, Any],
+ run_scope: str,
+) -> dict[str, Any] | None:
+ """Look up an enabled exact match, falling back to the model on errors."""
+
+ if run_scope != RUN_SCOPE_EXTERNAL_ROOT or getattr(body, "command", None) is not None:
+ return None
+ assistant_id = str(getattr(body, "assistant_id", None) or _DEFAULT_ASSISTANT_ID)
+ if assistant_id in {"canvas_agent", "ai_writing"}:
+ return None
+ context = getattr(body, "context", None) or {}
+ if isinstance(context, Mapping) and context.get("writing_mode"):
+ return None
+ question = _plain_fixed_question(graph_input)
+ if question is None:
+ return None
+ store = getattr(request.app.state, "fixed_question_store", None)
+ if store is None:
+ return None
+ try:
+ return await store.find_enabled(question)
+ except Exception:
+ # Configuration storage must never make normal chat unavailable.
+ logger.warning("Fixed-question lookup failed; continuing with model", exc_info=True)
+ return None
+
+
_DEFAULT_ASSISTANT_ID = "lead_agent"
# Built-in LangGraph graph IDs registered in langgraph.json.
@@ -869,21 +941,36 @@ async def start_run(
await _enforce_agent_access(body, request)
t_post_enforce = time.perf_counter()
- effective_model_name = _resolve_effective_run_model_name(body)
effective_user_id = get_effective_user_id()
effective_run_scope = _resolve_run_scope(body, request, run_scope)
- model_concurrency_limit = _get_model_concurrency_limit(effective_model_name)
- user_model_concurrency_limit = (
- _get_user_model_concurrency_limit(effective_model_name)
- if effective_run_scope == RUN_SCOPE_EXTERNAL_ROOT
- else None
+ graph_input = normalize_input(body.input)
+ fixed_answer = await _find_fixed_answer(
+ body=body,
+ request=request,
+ graph_input=graph_input,
+ run_scope=effective_run_scope,
)
+ if fixed_answer is not None:
+ effective_model_name = "fixed-answer"
+ model_concurrency_limit = None
+ user_model_concurrency_limit = None
+ else:
+ effective_model_name = _resolve_effective_run_model_name(body)
+ model_concurrency_limit = _get_model_concurrency_limit(effective_model_name)
+ user_model_concurrency_limit = (
+ _get_user_model_concurrency_limit(effective_model_name)
+ if effective_run_scope == RUN_SCOPE_EXTERNAL_ROOT
+ else None
+ )
run_id = str(uuid.uuid4())
run_metadata = dict(body.metadata or {})
run_metadata["model_name"] = effective_model_name
run_metadata["user_id"] = effective_user_id
run_metadata["run_scope"] = effective_run_scope
+ if fixed_answer is not None:
+ run_metadata["fixed_answer"] = True
+ run_metadata["fixed_question_id"] = fixed_answer["id"]
redis_lease_acquired = False
redis_gate_degraded = True
@@ -895,7 +982,7 @@ async def start_run(
"user_id": effective_user_id,
"user_model_accounted": user_model_accounted,
}
- if concurrency_gate is not None:
+ if concurrency_gate is not None and fixed_answer is None:
acquire = await concurrency_gate.try_acquire(
run_id=run_id,
thread_id=thread_id,
@@ -996,12 +1083,22 @@ async def start_run(
t_post_thread_upsert = time.perf_counter()
try:
- agent_factory = resolve_agent_factory(body.assistant_id)
+ if fixed_answer is not None:
+ from deerflow.agents.fixed_answer_agent import make_fixed_answer_agent
+
+ def agent_factory(config): # noqa: ANN001, ARG001 - worker factory contract
+ return make_fixed_answer_agent(
+ question=str(fixed_answer["question"]),
+ answer=str(fixed_answer["answer"]),
+ fixed_question_id=str(fixed_answer["id"]),
+ tokens_per_second=int(fixed_answer.get("tokens_per_second", 100)),
+ )
+ else:
+ agent_factory = resolve_agent_factory(body.assistant_id)
except Exception:
if concurrency_gate is not None and redis_lease_acquired:
await concurrency_gate.release(**lease_metadata)
raise
- graph_input = normalize_input(body.input)
# Admin-configurable prompt prefix: when the caller opted in via
# context.prompt_prefix_enabled, tag the human message with the configured
@@ -1011,14 +1108,18 @@ async def start_run(
# applies once. We intentionally do NOT gate on thread-row existence: the
# thread_meta row is frequently pre-created, which previously skipped the
# prefix for genuine first turns.
- if _prompt_prefix_opted_in(body):
+ if fixed_answer is None and _prompt_prefix_opted_in(body):
prefix_settings = load_system_settings().prompt_prefix
if prefix_settings.enabled and prefix_settings.prompt_prefix.strip():
graph_input = _prepend_prompt_prefix(graph_input, prefix_settings.prompt_prefix.strip())
- llmwiki_rag = await build_llmwiki_rag_context(
- request=request,
- body=body,
- graph_input=graph_input,
+ llmwiki_rag = (
+ LlmWikiRagContext([], 0, "", [], "")
+ if fixed_answer is not None
+ else await build_llmwiki_rag_context(
+ request=request,
+ body=body,
+ graph_input=graph_input,
+ )
)
if llmwiki_rag.prompt or llmwiki_rag.sources:
graph_input = attach_llmwiki_rag_to_graph_input(graph_input, llmwiki_rag)
diff --git a/offline-backend-20260512/backend/config.yaml b/offline-backend-20260512/backend/config.yaml
index 3f3661a9..83017f68 100644
--- a/offline-backend-20260512/backend/config.yaml
+++ b/offline-backend-20260512/backend/config.yaml
@@ -372,6 +372,15 @@ skills:
path: skills
security_scan:
enabled: false
+ # ES 灵活查询技能路由提示:开启后注入整体 system prompt;修改 prompt 可调整
+ # 触发边界和调用要求。关闭后该提示完全不注入。
+ es_query_routing:
+ enabled: true
+ prompt: |
+ 当用户的问题需要基于 Elasticsearch 查询语句(Query DSL)灵活构造、调整或执行查询时,必须优先使用 `es_query` 技能。
+ 典型场景包括:根据用户条件动态组合 bool/must/filter/should、范围、聚合、排序或分页等 ES 查询;或者用户直接提供、要求修改 ES 语句并据此查询数据。
+ 执行前先读取 `es_query` 技能的 SKILL.md,严格按照技能中的流程和参数调用;不要用通用搜索替代,也不要凭记忆臆测查询结果。
+ 仅当问题确实需要 ES 数据查询时使用;纯概念解释、普通代码说明或与 ES 无关的问题不要调用。
skill_evolution:
enabled: true
@@ -547,6 +556,45 @@ llmwiki:
web_base_url: "http://119.254.86.251:8088"
admin_email: "lqq@qq.com"
admin_password: "1qaz@WSX"
+ # DeerFlow-local Wiki mirror/vector index. Keep disabled until the
+ # OpenAI-compatible embedding endpoint below is configured and reachable.
+ local_wiki_index:
+ enabled: false
+ sync_enabled: false
+ internal_search_enabled: false
+ reference_ui_enabled: false
+ sync_interval_seconds: 120
+ chunk_max_chars: 1000
+ chunk_overlap_chars: 120
+ chunk_min_chars: 80
+ embedding:
+ enabled: true
+ provider: openai
+ model: bge-m3
+ base_url: ""
+ api_key: ""
+ dimensions: 1024
+ batch_size: 32
+ timeout_seconds: 30
+ max_retries: 3
+ engine: numpy_exact
+ strict_vector: true
+ top_k_sections: 30
+ top_k_pages: 8
+ max_sections_per_page: 2
+ min_similarity: 0.35
+ cache_max_knowledge_bases: 20
+ cache_revision_check_seconds: 5
+ external_api:
+ enabled: false
+ api_key: ""
+ previous_api_key: ""
+ max_query_chars: 1000
+ max_knowledge_bases: 20
+ max_results: 50
+ default_results: 10
+ include_content_max_chars: 12000
+ requests_per_minute: 60
log_system:
enabled: true
diff --git a/offline-backend-20260512/backend/docs/CONFIGURATION.md b/offline-backend-20260512/backend/docs/CONFIGURATION.md
index cc0778db..91b045bb 100644
--- a/offline-backend-20260512/backend/docs/CONFIGURATION.md
+++ b/offline-backend-20260512/backend/docs/CONFIGURATION.md
@@ -270,9 +270,21 @@ skills:
# Host path (optional, default: ../skills)
path: /custom/path/to/skills
- # Container mount path (default: /mnt/skills)
- container_path: /mnt/skills
-```
+ # Container mount path (default: /mnt/skills)
+ container_path: /mnt/skills
+ # Optional system-prompt routing rule for the es_query skill
+ es_query_routing:
+ enabled: true
+ prompt: |
+ 当用户的问题需要基于 Elasticsearch Query DSL 灵活构造、调整或执行查询时,优先使用 `es_query` 技能。
+ 执行前先读取该技能的 SKILL.md,并严格遵循其中的流程和参数。
+```
+
+`skills.es_query_routing.enabled` controls whether the routing rule is injected
+into the overall agent system prompt. Edit `prompt` to tune the matching boundary
+or invocation requirements. When disabled (the schema default), the block is not
+injected. Custom agents receive it only when their explicit skill allowlist
+contains `es_query`.
**How Skills Work**:
- Skills are stored in `deer-flow/skills/{public,custom}/`
diff --git a/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md b/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md
new file mode 100644
index 00000000..0a166d91
--- /dev/null
+++ b/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md
@@ -0,0 +1,2096 @@
+# WeKnora Wiki 数据同步至 DeerFlow 并建立本地向量索引——实现级设计方案
+
+> 文档状态:首版已实现(Implemented,2026-08-23;2026-08-24 增补整库未向量化时的 Wiki-only 检索回退、知识库所有者手动向量化及全量 Excel 导出)
+> 编写日期:2026-08-21
+> 适用代码库:当前 DeerFlow 工作区
+> WeKnora 适配基线:v0.7.2
+> 关联文档:`docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`
+> 核心边界:**不修改 WeKnora 后端;WeKnora 仅作为 Wiki 生成源,DeerFlow 保存完整 Wiki 镜像、章节向量和检索索引,并承接全部内部与外部检索。**
+> 现行产品补充决策:内部检索只有在单个知识库整库完成向量化后才使用本地向量;未完成时只调用 WeKnora Wiki 页面检索接口并读取 Wiki 正文,绝不调用原始文档/切片检索。外部 API 继续严格使用本地向量。Wiki 管理页支持所有者手动触发整库向量化,并将全部 Wiki 导出为包含当前目录、完整父级目录和逐级目录列的 Excel。
+
+---
+
+## 1. 文档目标
+
+本文是后续编码的默认实施蓝图,目标不是简单描述“可以做向量检索”,而是把以下工程问题一次性定义清楚:
+
+1. 如何从 WeKnora 拉取已经加工完成的 Wiki 页面,而不是继续检索原始文档切片。
+2. 如何把 Wiki 页面全文、元数据和向量持久化到 DeerFlow 自己的数据库中。
+3. 如何在不引入第二个重型向量服务的前提下,先用 DeerFlow 当前的 SQLite/PostgreSQL/MySQL 数据库与 NumPy 完成可用的精确向量检索。
+4. 如何保证 Wiki 新建、重新生成、人工编辑、回滚、归档和删除后,DeerFlow 本地镜像及向量不会长期失真。
+5. 如何让普通问答、智能体工具、页面检索、右侧参考文献和公共外部接口复用同一套检索结果。
+6. 如何保证个人库、公共库、智能体绑定、对话选择、Wiki 页面状态和外部发布开关在检索时被严格执行。
+7. 如何在外部开放公共 Wiki 检索时,避免把私有库、草稿页、系统“对话沉淀”、WeKnora 原始 ID 或内部凭据泄露出去。
+8. 如何迁移、灰度、回滚、监控和验收,而不是一次性替换后无法定位质量问题。
+
+本文提到的表名、字段、模块、接口和默认行为均作为实现时的默认方案。如编码阶段出现必要调整,应同步更新本文,避免文档与实现长期漂移。
+
+---
+
+## 2. 结论摘要
+
+### 2.1 可行性结论
+
+方案可行,整体难度为中等。DeerFlow 已经具备三项可直接复用的基础能力:
+
+- OpenAI 兼容的异步 Embedding 客户端:`deerflow/knowledge/embeddings.py`;
+- 按 Markdown 标题与段落切分的逻辑:`deerflow/knowledge/chunking.py`;
+- 余弦相似度、关键词评分和混合排序基础:`deerflow/knowledge/search.py`。
+
+本项目真正需要新增的核心能力是:
+
+1. Wiki 页面本地镜像数据模型;
+2. WeKnora → DeerFlow 的全量/增量同步与租约;
+3. Wiki 页面章节化、Embedding 和原子换代;
+4. DeerFlow 本地向量快照与权限过滤检索;
+5. 内部问答、引用面板和外部接口的统一结果协议;
+6. 索引状态、失败重试、重建、监控和安全边界。
+
+### 2.2 最终数据边界
+
+本方案完成后,数据职责如下:
+
+| 数据/能力 | WeKnora | DeerFlow |
+|---|---:|---:|
+| 原始文件、原始切片 | 保留并管理 | 不复制(除现有引用兼容所需元数据) |
+| Wiki 自动生成 | 负责 | 不负责 |
+| Wiki 页面主生成源 | 是 | 否,保存可检索镜像 |
+| Wiki 页面全文镜像 | 有 | **有** |
+| Wiki 页面章节切分 | 可有但不依赖 | **负责** |
+| Wiki 页面 Embedding | 不要求 | **负责** |
+| Wiki 向量持久化 | 不要求 | **负责** |
+| 内部问答检索 | 不再作为主查询路径 | **负责** |
+| 右侧参考文献详情 | 不再实时依赖 | **负责** |
+| 公共外部检索接口 | 不对外暴露 | **负责** |
+| 用户/公共/智能体权限 | 不作为最终边界 | **DeerFlow 为最终边界** |
+
+这里的“DeerFlow 保存数据”指:DeerFlow 保存一份完整 Wiki 页面镜像和用于检索的章节向量。WeKnora 仍然会保存它生成 Wiki 所需的数据;如果未来要求 WeKnora 完全不保存 Wiki,则必须把 Wiki 生成管道也迁入 DeerFlow,那是另一项更大的工程,不在本文范围内。
+
+### 2.3 推荐首版技术路线
+
+首版建议:
+
+- 数据库继续沿用 DeerFlow 的统一 SQLAlchemy 数据库;
+- Wiki 页面全文存 `PortableLongText`;
+- 向量使用归一化后的 `float32` 二进制 BLOB 持久化;
+- 检索时按知识库懒加载 NumPy 矩阵,并进行精确余弦相似度计算;
+- 每个 Gateway worker 保持自己的只读内存快照,通过数据库索引修订号判断是否需要重载;
+- 默认使用**纯向量检索**,Embedding/索引不可用时明确报错或返回“索引未就绪”,不偷偷回退原文 chunk;
+- 可配置开启轻量关键词加权,但它不能改变“结果来源必须是 Wiki 页面”的边界;
+- 当规模或多实例内存成本超过可接受范围时,再将向量执行层切换为 pgvector;页面镜像、权限、API 和引用协议保持不变。
+
+---
+
+## 3. 背景与现状诊断
+
+### 3.1 当前检索的是原始切片
+
+当前 `WeKnoraClient.search()` 调用:
+
+```text
+POST /api/v1/knowledge-search
+```
+
+返回的核心字段是:
+
+```text
+chunk_id
+content
+knowledge_id
+knowledge_base_id
+title / filename
+score
+chunk_index
+metadata
+```
+
+因此当前检索实体是 WeKnora 原始文档解析后的 chunk,而不是已经归纳、合并、编辑后的 Wiki 页面。
+
+该方法目前被三个主要生产入口复用:
+
+1. `app/gateway/llmwiki_rag.py`:普通问答发送给模型前的自动预检索;
+2. `deerflow/tools/builtins/llmwiki_search_tool.py`:智能体主动调用的 `llmwiki_search` 工具;
+3. `app/gateway/routers/llmwiki.py`:`POST /api/llmwiki/search` 页面检索。
+
+只修改其中一个入口会导致不同页面、智能体和问答路径继续返回两套来源,因此实现时必须统一切换。
+
+### 3.2 当前参考文献以 chunk/document 为中心
+
+当前引用 URL 主要为:
+
+```text
+/weknora-source-preview?knowledge_base_id=&chunk_id=
+```
+
+部分工具路径还会附带 `knowledge_id` / `document_id`。前端 `WeKnoraSourceDrawer` 在已有 `documentId` 时会直接加载整份原文件或解析后全文,导致用户看到的是整个文档,而不是对应 Wiki 页面。
+
+现有 `/api/llmwiki/sources/context` 虽然支持从 chunk metadata 中读取 `wiki_page_slug`,但它属于尽力映射:
+
+- 很多历史 chunk 并没有 `wiki_page_slug`;
+- Wiki 页可能综合多个文档和多个 chunk;
+- 现有映射函数主要依赖 chunk metadata 中直接出现 slug,并没有可靠建立 `chunk_refs → wiki page` 的本地反向索引;
+- 即使找到 Wiki,当前抽屉也仍可能先展示整份原文,再在下方追加 Wiki 内容。
+
+新方案不再以“先命中 chunk,再猜对应 Wiki”为主路径,而是直接检索 DeerFlow 本地 Wiki 章节向量,结果天然携带 `wiki_page_id + wiki_slug`。
+
+### 3.3 WeKnora 已提供 Wiki 页面读取能力
+
+当前适配器已经具备:
+
+- `list_wiki_pages()`;
+- `get_wiki_page()`;
+- Wiki 页面创建、更新、删除;
+- Wiki 图谱读取。
+
+因此不需要修改 WeKnora 后端即可拉取页面。需要新增的是 DeerFlow 侧同步器、本地存储和向量化流程。
+
+### 3.4 DeerFlow 现有向量实现的可复用点与限制
+
+当前产品知识库的向量功能使用:
+
+- `KnowledgeEmbeddingRow.vector`:JSON 浮点数组;
+- `KnowledgeRepository.all_embeddings()`:一次加载全部向量;
+- Python 循环逐条调用 `cosine_similarity()`。
+
+这套实现适合小规模功能验证,但不适合作为公共 Wiki 的正式索引,因为:
+
+1. JSON 浮点向量体积大、解析慢;
+2. 每次查询加载全部向量会放大数据库 IO;
+3. Python 标量循环不能充分利用 NumPy/BLAS;
+4. 没有按知识库权限预分片;
+5. 没有多 worker 索引修订和缓存失效机制。
+
+本方案复用 Embedding 客户端和 Markdown 切分思想,但为 LLMWiki Wiki 新建独立表与检索快照,不直接塞入 `knowledge_notes/knowledge_embeddings`。
+
+---
+
+## 4. 目标、非目标与默认决策
+
+### 4.1 功能目标
+
+1. DeerFlow 能完整保存每个已映射 WeKnora 知识库的 Wiki 页面镜像。
+2. DeerFlow 能按 Wiki 页面 Markdown 结构生成章节向量。
+3. 普通问答、智能体工具和页面检索都能按向量搜索 Wiki 页面。
+4. 大模型收到的是命中 Wiki 页的相关章节,而不是原始文档 chunk。
+5. 右侧参考文献主视图只展示 Wiki 页面。
+6. 公共 Wiki 可通过受控外部 API 进行向量检索。
+7. WeKnora 不可用时,已经同步成功的 Wiki 仍可检索和展示。
+8. Wiki 更新后可自动增量重建对应页面向量,不全库重算。
+9. Embedding 模型变化时可明确触发全量重建。
+10. 用户取消公共发布或关闭外部开放后,外部查询立即失去访问权,不依赖异步清理完成。
+
+### 4.2 明确非目标
+
+- 不修改或 fork WeKnora 后端。
+- 不把 WeKnora 原始文件和全部原始 chunks 复制进 DeerFlow。
+- 不让 DeerFlow 首版承担 Wiki 自动生成。
+- 不使用“把 Wiki 再上传成隐藏文档库”的循环导入方案。
+- 不在首版实现分布式 HNSW/Milvus/Qdrant 集群。
+- 不在首版提供原始 embedding 向量下载接口。
+- 不保证历史 chunk 引用全部能够转换为 Wiki 引用。
+- 不允许外部调用者通过 API 传入 WeKnora 原始知识库 ID。
+- 不允许向量服务失败后静默改查原始文档。
+
+### 4.3 默认产品决策
+
+如实现前未另行变更,采用以下默认值:
+
+1. 同步范围:所有 DeerFlow 已映射、远端存在且启用 Wiki 的知识库,包括个人库和公共库。
+2. 页面范围:同步 `draft/published/archived` 全部状态,检索时再按调用者权限和页面状态过滤。
+3. 内部检索:默认只检索 `published` 页面;所有者/管理员可以通过明确参数包含 `draft`,普通用户不能。
+4. 外部检索:只检索 `published` 页面。
+5. 外部开放:新增 `external_search_enabled`,默认 `false`,不能仅凭现有 `publication_status=published` 自动向互联网开放。
+6. 系统“对话沉淀”知识库:允许内部按现有权限使用,但外部接口永远排除。
+7. 检索模式:严格向量;索引未就绪时返回明确状态,不回退原文。
+8. 引用单位:Wiki 页面;检索单位:Wiki 页面内的章节 chunk。
+9. 首版向量执行器:按知识库懒加载的 NumPy 精确检索。
+10. 外部搜索默认不返回完整正文,只返回摘要和命中片段;完整页面通过详情接口读取。
+
+---
+
+## 5. 总体架构
+
+### 5.1 写入与同步链路
+
+```mermaid
+flowchart LR
+ WK["WeKnora Wiki 生成/编辑"] --> API["WeKnora Wiki REST API"]
+ API --> SYNC["DeerFlow Wiki Sync Service"]
+ SYNC --> PAGE["llmwiki_wiki_pages"]
+ PAGE --> CHUNK["Markdown 章节切分"]
+ CHUNK --> EMB["DeerFlow Embedding Client"]
+ EMB --> VECTOR["llmwiki_wiki_vectors"]
+ VECTOR --> REV["知识库索引修订号 +1"]
+ REV --> CACHE["各 Gateway worker 懒重载 NumPy 快照"]
+```
+
+### 5.2 内部检索链路
+
+```mermaid
+flowchart LR
+ Q["用户问题"] --> AUTH["解析对话选择/智能体绑定/实时权限"]
+ AUTH --> QE["生成一次查询向量"]
+ QE --> IDX["按允许的知识库快照检索 TopK 章节"]
+ IDX --> GROUP["按 Wiki 页面聚合/去重"]
+ GROUP --> PROMPT["相关章节注入模型"]
+ GROUP --> REF["结构化 Wiki 引用"]
+ REF --> DRAWER["右侧展示 DeerFlow 本地 Wiki 页面"]
+```
+
+### 5.3 外部检索链路
+
+```mermaid
+flowchart LR
+ EXT["外部系统 + API Key"] --> GATE["限流/鉴权/输入校验"]
+ GATE --> PUB["published + external_search_enabled"]
+ PUB --> LOCAL["DeerFlow 本地向量快照"]
+ LOCAL --> RESULT["公共 Wiki 摘要/片段/页面标识"]
+ RESULT --> DETAIL["可选读取完整公共 Wiki 页面"]
+```
+
+### 5.4 查询期间不依赖 WeKnora
+
+同步完成后,以下请求不得实时访问 WeKnora:
+
+- 内部向量搜索;
+- 普通问答预检索;
+- `llmwiki_search` 工具;
+- 右侧 Wiki 参考文献详情;
+- 外部公共 Wiki 搜索;
+- 外部公共 Wiki 页面详情。
+
+WeKnora 只出现在同步与管理链路中。这样可以降低延迟、避免跨库扇出,并确保外部流量不会直接放大到 WeKnora。
+
+---
+
+## 6. 配置设计
+
+在 `deerflow/config/llmwiki_config.py` 的 `LlmWikiConfig` 下增加独立的 `local_wiki_index` 配置,避免和产品“我的知识”功能强耦合。
+
+建议配置:
+
+```yaml
+llmwiki:
+ weknora:
+ api_base_url: "http://weknora-app:8080"
+ web_base_url: "http://weknora-app:8080"
+ admin_email: "$WEKNORA_ADMIN_EMAIL"
+ admin_password: "$WEKNORA_ADMIN_PASSWORD"
+
+ local_wiki_index:
+ enabled: true
+
+ # 同步
+ sync_enabled: true
+ sync_interval_seconds: 120
+ sync_page_size: 100
+ sync_concurrency: 3
+ sync_lease_seconds: 600
+ sync_request_timeout_seconds: 30
+ sync_max_retries: 3
+ sync_retry_backoff_seconds: 5
+
+ # 切分
+ chunk_max_chars: 1000
+ chunk_overlap_chars: 120
+ chunk_min_chars: 80
+ title_prefix_enabled: true
+
+ # Embedding:OpenAI-compatible /v1/embeddings
+ embedding:
+ enabled: true
+ provider: openai
+ model: bge-m3
+ base_url: "$WIKI_EMBEDDING_BASE_URL"
+ api_key: "$WIKI_EMBEDDING_API_KEY"
+ dimensions: 1024
+ batch_size: 32
+ timeout_seconds: 30
+ max_retries: 3
+
+ # 检索
+ engine: numpy_exact
+ strict_vector: true
+ top_k_sections: 30
+ top_k_pages: 8
+ max_sections_per_page: 2
+ min_similarity: 0.35
+ keyword_boost_enabled: false
+ keyword_boost_weight: 0.15
+ cache_max_knowledge_bases: 20
+ cache_revision_check_seconds: 5
+
+ # 外部接口
+ external_api:
+ enabled: false
+ api_key: "$LLMWIKI_EXTERNAL_SEARCH_API_KEY"
+ max_query_chars: 1000
+ max_knowledge_bases: 20
+ max_results: 50
+ default_results: 10
+ include_content_max_chars: 12000
+ requests_per_minute: 60
+```
+
+### 6.1 配置校验规则
+
+启动时应执行:
+
+1. `enabled=true` 时,Embedding `base_url` 和 `model` 必须非空;
+2. `dimensions` 可省略,但第一次成功 Embedding 后必须固定并写入索引 profile;
+3. `chunk_overlap_chars < chunk_max_chars`;
+4. `top_k_pages <= top_k_sections`;
+5. `external_api.enabled=true` 时必须配置非空 API Key;
+6. API Key 不得出现在日志、状态接口或前端运行时配置;
+7. `strict_vector=true` 时,Embedding 不可用必须反映为索引失败/搜索不可用,不能静默关键词回退;
+8. 更换 `model/base_url/dimensions` 后计算出的 `embedding_fingerprint` 变化,系统应标记全量索引过期。
+
+### 6.2 Embedding 指纹
+
+建议指纹:
+
+```text
+sha256(provider + "\0" + base_url_normalized + "\0" + model + "\0" + dimensions)
+```
+
+不把 API Key放入指纹。指纹用于:
+
+- 判断现有向量是否可继续使用;
+- 避免不同维度向量进入同一矩阵;
+- 触发管理员重建;
+- 在状态接口中显示短指纹,便于排障。
+
+---
+
+## 7. 数据模型
+
+### 7.1 为什么使用独立表
+
+不建议复用 `knowledge_notes` 和 `knowledge_embeddings`,原因是:
+
+- 产品知识库当前是全局共享语义,LLMWiki 有个人/公共/智能体绑定权限;
+- Wiki 页面需要保存 WeKnora mapping、slug、版本、状态和同步信息;
+- Wiki 页面可能频繁由生成管道重写,生命周期与人工知识笔记不同;
+- 避免出现在“我的知识”列表、目录、导出和图谱中;
+- 后续可单独迁移到 pgvector,不影响现有知识功能。
+
+### 7.2 扩展 `llmwiki_knowledge_bases`
+
+建议新增字段:
+
+| 字段 | 类型 | 默认 | 说明 |
+|---|---|---:|---|
+| `wiki_index_enabled` | bool | true | 是否同步并建立本地 Wiki 索引 |
+| `external_search_enabled` | bool | false | 是否允许外部 API 检索 |
+| `external_search_updated_at` | datetime nullable | null | 外部开关最后变化时间 |
+
+注意:
+
+- `publication_status=published` 继续表示 DeerFlow 内部公共;
+- `external_search_enabled=true` 才表示允许对外;
+- 外部检索必须同时满足两者;
+- 系统“对话沉淀”即使字段误设为 true,也应在代码层强制排除。
+
+### 7.3 `llmwiki_wiki_pages`
+
+建议 ORM 模型路径:
+
+```text
+deerflow/persistence/llmwiki_index/model.py
+```
+
+字段设计:
+
+| 字段 | 类型 | 约束/索引 | 说明 |
+|---|---|---|---|
+| `id` | String(64) | PK | DeerFlow 本地稳定页面 ID |
+| `knowledge_base_mapping_id` | String(64) | FK + index | DeerFlow 映射 ID,不是 WeKnora ID |
+| `remote_page_id` | String(128) nullable | index | WeKnora 页面 ID,仅内部使用 |
+| `slug` | String(1024) | 普通字段 | Wiki 页面路径 |
+| `slug_hash` | String(64) | unique(mapping, hash) | 避免 MySQL 长索引限制 |
+| `title` | String(512) | index/prefix index | 页面标题 |
+| `summary` | PortableLongText | | 页面摘要 |
+| `content_md` | PortableLongText | | 完整 Wiki Markdown 镜像 |
+| `page_type` | String(32) | index | summary/entity/concept/... |
+| `status` | String(32) | index | draft/published/archived |
+| `aliases` | PortableJSON | | 别名数组 |
+| `category_path` | PortableJSON | | Wiki 目录路径 |
+| `parent_slug` | String(1024) nullable | | 父页面 |
+| `wiki_path` | String(1024) nullable | | 上游规范化路径 |
+| `source_refs` | PortableJSON | | 原文文档级溯源,仅内部使用 |
+| `chunk_refs` | PortableJSON | | 原始 chunk 级溯源,仅内部使用 |
+| `page_version` | int | index | 上游 Wiki 版本 |
+| `content_hash` | String(64) | index | 参与向量化内容哈希 |
+| `remote_created_at` | datetime nullable | | 上游创建时间 |
+| `remote_updated_at` | datetime nullable | index | 上游更新时间 |
+| `index_status` | String(24) | index | pending/indexing/ready/failed/stale/deleted |
+| `indexed_fingerprint` | String(64) nullable | | 已生成向量的 Embedding 指纹 |
+| `indexed_at` | datetime nullable | | 最近成功索引时间 |
+| `last_error` | Text nullable | | 截断、脱敏后的错误 |
+| `is_remote_deleted` | bool | index | 完整同步确认上游不存在 |
+| `last_seen_sync_id` | String(64) nullable | index | 本轮完整扫描标记 |
+| `created_at` | datetime | | 本地创建时间 |
+| `updated_at` | datetime | index | 本地更新时间 |
+
+本地 `id` 建议是 UUID,不直接使用 remote ID。唯一性使用:
+
+```text
+UNIQUE(knowledge_base_mapping_id, slug_hash)
+slug_hash = sha256(normalized_slug)
+```
+
+写入时如果 hash 相同但 slug 不同,应视为极低概率冲突并拒绝覆盖,记录显式错误。
+
+### 7.4 `llmwiki_wiki_vectors`
+
+| 字段 | 类型 | 约束/索引 | 说明 |
+|---|---|---|---|
+| `id` | String(64) | PK | 向量记录 ID |
+| `page_id` | String(64) | FK cascade + index | 本地 Wiki 页面 |
+| `knowledge_base_mapping_id` | String(64) | index | 冗余字段,便于分片加载 |
+| `section_index` | int | unique(page,index) | 页面内章节序号 |
+| `heading` | String(1024) nullable | | 命中章节标题 |
+| `section_content` | PortableLongText | | 送给模型和展示的片段 |
+| `content_hash` | String(64) | index | 章节文本哈希 |
+| `embedding_fingerprint` | String(64) | index | 模型指纹 |
+| `embedding_dimensions` | int | | 维度 |
+| `vector_blob` | LargeBinary | | 归一化 float32 little-endian |
+| `created_at` | datetime | | 创建时间 |
+
+约束:
+
+```text
+UNIQUE(page_id, section_index, embedding_fingerprint)
+```
+
+向量写入前必须:
+
+1. 校验长度等于 `embedding_dimensions`;
+2. 拒绝 NaN/Inf;
+3. L2 归一化;
+4. 转为 little-endian float32;
+5. 保存 BLOB;
+6. 读取时再次验证 BLOB 字节数为 `dimensions * 4`。
+
+### 7.5 `llmwiki_wiki_sync_states`
+
+每个知识库一行:
+
+| 字段 | 类型 | 说明 |
+|---|---|---|
+| `knowledge_base_mapping_id` | String(64) PK | 本地映射 ID |
+| `state` | String(24) | idle/running/partial/failed/disabled |
+| `active_sync_id` | String(64) nullable | 当前扫描 ID |
+| `lease_owner` | String(128) nullable | worker 标识 |
+| `lease_expires_at` | datetime nullable | 多 worker 租约 |
+| `last_started_at` | datetime nullable | 最近启动 |
+| `last_completed_at` | datetime nullable | 最近完整成功 |
+| `last_success_at` | datetime nullable | 最近任意成功 |
+| `last_error` | Text nullable | 最近错误 |
+| `remote_page_count` | int | 上游扫描数量 |
+| `local_page_count` | int | 本地有效页面数量 |
+| `ready_page_count` | int | 已就绪页面数量 |
+| `vector_count` | int | 向量数量 |
+| `failed_page_count` | int | 失败页面数量 |
+| `index_revision` | bigint | 该知识库索引修订号 |
+| `embedding_fingerprint` | String(64) nullable | 当前期望指纹 |
+| `updated_at` | datetime | 状态更新时间 |
+
+`index_revision` 只在可见检索数据真正变化后递增,例如:
+
+- 页面首次索引成功;
+- 页面向量被替换;
+- 页面被删除/归档;
+- 页面状态变化影响检索;
+- Embedding 指纹重建完成。
+
+单纯更新进度、错误文字或同步时间不应递增。
+
+### 7.6 可选任务明细表
+
+首版可仅使用 sync state + 页面状态。如果需要管理页展示逐页面进度、失败重试和历史记录,可增加:
+
+```text
+llmwiki_wiki_index_jobs
+```
+
+记录任务类型(sync/rebuild/page_reindex)、状态、总数、完成数、失败数、发起人和错误摘要。该表不是检索正确性的必要条件,可以第二阶段增加。
+
+---
+
+## 8. Wiki 同步设计
+
+### 8.1 同步触发方式
+
+必须同时支持:
+
+1. **周期完整扫描**:捕获 WeKnora 异步 Wiki 生成、后台修订和不经过 DeerFlow BFF 的编辑;
+2. **管理操作后即时同步**:通过 DeerFlow 创建、更新、删除、回滚 Wiki 页面后,立即入队同步该页;
+3. **管理员手动同步**:用于排障和首次导入;
+4. **管理员全量重建**:用于切换 Embedding 模型/维度或修复索引。
+
+### 8.2 知识库发现
+
+每轮调度:
+
+1. 从 `llmwiki_knowledge_bases` 获取 `wiki_index_enabled=true` 的映射;
+2. 排除已删除映射;
+3. 向 WeKnora 查询远端知识库存在性与 Wiki 能力;
+4. 远端不存在时不立即删除本地数据,而是标记同步失败;
+5. 只有显式删除映射或连续确认上游知识库已删除时,才进入本地清理流程;
+6. 每个知识库独立获得租约,避免多 Gateway worker 重复同步。
+
+### 8.3 完整扫描算法
+
+伪代码:
+
+```python
+async def sync_knowledge_base(mapping_id):
+ state = await repo.try_acquire_lease(mapping_id, owner, ttl=600)
+ if not state.acquired:
+ return "already_running"
+
+ sync_id = uuid4()
+ complete_scan = False
+ try:
+ remote_kb = await weknora.get_knowledge_base(remote_id)
+ assert wiki_enabled(remote_kb)
+
+ page = 1
+ while True:
+ batch = await weknora.list_wiki_pages(remote_id, page=page, page_size=100)
+ for remote_page in batch.pages:
+ await mirror_and_maybe_index(mapping_id, remote_page, sync_id)
+ await repo.renew_lease(mapping_id, owner)
+ if page >= batch.total_pages:
+ break
+ page += 1
+
+ complete_scan = True
+ await repo.mark_missing_pages_deleted(mapping_id, last_seen_sync_id_not=sync_id)
+ await repo.finish_sync_success(mapping_id, sync_id)
+ except Exception as exc:
+ await repo.finish_sync_failure(mapping_id, sync_id, safe_error(exc))
+ raise
+ finally:
+ await repo.release_lease(mapping_id, owner)
+```
+
+关键安全规则:**只有本轮所有分页都成功完成时,才能把“未出现页面”标为删除。** 中途超时或某一页失败时,绝不能根据不完整集合删除本地页面。
+
+### 8.4 页面变化判断
+
+用于 `content_hash` 的规范化内容建议为:
+
+```text
+title
+aliases(稳定排序)
+summary
+content_md(统一换行,保留 Markdown)
+page_type
+status
+```
+
+不把更新时间本身放入 hash,避免上游只刷新时间就重复调用 Embedding。
+
+判断:
+
+| 场景 | 动作 |
+|---|---|
+| 本地没有页面 | 保存镜像,状态 pending,建立向量 |
+| hash 和 embedding fingerprint 都相同 | 仅更新 `last_seen_sync_id` 和必要元数据 |
+| hash 变化 | 保存新镜像,重建该页全部章节向量 |
+| fingerprint 变化 | 内容不变也要重建向量 |
+| 状态变 archived | 保存镜像,删除/停用活动向量,revision +1 |
+| 状态从 archived 变 published | 重新建立或恢复向量,revision +1 |
+| 完整扫描确认页面消失 | 标记 remote_deleted,删除向量,revision +1 |
+
+### 8.5 页面镜像与向量的原子可见性
+
+不能先删除旧向量,再调用远程 Embedding,否则 Embedding 失败期间页面完全不可检索。
+
+推荐流程:
+
+1. 在事务外准备规范化文本、chunks 和新 embedding;
+2. Embedding 全部成功并校验维度;
+3. 开启数据库事务;
+4. upsert 页面镜像;
+5. 删除该页旧活动向量;
+6. 批量插入新向量;
+7. 设置 `index_status=ready`、更新 fingerprint;
+8. 对知识库 `index_revision + 1`;
+9. 提交事务。
+
+如果内容镜像已经变化但向量生成失败,有两种选择:
+
+- 保留旧页面+旧向量,直到新向量成功;
+- 保存新页面但继续用旧向量。
+
+本文默认采用更一致的第一种:**页面检索镜像与向量同事务换代**。可另存 `pending_content_hash/last_error` 记录待更新状态,但查询始终看到同一版本的页面与向量。
+
+### 8.6 删除和归档
+
+- 上游页面归档:本地保留页面镜像用于审计,但搜索快照排除;
+- 上游页面删除:本地标记 `is_remote_deleted=true`,删除向量;
+- DeerFlow 映射删除:级联清理本地页面和向量;
+- 知识库取消公共发布:不删除本地向量,因为所有者仍可能内部检索;只通过权限过滤立即禁止公共/外部访问;
+- 关闭 `wiki_index_enabled`:停止同步并从搜索快照排除,是否物理清理由管理员另行执行。
+
+### 8.7 多 worker 租约
+
+租约获取应使用数据库原子条件更新:
+
+```text
+lease_owner IS NULL OR lease_expires_at < now OR lease_owner = current_owner
+```
+
+同步过程中每完成一个分页或一批 Embedding 续租。进程崩溃后,租约到期由其它 worker 接管。不要只使用本地文件锁,因为 PostgreSQL/MySQL 多实例部署下文件锁不可见。
+
+SQLite 单机仍可使用同一数据库租约逻辑,不需要分支实现。
+
+---
+
+## 9. Wiki 页面切分与 Embedding
+
+### 9.1 为什么检索章节、引用页面
+
+完整 Wiki 页面可能很长。如果整页只生成一个向量:
+
+- 多主题页面语义被平均;
+- 局部事实召回变差;
+- 送给模型时正文过长;
+- 页面更新任一小段都会改变整页表达。
+
+因此:
+
+- **向量实体**:Wiki 页面章节;
+- **引用实体**:Wiki 页面;
+- **展示实体**:Wiki 页面;
+- **给模型的内容**:命中的 1~2 个章节。
+
+### 9.2 文本预处理
+
+处理顺序:
+
+1. 统一 `\r\n` 为 `\n`;
+2. 去除 YAML frontmatter,但将有价值的 aliases/category 显式加入标题前缀;
+3. 保留 Markdown 标题文字;
+4. 保留表格文本,但可把过长表格按行切分;
+5. 移除纯 HTML script/style;
+6. 不执行 Markdown 内嵌 HTML;
+7. 图片仅保留 alt 文本和标题,不下载图片生成 embedding;
+8. 规范化连续空行;
+9. 对“对话沉淀”继续沿用已有品牌清理规则;
+10. 不对 Wiki 正文做展示层敏感词替换后再向量化,避免管理配置变化导致全量重建;敏感信息是否允许进入索引应在同步前使用专门安全策略决定。
+
+### 9.3 切分格式
+
+建议每个 embedding 输入都带页面上下文:
+
+```text
+页面:{title}
+别名:{aliases}
+类型:{page_type}
+章节:{heading_path}
+
+{section_content}
+```
+
+这样即使命中片段正文没有重复实体名,向量仍能感知页面主题。
+
+首版参数:
+
+- `max_chars=1000`;
+- `overlap=120`;
+- `min_chars=80`;
+- 优先按 H1~H6 切分;
+- 过长章节按段落打包;
+- 单段仍超长时硬切;
+- 空章节跳过;
+- summary 可作为第 0 段,格式为“页面摘要”。
+
+### 9.4 内容去重
+
+同一页面内如果两个 chunk 规范化后 hash 相同,只保留一个。不同页面内容相同也不全局去重,因为它们可能属于不同权限知识库和不同引用页面。
+
+### 9.5 Embedding 调用要求
+
+现有 `EmbeddingClient` 是 best-effort 设计,失败时返回空数组。Wiki 正式索引应封装一个严格模式:
+
+- 支持批量请求;
+- 指数退避重试;
+- 区分超时、鉴权、限流、响应格式和维度错误;
+- 返回数量必须与输入数量相同;
+- 任一向量为空、维度不同、含 NaN/Inf,则本批失败;
+- 错误写入页面 `last_error`,但不覆盖旧索引;
+- 日志不输出正文、API Key 或完整响应体;
+- 记录模型名、批大小、耗时、状态码和安全错误分类。
+
+### 9.6 模型切换
+
+Embedding 指纹变化后:
+
+1. 新查询不得把旧模型向量和新查询向量比较;
+2. 状态接口显示 `rebuild_required=true`;
+3. 管理员触发全量重建;
+4. 重建可逐知识库进行;
+5. 某个知识库全部新向量完成后,再切换其 active fingerprint;
+6. 未完成的知识库继续使用旧 fingerprint 与旧查询模型是不现实的,因此建议全量重建期间暂停严格向量查询,或同时保留旧、新两个 Embedding 客户端配置。首版选择“显示维护中并暂停该库搜索”,降低复杂度。
+
+---
+
+## 10. DeerFlow 本地向量执行器
+
+### 10.1 首版选择:NumPy 精确检索
+
+当前依赖中已有 NumPy。首版无需引入 FAISS/HNSW/Qdrant,采用:
+
+1. 从数据库读取某知识库当前 fingerprint 的 ready vectors;
+2. 将 BLOB 解码为 `numpy.ndarray(dtype=float32)`;
+3. 堆叠为形状 `[N, D]` 的归一化矩阵;
+4. 查询向量归一化为 `[D]`;
+5. 计算 `scores = matrix @ query_vector`;
+6. 使用 `numpy.argpartition` 取 TopK,而不是完整排序;
+7. 根据 metadata 返回页面和章节。
+
+这是精确余弦检索,不是近似检索。
+
+### 10.2 为什么按知识库懒加载
+
+如果一次加载全部 Wiki:
+
+- 多 worker 会重复占用大量内存;
+- 权限过滤需要在全矩阵上构造 mask;
+- 热门小库会被冷门大库拖累;
+- 单库更新导致全局矩阵重建。
+
+因此每个 worker 保存 LRU:
+
+```text
+mapping_id -> WikiVectorSnapshot
+```
+
+快照包含:
+
+```python
+class WikiVectorSnapshot:
+ mapping_id: str
+ index_revision: int
+ embedding_fingerprint: str
+ dimensions: int
+ matrix: np.ndarray
+ rows: list[VectorMetadata]
+ loaded_at: datetime
+```
+
+查询多个知识库时,对各知识库快照分别取 TopK,再做全局合并。这样无需把不同库物理拼成一个永久矩阵。
+
+### 10.3 缓存失效
+
+每次查询或最多每隔 `cache_revision_check_seconds` 检查数据库中的 `index_revision`:
+
+- 修订号相同:复用快照;
+- 修订号变化:为该知识库加载新快照并原子替换;
+- 加载失败:继续保留旧快照,但若权限/发布状态变化,权限层仍会立即阻断;
+- 映射删除/索引禁用:立即从 LRU 移除;
+- LRU 超过 `cache_max_knowledge_bases`:驱逐最久未使用快照。
+
+同一 worker 内多个请求同时发现修订变化时,用 per-KB async lock 合并重载,避免击穿数据库。
+
+### 10.4 内存估算
+
+归一化 float32 矩阵仅向量部分约占:
+
+```text
+向量数 × 维度 × 4 字节
+```
+
+示例(不含 Python metadata 开销):
+
+| 章节向量数 | 维度 | 纯矩阵内存 |
+|---:|---:|---:|
+| 10,000 | 1,024 | 约 39 MiB |
+| 50,000 | 1,024 | 约 195 MiB |
+| 100,000 | 1,024 | 约 391 MiB |
+| 100,000 | 768 | 约 293 MiB |
+
+因此:
+
+- 首版上线前必须用真实 Wiki 数量做容量评估;
+- 多 worker 内存要乘以 worker 数;
+- LRU 应限制同时驻留的知识库数量;
+- 大库可进一步按固定 shard 切分;
+- 若单库达到数十万章节向量或内存不可接受,应切换 pgvector,而不是继续扩大 Python 进程内存。
+
+上述数量仅用于容量估算,不作为硬性能承诺,最终阈值由目标机器基准测试确定。
+
+### 10.5 查询算法
+
+伪代码:
+
+```python
+async def vector_search(query, allowed_mapping_ids, top_k_sections=30):
+ q = await embedding.embed_query(query)
+ q = validate_and_normalize(q)
+
+ candidates = []
+ for mapping_id in allowed_mapping_ids:
+ snapshot = await cache.get(mapping_id)
+ if snapshot.fingerprint != active_fingerprint:
+ continue
+ scores = snapshot.matrix @ q
+ local_top = argpartition_top(scores, min(top_k_sections, len(scores)))
+ candidates.extend(snapshot.materialize(local_top, scores))
+
+ candidates.sort(key=lambda item: item.score, reverse=True)
+ candidates = [c for c in candidates if c.score >= min_similarity]
+ return group_by_page(candidates)
+```
+
+### 10.6 页面聚合
+
+同一页可能命中多个章节。聚合规则:
+
+1. 页面分数默认取最高章节分数;
+2. 可选加入第二章节小幅加权,但不能简单相加导致长页面占优;
+3. 每页最多保留 `max_sections_per_page=2`;
+4. 页面按最终分数排序;
+5. 同一 slug 在不同知识库中视为不同页面;
+6. 返回 `matched_sections`,模型可接收多个片段,右侧只列一张页面卡片。
+
+建议页面分数:
+
+```text
+page_score = max_section_score + 0.05 * second_section_score(若存在)
+```
+
+最终裁剪到 1.0 仅用于展示;内部排序保留原始值。
+
+### 10.7 可选关键词加权
+
+用户已明确要求向量检索,因此关键词不得成为隐式回退。可选 hybrid 仅用于重排:
+
+```text
+final_score = (1 - w) * vector_score + w * keyword_score
+```
+
+默认 `w=0`。若后续启用,关键词评分只针对已由向量召回的候选,不额外从原始文档召回。
+
+---
+
+## 11. 权限模型
+
+### 11.1 内部检索权限
+
+检索前必须先得到 DeerFlow 本地 mapping ID 列表,并沿用现有规则:
+
+1. 当前对话明确选择的 `llmwiki_knowledge_base_ids`;
+2. 当前智能体 `config.yaml` 允许绑定的范围;
+3. 当前用户对映射的实时读取权限;
+4. 对话明确空列表表示不检索;
+5. 用户权限撤销后,旧线程下一轮查询也必须失效;
+6. 不接受前端或模型传入 WeKnora remote ID。
+
+完成上述交集后,才允许读取相应本地向量快照。
+
+### 11.2 页面状态
+
+默认:
+
+- 普通用户:仅 published;
+- 公共库访问者:仅 published;
+- 所有者/管理员:默认仍仅 published,明确 `include_drafts=true` 才包含 draft;
+- archived/deleted:任何搜索都排除;
+- 详情接口对所有者/管理员可按管理场景读取 archived,但引用详情不读取。
+
+### 11.3 外部接口权限
+
+每个返回结果必须同时满足:
+
+```text
+mapping.publication_status == "published"
+AND mapping.external_search_enabled == true
+AND mapping.wiki_index_enabled == true
+AND page.status == "published"
+AND page.is_remote_deleted == false
+AND page.index_status == "ready"
+AND mapping 不是系统“对话沉淀”
+```
+
+即使 NumPy 快照仍在内存中,只要数据库映射已取消发布或关闭外部开关,权限检查就应在查询入口立即阻断。因此缓存失效延迟不会造成权限泄漏。
+
+### 11.4 外部开放开关为什么独立
+
+当前“公共库”语义是 DeerFlow 登录用户可读,不等于互联网匿名/服务对服务可读。如果直接把所有 published 库放进外部 API,会扩大历史数据的受众范围。
+
+默认策略:
+
+- 数据库迁移后,所有现有公共库 `external_search_enabled=false`;
+- 所有者或管理员明确开启;
+- UI 显示“内部公共”和“外部 API 开放”是两个开关;
+- 开启外部开放前提示内容将可由持有 API Key 的系统检索;
+- 关闭后立即生效。
+
+### 11.5 系统“对话沉淀”
+
+该知识库在现有逻辑中始终 public,但可能含用户对话内容。因此:
+
+- 外部搜索强制排除,不能仅依赖可编辑字段;
+- 外部详情接口同样排除;
+- 状态接口不得把其 remote ID 暴露出去;
+- 内部检索继续按当前产品策略使用;
+- 未来若要外部开放,必须另做数据脱敏和用户授权设计,不属于本方案。
+
+---
+
+## 12. 统一检索结果协议
+
+### 12.1 内部结构
+
+建议统一结果:
+
+```json
+{
+ "kind": "wiki_page",
+ "id": "local-page-id",
+ "knowledge_base_id": "deerflow-mapping-id",
+ "knowledge_base_name": "公共知识库",
+ "wiki_page_id": "local-page-id",
+ "wiki_slug": "entity/alice",
+ "title": "Alice",
+ "summary": "页面摘要",
+ "page_type": "entity",
+ "page_version": 3,
+ "updated_at": "2026-08-21T10:00:00+08:00",
+ "score": 0.8621,
+ "matched_sections": [
+ {
+ "section_index": 2,
+ "heading": "主要经历",
+ "content": "命中的 Wiki 章节正文",
+ "score": 0.8621
+ }
+ ],
+ "url": "/weknora-source-preview?knowledge_base_id=deerflow-mapping-id&wiki_slug=entity%2Falice",
+ "source": "公共知识库",
+ "type": "llmwiki_wiki",
+ "mode": "citation"
+}
+```
+
+注意:URL 名称可继续保持 `/weknora-source-preview` 以兼容现有解析器,但它实际上读取 DeerFlow 本地镜像。也可以新增更准确的 `/llmwiki-wiki-preview`,并同时兼容旧链接。实现时推荐新增链接格式并保留旧格式解析。
+
+### 12.2 不对外返回的字段
+
+外部 API 默认不返回:
+
+- `remote_page_id`;
+- WeKnora remote knowledge base ID;
+- `source_refs`;
+- `chunk_refs`;
+- 向量 BLOB;
+- Embedding 模型内部地址或 API Key;
+- owner user ID;
+- 同步错误原文;
+- 草稿/归档页面存在性。
+
+### 12.3 给大模型的上下文
+
+建议格式:
+
+```text
+
+[1] 页面标题
+知识库:xxx
+页面:entity/alice
+命中章节:主要经历
+相关度:0.8621
+正文:...
+
+[2] ...
+
+```
+
+要求模型:
+
+- 正文引用只能使用存在的 `[N]`;
+- 同一 Wiki 页面多个章节仍只对应一个编号;
+- 不把 `source_refs/chunk_refs` 暴露给模型;
+- 资料不足时明确说明;
+- 不自行把 Wiki 内容描述成原始一手文献。
+
+---
+
+## 13. 后端接口设计
+
+### 13.1 内部搜索接口
+
+建议新增:
+
+```http
+POST /api/llmwiki/wiki-vector-search
+Authorization: Bearer
+Content-Type: application/json
+```
+
+请求:
+
+```json
+{
+ "query": "任正非早期创业经历",
+ "knowledge_base_ids": ["deerflow-mapping-id"],
+ "top_k": 8,
+ "include_drafts": false,
+ "include_content": true
+}
+```
+
+约束:
+
+- `query` 1~4000 字符;
+- mapping IDs 最多 100;
+- `top_k` 1~50;
+- 服务端重新鉴权,不能相信前端选择;
+- `include_drafts` 仅所有者/管理员生效;
+- strict vector 下索引未就绪返回 409/503,不走 chunk search。
+
+响应:
+
+```json
+{
+ "query": "任正非早期创业经历",
+ "mode": "vector",
+ "results": [],
+ "knowledge_base_count": 1,
+ "index_fingerprint": "abc123...",
+ "partial": false,
+ "warnings": []
+}
+```
+
+### 13.2 本地 Wiki 页面详情
+
+```http
+GET /api/llmwiki/knowledge-bases/{mapping_id}/local-wiki/pages/{slug:path}
+```
+
+返回本地镜像,不访问 WeKnora。权限与现有 `get_authorized(write=false)` 一致。
+
+响应包含:
+
+- title/summary/content/page_type/status;
+- aliases/category/version/updated_at;
+- 索引状态和最近同步时间(管理者可见);
+- 普通引用视图不返回 source_refs/chunk_refs;
+- 管理详情可增加 `include_provenance=true`,仅所有者/管理员读取溯源。
+
+### 13.3 索引管理接口
+
+管理员接口:
+
+```http
+GET /api/llmwiki/wiki-index/status
+POST /api/llmwiki/wiki-index/sync
+POST /api/llmwiki/wiki-index/rebuild
+POST /api/llmwiki/wiki-index/knowledge-bases/{mapping_id}/sync
+POST /api/llmwiki/wiki-index/knowledge-bases/{mapping_id}/rebuild
+```
+
+`sync` 请求示例:
+
+```json
+{
+ "knowledge_base_ids": ["..."],
+ "force": false
+}
+```
+
+`rebuild` 请求示例:
+
+```json
+{
+ "knowledge_base_ids": ["..."],
+ "reason": "embedding_model_changed"
+}
+```
+
+长任务应返回 `202 Accepted + job_id`,不要在 HTTP 请求中等待整个知识库完成。
+
+### 13.4 外部向量搜索接口
+
+推荐路径:
+
+```http
+POST /api/external/llmwiki/wiki/vector-search
+X-API-Key:
+Content-Type: application/json
+```
+
+请求:
+
+```json
+{
+ "query": "任正非早期创业经历",
+ "knowledge_base_ids": [],
+ "limit": 10,
+ "include_content": false
+}
+```
+
+语义:
+
+- 空 `knowledge_base_ids` 表示所有允许外部搜索的公共 Wiki;
+- 非空列表必须是 DeerFlow mapping ID;
+- 请求私有/未开放 ID 时建议统一返回 404 或空结果,避免枚举资源;
+- 查询只使用本地向量;
+- 默认返回摘要和命中片段;
+- `include_content=true` 也要受单页和总响应字符上限约束。
+
+响应:
+
+```json
+{
+ "query": "任正非早期创业经历",
+ "mode": "vector",
+ "items": [
+ {
+ "knowledge_base_id": "public-mapping-id",
+ "knowledge_base_name": "人物知识库",
+ "wiki_slug": "entity/ren-zhengfei",
+ "title": "任正非",
+ "summary": "...",
+ "matched_heading": "早期创业",
+ "snippet": "...",
+ "page_type": "entity",
+ "page_version": 5,
+ "updated_at": "...",
+ "score": 0.89
+ }
+ ],
+ "count": 1,
+ "partial": false
+}
+```
+
+### 13.5 外部页面详情
+
+```http
+GET /api/external/llmwiki/wiki/pages/{mapping_id}/{slug:path}
+X-API-Key:
+```
+
+再次实时检查公共/外部开关和页面 published 状态,不能因为搜索结果曾经返回过就绕过当前权限。
+
+### 13.6 外部 API 鉴权接入
+
+当前 `/api/public/` 前缀会绕过全局登录鉴权,不适合直接承载未加保护的重型向量接口。推荐:
+
+1. 使用 `/api/external/`;
+2. 在 AuthMiddleware 中增加仅对该前缀生效的外部 API Key 分支;
+3. 使用 `hmac.compare_digest()` 常量时间比较;
+4. API Key 从环境变量解析,不存数据库明文;
+5. 缺失/错误统一返回 401,不透露配置状态;
+6. 日志只记 key 指纹前 8 位或完全不记;
+7. CORS 不作为服务到服务鉴权手段;
+8. 生产环境同时在 Nginx/API Gateway 做 IP、速率和请求体大小限制。
+
+如果为了少改 AuthMiddleware 而把接口挂到 `/api/public/`,路由自身也必须强制校验 API Key;但长期推荐独立 `/api/external/` 语义。
+
+### 13.7 状态码约定
+
+| 状态码 | 场景 |
+|---:|---|
+| 200 | 搜索成功,包括 0 结果 |
+| 202 | 同步/重建任务已接受 |
+| 400 | 参数格式错误 |
+| 401 | 外部 API Key 或登录凭据无效 |
+| 403 | 已认证但无操作权限 |
+| 404 | 映射/页面不存在或不可见 |
+| 409 | 索引未就绪、正在重建或模型指纹不一致 |
+| 413 | 查询/响应内容请求过大 |
+| 429 | 限流 |
+| 502 | 同步阶段 WeKnora 异常(搜索阶段不应返回) |
+| 503 | Embedding 服务或本地向量执行器不可用 |
+
+---
+
+## 14. 内部问答与工具接入
+
+### 14.1 普通问答预检索
+
+修改 `app/gateway/llmwiki_rag.py`:
+
+- 原 `build_weknora_client(...).search()` 替换为本地 WikiVectorSearchService;
+- 仍沿用对话选择和用户权限;
+- `_source_from_result()` 改为 Wiki 结构;
+- prompt 只包含命中章节;
+- `_MAX_SOURCES` 仍可保留 8;
+- `_MAX_SOURCE_CHARS` 应对每页所有命中章节的合计做限制;
+- 索引未就绪时给模型明确提示,不能声称“知识库没有内容”。
+
+### 14.2 智能体工具
+
+修改 `deerflow/tools/builtins/llmwiki_search_tool.py`:
+
+- 工具名可继续为 `llmwiki_search`,避免技能提示词迁移;
+- 实现改为查询 DeerFlow 本地 Wiki 索引;
+- 实时检查智能体绑定、对话选择和用户权限;
+- 返回 JSON 中 results 全部为 `kind=wiki_page`;
+- URL 只使用 mapping ID + wiki slug;
+- 不再返回原始 `chunk_id/knowledge_id/document_id` 作为主引用;
+- 工具描述改成“检索已处理的 Wiki 页面”。
+
+由于 harness 包不能 import `app.*`,本地索引 repository/search service 必须放在 `deerflow.*` 下,并通过 runtime setter 或应用初始化注入,不能让工具直接引用 Gateway router。
+
+### 14.3 页面检索接口
+
+现有 `POST /api/llmwiki/search` 可做兼容升级:
+
+- 默认 `source_kind=wiki`;
+- 新增 `mode=vector`;
+- Wiki 库走本地向量;
+- 标准非 Wiki 库是否保留原始检索需产品明确。本文默认:该接口如果明确选择标准库,可保留 `raw_chunk`,但问答中选中 Wiki 库时严格只查 Wiki;
+- 更清晰的做法是保留旧接口并新增 `/wiki-vector-search`,前端逐步迁移后再废弃旧模式。
+
+### 14.4 深度研究和 AI 写作
+
+它们通过 `llmwiki_search` 或共享参考文献归一化间接使用 WeKnora 时,应自然收到新的 Wiki result。需要回归:
+
+- `deep_research.adapters.material_provider`;
+- `deep_research.collection.thread_harvester`;
+- AI 写作 `_skill_agent.py` 参考文献归一化;
+- `deerflow.runtime.references` 对 raw 字段的保留。
+
+必须确保 `wiki_slug/page_id/matched_sections` 不在归一化过程中被丢弃。
+
+---
+
+## 15. 前端改造
+
+### 15.1 引用目标解析
+
+修改:
+
+```text
+frontend-web/src/core/llmwiki/source-links.ts
+```
+
+当前 preview URL 解析要求 `knowledge_base_id && chunk_id`。新逻辑应允许:
+
+```text
+knowledge_base_id && (wiki_slug || chunk_id || document_id)
+```
+
+优先级:
+
+1. `wiki_slug`;
+2. `chunk_id`(历史兼容);
+3. `document_id`(历史兼容)。
+
+### 15.2 右侧参考文献抽屉
+
+修改 `WeKnoraSourceDrawer.tsx` 或拆出 `LlmWikiWikiSourceDrawer`:
+
+当 target 有 `wikiSlug`:
+
+- 调用 DeerFlow 本地页面详情接口;
+- 不调用 WeKnora 文档 preview;
+- 不加载 document chunks;
+- 不展示“下载原文”作为主按钮;
+- 展示标题、摘要、页面类型、更新时间、版本、完整 Markdown;
+- 可高亮 `matched_heading`;
+- 可以有默认折叠的“支撑来源”入口,但仅登录且有权限时可用;
+- 外部公共页面不能借此读取 source_refs/chunk_refs。
+
+当 target 只有旧 `chunkId/documentId`:
+
+- 保留当前历史引用预览;
+- 可尝试本地反向映射到 Wiki;
+- 找不到则继续显示原文,不阻断历史聊天。
+
+### 15.3 参考文献卡片
+
+卡片显示:
+
+- Wiki 页面标题;
+- 知识库名称;
+- 页面类型;
+- 命中章节标题;
+- Wiki 摘要或命中片段;
+- 向量相关度;
+- 更新时间;
+- “查看 Wiki”而非“查看原文”。
+
+不要在卡片上显示:
+
+- 原文件名作为主标题;
+- chunk index;
+- WeKnora remote ID;
+- 向量维度/模型;
+- source_refs 原始 ID。
+
+### 15.4 管理页面建议
+
+在知识库详情或系统设置增加“本地向量索引”状态:
+
+- 已同步页面数;
+- 已索引页面数;
+- 向量章节数;
+- 最近同步时间;
+- 当前状态/错误;
+- Embedding 模型与短指纹;
+- “立即同步”;
+- “重建向量”;
+- “允许外部 API 检索”开关;
+- 重建进度。
+
+首版如果不做完整管理页,至少保证管理员 API 和日志可用。
+
+---
+
+## 16. 后端模块划分
+
+建议新增:
+
+```text
+packages/harness/deerflow/persistence/llmwiki_index/
+ __init__.py
+ base.py
+ model.py
+ memory.py
+ sql.py
+
+packages/harness/deerflow/integrations/weknora/local_index/
+ __init__.py
+ schemas.py
+ normalize.py
+ chunking.py
+ embedding.py
+ repository.py # 可直接使用 persistence store 时省略
+ vector_codec.py
+ vector_cache.py
+ search.py
+ sync.py
+ runtime.py
+
+app/gateway/routers/
+ llmwiki_index.py
+ external_llmwiki.py
+
+app/gateway/
+ llmwiki_index_scheduler.py
+```
+
+也可以将 local_index 放到 `deerflow/llmwiki_index/`,关键要求是:
+
+- harness 不 import app;
+- Gateway 负责 HTTP、用户解析、调度启动;
+- harness 负责数据模型、同步核心、向量编码、缓存和搜索;
+- WeKnora client 仍只负责官方 REST 适配;
+- 路由不直接写 SQL。
+
+### 16.1 Store 接口建议
+
+```python
+class LlmWikiIndexStore(ABC):
+ async def get_page(...): ...
+ async def list_pages(...): ...
+ async def stage_or_replace_page_vectors(...): ...
+ async def mark_page_failed(...): ...
+ async def mark_missing_pages_deleted(...): ...
+ async def load_vector_snapshot(mapping_id, fingerprint): ...
+ async def get_index_revision(mapping_id): ...
+ async def try_acquire_sync_lease(...): ...
+ async def renew_sync_lease(...): ...
+ async def finish_sync(...): ...
+ async def list_index_status(...): ...
+```
+
+提供 SQL 和 memory 实现,便于单元测试及 auth-disabled 开发模式。
+
+### 16.2 应用初始化
+
+在 Gateway lifespan 中:
+
+1. 创建/取得 index store;
+2. 创建严格 Embedding client;
+3. 创建 WikiVectorCache;
+4. 创建 WikiVectorSearchService;
+5. 创建 WikiSyncService;
+6. 挂到 `app.state`;
+7. 给 harness runtime setter 注入只读 search/store;
+8. 若 sync enabled,启动调度 loop;
+9. shutdown 时停止调度并清理缓存/HTTP client。
+
+调度器需要和现有 lifespan 后台任务保持相同的取消与超时风格。
+
+---
+
+## 17. 数据库迁移与兼容性
+
+### 17.1 Alembic 迁移
+
+新增迁移:
+
+```text
+deerflow/persistence/migrations/versions/_01_llmwiki_local_wiki_index.py
+```
+
+迁移内容:
+
+1. 扩展 `llmwiki_knowledge_bases`;
+2. 创建 `llmwiki_wiki_pages`;
+3. 创建 `llmwiki_wiki_vectors`;
+4. 创建 `llmwiki_wiki_sync_states`;
+5. 建立 FK、唯一约束和索引;
+6. 所有现有 mapping 的 `external_search_enabled=false`;
+7. `wiki_index_enabled=true` 或按配置决定;
+8. 不在迁移过程中调用 WeKnora 或 Embedding。
+
+迁移必须兼容 SQLite、PostgreSQL 和 MySQL。注意 MySQL utf8mb4 长索引限制,slug 使用 hash 唯一约束。
+
+### 17.2 向量 BLOB 可移植性
+
+- SQLite:BLOB;
+- PostgreSQL:BYTEA;
+- MySQL:LONGBLOB/BLOB,按最大维度选择;
+- SQLAlchemy 使用 `LargeBinary`;
+- 字节序固定 little-endian;
+- 模型维度显式保存;
+- 不依赖数据库特定 vector 类型,便于首版跨后端运行。
+
+### 17.3 首次回填
+
+迁移完成后不自动阻塞启动进行全量回填。建议:
+
+1. 服务正常启动;
+2. 管理员查看状态为 `not_synced`;
+3. 手动或后台分库同步;
+4. 同步完成一个库就允许该库向量检索;
+5. 全部稳定后再将问答默认切换为 local wiki vector;
+6. 外部 API 最后开启。
+
+### 17.4 历史引用
+
+历史消息中的引用仍可能只有 chunk/document。必须保留:
+
+- 旧 URL 解析;
+- 旧 source drawer;
+- `/api/llmwiki/sources/context`;
+- 原文预览权限。
+
+新消息只生成 Wiki 引用。不要批量重写历史消息,除非后续建立可靠 `chunk_ref → local wiki page` 反向表并单独实施迁移。
+
+---
+
+## 18. 外部 API 安全设计
+
+### 18.1 威胁模型
+
+需要防范:
+
+- 枚举私有 mapping ID;
+- 通过 query 大小或 top_k 放大 CPU/Embedding 成本;
+- 高频调用耗尽 Embedding 服务配额;
+- 请求全部公共库导致加载大量快照;
+- 返回完整 Wiki 内容造成带宽放大;
+- Markdown 中的 HTML/script 被下游不安全渲染;
+- API Key 泄漏;
+- 关闭发布后缓存继续返回;
+- “对话沉淀”误开放;
+- 错误日志包含正文或敏感标识。
+
+### 18.2 控制措施
+
+1. 外部 API Key 必填;
+2. query 字符数、mapping 数、top_k、响应字符数硬上限;
+3. 请求体大小由应用和反向代理共同限制;
+4. 限流按 API Key + IP;
+5. 查询向量短 TTL 缓存可减少重复 embedding,但缓存 key 不保存原文日志;
+6. 外部查询只加载明确开放的 KB;
+7. 权限在检索前和返回前各检查一次;
+8. 页面详情接口再次检查;
+9. 返回 Markdown 时标注 content type,前端渲染必须禁用危险 HTML;
+10. 不返回 remote ID、source refs、chunk refs、owner 信息;
+11. 日志对 query 做长度裁剪或 hash;
+12. “对话沉淀”代码级 deny;
+13. API Key 支持轮换:可配置 current + previous,短时间双 key 过渡;
+14. 对外接口默认关闭。
+
+### 18.3 Embedding 成本控制
+
+外部每次查询需要一次 embedding。建议:
+
+- 对规范化 query 做 1~5 分钟进程内 LRU 缓存;
+- 缓存 key 使用 `embedding_fingerprint + sha256(query)`;
+- 缓存值是查询向量,不是结果权限;
+- 权限和结果仍每次实时计算;
+- 不跨 fingerprint 复用;
+- 缓存大小有上限;
+- API Key 限流在调用 Embedding 前执行。
+
+---
+
+## 19. 失败处理与一致性
+
+| 故障 | 预期行为 | 是否回退原文 |
+|---|---|---:|
+| WeKnora 同步超时 | 保留旧镜像/向量,状态 partial/failed | 否 |
+| 某页读取失败 | 旧页继续可用,记录页面错误 | 否 |
+| Embedding 超时 | 不覆盖旧向量,重试 | 否 |
+| Embedding 401/403 | 标记配置错误,停止无意义重试 | 否 |
+| 返回向量数量不一致 | 本批失败,不落库 | 否 |
+| 维度变化 | 标记 rebuild_required | 否 |
+| BLOB 损坏 | 跳过损坏库/页并告警,触发重建 | 否 |
+| NumPy 快照加载失败 | 保留旧快照;无旧快照则索引不可用 | 否 |
+| 单个 KB 搜索失败 | 内部可返回 partial + warning;外部默认 partial | 否 |
+| 所有 KB 不可用 | 返回 409/503 | 否 |
+| 取消公共发布 | 权限层立即过滤 | 不适用 |
+| 外部开关关闭 | 权限层立即过滤 | 不适用 |
+| Gateway 重启 | 从 DB 懒加载,无需重做 embedding | 不适用 |
+
+### 19.1 “没有命中”和“索引不可用”必须区分
+
+- 正常 0 结果:HTTP 200,`results=[]`;
+- 未同步:409,错误码 `WIKI_INDEX_NOT_READY`;
+- 正在重建:409,`WIKI_INDEX_REBUILDING`;
+- Embedding 服务失败:503,`WIKI_EMBEDDING_UNAVAILABLE`;
+- 指纹不匹配:409,`WIKI_INDEX_FINGERPRINT_MISMATCH`;
+- 无权限/不存在:404 或权限语义对应状态。
+
+模型提示也必须区分,避免把系统故障回答成“知识库没有相关资料”。
+
+---
+
+## 20. 可观测性
+
+### 20.1 结构化日志
+
+同步日志字段:
+
+```text
+mapping_id
+sync_id
+page_number
+remote_page_count
+changed_pages
+unchanged_pages
+deleted_pages
+embedded_chunks
+failed_pages
+duration_ms
+error_category
+```
+
+检索日志字段:
+
+```text
+request_id
+caller_type (internal/external/tool/pre_rag)
+authorized_kb_count
+loaded_snapshot_count
+candidate_section_count
+result_page_count
+embedding_ms
+search_ms
+total_ms
+partial
+```
+
+不得记录:API Key、完整 query、Wiki 全文、向量。
+
+### 20.2 指标
+
+建议指标:
+
+- `llmwiki_sync_runs_total{status}`;
+- `llmwiki_sync_duration_seconds`;
+- `llmwiki_pages_total{status,index_status}`;
+- `llmwiki_vectors_total`;
+- `llmwiki_embedding_requests_total{status}`;
+- `llmwiki_embedding_duration_seconds`;
+- `llmwiki_search_requests_total{caller,status}`;
+- `llmwiki_search_duration_seconds{stage}`;
+- `llmwiki_search_results_count`;
+- `llmwiki_vector_cache_hits_total`;
+- `llmwiki_vector_cache_reloads_total`;
+- `llmwiki_vector_cache_bytes`;
+- `llmwiki_external_rate_limited_total`。
+
+若当前没有 Prometheus,可先接现有日志/工具指标体系,并保留后续扩展点。
+
+### 20.3 状态接口
+
+管理员状态页至少能回答:
+
+- 哪些库尚未同步;
+- 哪些库正在同步;
+- 哪些页面失败;
+- 当前模型指纹;
+- 是否需要重建;
+- 最近一次完整成功时间;
+- 本地页数和向量数;
+- 缓存是否加载;
+- 外部开放是否开启。
+
+---
+
+## 21. 性能与扩展路线
+
+### 21.1 首版基准测试要求
+
+用真实或合成数据覆盖:
+
+- 1,000 / 10,000 / 50,000 / 100,000 章节;
+- 768 / 1,024 / 1,536 维;
+- 单库与 5/20 库并发查询;
+- 1/4 Gateway workers;
+- 冷加载与热缓存;
+- 同步和搜索并行;
+- SQLite 与生产数据库后端。
+
+记录:
+
+- 冷加载时间;
+- 单次向量计算 P50/P95/P99;
+- 进程 RSS;
+- DB BLOB 读取吞吐;
+- Embedding 延迟;
+- 多并发下事件循环阻塞情况。
+
+NumPy 矩阵计算和大 BLOB 解码应放入 `asyncio.to_thread()`,避免阻塞事件循环。
+
+### 21.2 何时迁移 pgvector
+
+出现以下任一情况时评估 pgvector:
+
+- 单库快照内存不可接受;
+- 多 worker 重复内存过高;
+- 冷启动/重载时间过长;
+- 向量数量达到数十万以上且查询并发提高;
+- 需要数据库端按 mapping/status 过滤并近似索引;
+- 需要多节点共享统一向量执行层。
+
+迁移时保持:
+
+- 页面表不变;
+- 统一结果协议不变;
+- 权限服务不变;
+- API 不变;
+- 只替换 `WikiVectorEngine` 实现。
+
+建议抽象:
+
+```python
+class WikiVectorEngine(ABC):
+ async def search(self, query_vector, mapping_ids, top_k, fingerprint): ...
+ async def invalidate(self, mapping_id): ...
+```
+
+首版 `NumpyExactWikiVectorEngine`,未来 `PgVectorWikiVectorEngine`。
+
+---
+
+## 22. 测试方案
+
+### 22.1 单元测试
+
+1. Markdown 标题切分、长段落、表格、空页、frontmatter;
+2. title/alias/heading 前缀构造;
+3. chunk hash 与页面 hash 稳定性;
+4. float32 BLOB 编解码;
+5. NaN/Inf/维度错误拒绝;
+6. 向量归一化;
+7. NumPy TopK 与手工余弦结果一致;
+8. 页面聚合和每页章节上限;
+9. min similarity;
+10. LRU 和 revision 失效;
+11. API Key 常量时间校验;
+12. 系统“对话沉淀”排除;
+13. embedding fingerprint;
+14. 错误脱敏。
+
+### 22.2 Repository 测试
+
+使用 SQLite 测:
+
+- 创建页面/向量;
+- 原子替换;
+- FK cascade;
+- unique mapping+slug_hash;
+- 完整扫描后删除缺失页;
+- 不完整扫描不删除;
+- lease 获取/续租/过期接管;
+- revision 只在检索数据变化时递增;
+- public/external 开关默认值;
+- BLOB round trip。
+
+生产若使用 MySQL/PostgreSQL,补容器集成测试验证字段和索引兼容性。
+
+### 22.3 同步集成测试
+
+使用假的 WeKnora client:
+
+1. 首次 3 页同步并建立向量;
+2. 第二次无变化不调用 embedding;
+3. 一页 version/content 变化只重建该页;
+4. 一页归档后不再检索;
+5. 一页删除后清理向量;
+6. 分页第 2 页失败不误删本地页;
+7. Embedding 失败保留旧索引;
+8. 多 worker 只有一个获得租约;
+9. fingerprint 改变标记重建;
+10. 服务重启后从 DB 恢复快照。
+
+### 22.4 检索与权限测试
+
+1. 查询仅存在于 Wiki、原始文档无该措辞,仍能命中;
+2. 查询只存在于原始 chunk、Wiki 不包含,严格模式不命中;
+3. 用户 A 不能检索用户 B 私有库;
+4. 公共库对登录用户可检索;
+5. 智能体未绑定库不能越权;
+6. 对话明确空选择不检索;
+7. 撤销公共发布后下一请求立即无结果;
+8. draft 默认不命中;
+9. 所有者明确 include_drafts 可以命中;
+10. archived/deleted 永不命中;
+11. 同一页多个章节只产生一个引用编号;
+12. 多库同 slug 不互相覆盖。
+
+### 22.5 外部 API 安全测试
+
+1. 无 key/错 key → 401;
+2. 私有库/未外部开放 → 不可枚举;
+3. public 但 `external_search_enabled=false` → 不返回;
+4. draft/archived → 不返回;
+5. “对话沉淀”永不返回;
+6. 关闭外部开关立即失效;
+7. 超长 query → 400/413;
+8. 超大 top_k → 拒绝或钳制;
+9. 高频请求 → 429;
+10. 响应没有 remote ID/source_refs/chunk_refs/vector;
+11. 日志没有 API Key 和正文;
+12. 详情接口重新鉴权,不依赖旧搜索结果。
+
+### 22.6 前端测试
+
+1. `knowledge_base_id + wiki_slug` 可解析为有效 target;
+2. Wiki target 不请求文档 preview/chunks;
+3. 抽屉展示本地 Wiki Markdown;
+4. 卡片显示 Wiki 标题和命中章节;
+5. 历史 chunk 引用仍能打开;
+6. 权限撤销后详情返回 404,前端正确提示;
+7. Wiki 内容 Markdown/HTML 安全渲染;
+8. loading/error/empty 状态。
+
+### 22.7 性能测试
+
+至少验证:
+
+- 50k × 1024 维单库热查询 P95;
+- 5 个库合并查询;
+- 冷加载内存和耗时;
+- 10/50 并发查询;
+- 同步时搜索延迟;
+- 多 worker 总内存;
+- 查询 embedding 缓存命中效果。
+
+验收阈值应在目标部署机器基准后写入配置/运维文档,不在未测量前拍脑袋承诺具体毫秒数。
+
+---
+
+## 23. 分阶段实施计划
+
+### 阶段 0:契约与配置
+
+- 增加配置模型;
+- 定义统一 result schema;
+- 定义错误码;
+- 增加 feature flag,默认关闭;
+- 确认 Embedding endpoint/model/dimensions。
+
+完成标准:应用在功能关闭时行为完全不变。
+
+### 阶段 1:本地镜像与同步
+
+- 新增迁移和三张表;
+- 实现 SQL/memory store;
+- WeKnora 页面规范化;
+- 完整扫描、页面 upsert、缺失删除;
+- 租约、状态、手动同步 API;
+- 暂不切换问答。
+
+完成标准:DeerFlow DB 中 Wiki 页面和 WeKnora 一致,可独立读取详情。
+
+### 阶段 2:Embedding 与本地向量
+
+- 严格 Embedding client;
+- Markdown chunks;
+- BLOB codec;
+- 页面原子换代;
+- rebuild;
+- 状态与失败重试。
+
+完成标准:所有 ready 页面有正确 fingerprint 和向量;无变化同步不重复 embedding。
+
+### 阶段 3:NumPy 检索服务
+
+- per-KB snapshot;
+- LRU/revision;
+- TopK/阈值/聚合;
+- 权限过滤;
+- 内部搜索 API;
+- 性能基准。
+
+完成标准:可稳定从本地向量返回 Wiki 页与命中章节。
+
+### 阶段 4:问答和参考文献切换
+
+- 普通问答预检索;
+- `llmwiki_search` 工具;
+- 页面检索;
+- runtime references;
+- 前端 source links/drawer/card;
+- 历史兼容。
+
+完成标准:新问答不再产生原始文档 chunk 引用,右侧只显示 Wiki 主视图。
+
+### 阶段 5:外部公共 API
+
+- 外部开关;
+- `/api/external` 鉴权;
+- vector-search/page detail;
+- 限流/响应上限;
+- “对话沉淀”强制排除;
+- 安全测试。
+
+完成标准:仅明确外部开放的公共 published Wiki 可被持 key 系统检索。
+
+### 阶段 6:运维与灰度
+
+- 管理状态;
+- 指标/告警;
+- 分库灰度;
+- 容量评估;
+- 故障演练;
+- 回滚验证。
+
+---
+
+## 24. 灰度与回滚
+
+### 24.1 Feature flags
+
+建议至少有:
+
+```text
+local_wiki_index.enabled
+local_wiki_index.sync_enabled
+local_wiki_index.internal_search_enabled
+local_wiki_index.reference_ui_enabled
+local_wiki_index.external_api.enabled
+```
+
+可以逐步:
+
+1. 只同步;
+2. 只提供管理员搜索对比;
+3. 指定测试用户/知识库使用向量;
+4. 全量内部问答;
+5. 引用 UI;
+6. 外部 API。
+
+### 24.2 双跑对比
+
+灰度期可在不影响回答的情况下后台双跑:
+
+- 旧 chunk search;
+- 新 local wiki vector search。
+
+只记录安全的对比指标:
+
+- 命中数;
+- 标题集合 hash;
+- TopK 重合率;
+- 查询耗时;
+- 是否为空;
+- 人工抽样质量。
+
+不要把两套内容同时注入模型,以免引用编号混乱。
+
+### 24.3 回滚
+
+出现问题时:
+
+- 关闭 internal search flag,可恢复旧 chunk 检索;
+- 关闭 reference UI flag,可恢复旧抽屉;
+- 本地页面/向量表保留,便于排查;
+- 关闭 sync 不删除数据;
+- 外部 API 可独立关闭;
+- 数据库迁移不需要立即 downgrade;
+- 新消息引用格式解析器应向后兼容,避免回滚后打不开已生成的新 Wiki 引用。
+
+注意:产品最终目标是严格 Wiki 检索。旧 chunk 回滚仅用于上线故障处置,不应成为长期隐式 fallback。
+
+---
+
+## 25. 验收标准
+
+### 25.1 数据同步
+
+- 所有目标 Wiki 页面在 DeerFlow 有镜像;
+- title/summary/content/slug/version/status 与上游一致;
+- 无变化不重复 embedding;
+- 单页更新只重建该页;
+- 完整扫描失败不误删;
+- 删除/归档后搜索立即排除;
+- Gateway 重启不丢数据。
+
+### 25.2 向量检索
+
+- 查询向量只生成一次;
+- 检索只使用 DeerFlow 本地 Wiki vectors;
+- 不访问 WeKnora search;
+- TopK 返回相关 Wiki 章节;
+- 同页多章节只占一个引用;
+- 索引未就绪不会误报“无资料”;
+- Embedding 失败不回退原始文档。
+
+### 25.3 问答与参考文献
+
+- 普通问答、智能体工具、页面搜索结果一致;
+- 新引用携带 `wiki_page_id/wiki_slug`;
+- 右侧展示完整 Wiki 页面;
+- 不自动展示整个原文文档;
+- 历史 chunk 引用仍可用;
+- 撤权后引用详情不可访问。
+
+### 25.4 外部接口
+
+- 只返回 published + external enabled;
+- 私有/草稿/归档/对话沉淀永不泄漏;
+- 不暴露 remote IDs、vectors、source refs;
+- key、限流、请求上限生效;
+- 取消发布或关闭外部开关立即生效;
+- WeKnora 不可用时已有本地索引仍可查询。
+
+---
+
+## 26. 风险清单与处理
+
+| 风险 | 影响 | 处理 |
+|---|---|---|
+| WeKnora 没有增量变更 feed | 需要完整分页扫描 | 周期完整扫描 + BFF 操作即时同步;后续如上游支持 cursor 再优化 |
+| Wiki 生成有延迟 | 原文已上传但本地暂无 Wiki | 显示 not ready,不回退原文 |
+| Embedding 服务不稳定 | 新页面无法入索引 | 保留旧索引、严格错误、后台重试 |
+| 模型维度变化 | 新旧向量不可比较 | fingerprint + 全量重建 |
+| 大库内存高 | 多 worker OOM | per-KB LRU、容量测试、后续 pgvector |
+| SQLite 写锁 | 同步与业务写竞争 | 小事务、批量写、WAL、低并发同步;生产大规模用 Postgres/MySQL |
+| 页面和向量版本不一致 | 引用内容与召回语义错位 | 原子换代 |
+| 公共库语义扩大 | 历史内部公共内容外泄 | 独立 external flag,默认关闭 |
+| 对话沉淀误开放 | 用户对话泄漏 | 代码级 deny |
+| 历史引用无 slug | 不能自动展示 Wiki | 保留旧引用兼容,不强制迁移 |
+| Markdown 恶意 HTML | 下游 XSS | API 无执行;前端安全渲染/禁危险 HTML |
+| 多 worker 重复同步 | 重复 Embedding/写冲突 | DB 租约 |
+| 缓存旧权限 | 取消发布后仍可查 | 权限不缓存或短缓存;返回前复核 |
+
+---
+
+## 27. 文件级实施清单
+
+### 后端配置与模型
+
+- `deerflow/config/llmwiki_config.py`:新增 local index/embedding/external 配置;
+- `deerflow/persistence/llmwiki/model.py`:mapping 外部开关;
+- `deerflow/persistence/llmwiki/{base,memory,sql}.py`:读写新开关;
+- `deerflow/persistence/llmwiki_index/*`:新增页面、向量、同步状态 store;
+- Alembic migration:新增表与字段;
+- `deerflow/persistence/models/__init__.py`:注册模型。
+
+### 同步与向量
+
+- `deerflow/integrations/weknora/client.py`:确保 Wiki list/get 字段完整,保留 `chunk_refs`;
+- `deerflow/integrations/weknora/local_index/normalize.py`:规范化;
+- `.../chunking.py`:Wiki 专用 chunks;
+- `.../embedding.py`:严格 Embedding;
+- `.../vector_codec.py`:BLOB;
+- `.../sync.py`:同步、租约、原子换代;
+- `.../vector_cache.py`:NumPy 快照;
+- `.../search.py`:权限后的本地向量检索;
+- `app/gateway/llmwiki_index_scheduler.py`:周期任务;
+- `app/gateway/app.py`:初始化/关闭。
+
+### 路由与业务接入
+
+- `app/gateway/routers/llmwiki_index.py`:内部搜索/状态/同步/重建;
+- `app/gateway/routers/external_llmwiki.py`:外部搜索/详情;
+- `app/gateway/auth_middleware.py`:外部 API Key 分支;
+- `app/gateway/llmwiki_rag.py`:预检索切换;
+- `deerflow/tools/builtins/llmwiki_search_tool.py`:工具切换;
+- `app/gateway/routers/llmwiki.py`:页面搜索、详情和外部开关接入;
+- `deerflow/runtime/references.py`:保留 Wiki 结构字段;
+- deep research / AI writing:回归兼容。
+
+### 前端
+
+- `frontend-web/src/core/llmwiki/source-links.ts`;
+- `frontend-web/src/components/workspace/skill-display-sources.tsx`;
+- `frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx`;
+- `frontend-web/src/strategy-components/api/llmwiki.ts`;
+- 知识库详情/系统设置中的索引状态和外部开关(如纳入首版 UI)。
+
+### 测试
+
+- 新增 `tests/test_llmwiki_local_index_repository.py`;
+- 新增 `tests/test_llmwiki_wiki_sync.py`;
+- 新增 `tests/test_llmwiki_vector_search.py`;
+- 新增 `tests/test_external_llmwiki_search.py`;
+- 扩展 `tests/test_llmwiki_weknora.py`;
+- 前端增加 source target / drawer 测试;
+- 增加性能基准脚本,不放入默认每次单测。
+
+---
+
+## 28. 实现前必须确认的部署输入
+
+编码本身可以先做,但上线前必须明确:
+
+1. DeerFlow 可调用的 OpenAI-compatible Embedding 地址;
+2. 模型名称;
+3. 向量维度;
+4. 最大 Wiki 页面数、平均页面长度、预计章节数;
+5. Gateway worker 数;
+6. 生产数据库是 SQLite、PostgreSQL 还是 MySQL;
+7. 外部接口是否只给内网系统,是否有统一 API Gateway;
+8. 外部 API Key 的生成、分发和轮换责任;
+9. 哪些现有公共库允许外部开放;
+10. 同步允许的最大延迟,例如 1 分钟或 5 分钟。
+
+若没有额外输入,开发默认使用:
+
+- 本地/测试 SQLite;
+- bge-m3 类 1024 维模型(实际以配置为准);
+- 120 秒轮询;
+- NumPy exact;
+- 外部 API 默认关闭;
+- strict vector;
+- 每页最多 2 个命中章节;
+- Top 8 页面。
+
+---
+
+## 29. 最终推荐
+
+本需求不应被实现成“把 WeKnora 的搜索接口换个 URL”,也不应通过隐藏文档库重复导入来绕过同步问题。推荐把它建设为 DeerFlow 内部的正式 Wiki 检索子系统:
+
+1. WeKnora 负责生成;
+2. DeerFlow 完整镜像;
+3. DeerFlow 按章节向量化;
+4. DeerFlow 本地持久化;
+5. DeerFlow 统一检索和权限;
+6. 引用始终落到 Wiki 页面;
+7. 外部接口只查询明确开放的公共 Wiki;
+8. 未来只替换向量执行器即可扩容,不推翻页面、权限、API 和引用设计。
+
+该方案能同时满足:不修改 WeKnora 后端、检索处理后的 Wiki、右侧展示 Wiki、向量语义召回、公共 Wiki 外部检索、WeKnora 查询故障隔离,以及后续向 pgvector/分布式向量引擎平滑演进。
+
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/fixed_answer_agent.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/fixed_answer_agent.py
new file mode 100644
index 00000000..222f9be8
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/fixed_answer_agent.py
@@ -0,0 +1,186 @@
+"""Small local streaming graph used for exact-match fixed answers.
+
+The graph deliberately uses a local ``BaseChatModel`` implementation so
+LangGraph emits the same token/message stream events as a normal model-backed
+answer, while making no network or model-provider request.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import codecs
+import math
+import uuid
+from collections.abc import AsyncIterator, Sequence
+from typing import Any
+
+import tiktoken
+from langchain_core.callbacks import AsyncCallbackManagerForLLMRun, CallbackManagerForLLMRun
+from langchain_core.language_models.chat_models import BaseChatModel
+from langchain_core.messages import AIMessage, AIMessageChunk, BaseMessage
+from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult
+from langchain_core.runnables import RunnableConfig
+from langgraph.graph import END, START, StateGraph
+
+from deerflow.agents.thread_state import ThreadState
+
+_TOKEN_ENCODING = tiktoken.get_encoding("cl100k_base")
+_TARGET_CHUNKS_PER_SECOND = 10
+_MAX_TOKENS_PER_CHUNK = 20
+_MAX_TOKENS_PER_SECOND = 1_000
+
+
+class FixedAnswerChatModel(BaseChatModel):
+ """A local streaming model that only replays an administrator answer."""
+
+ answer: str
+ message_id: str
+ fixed_question_id: str
+ tokens_per_second: int = 100
+
+ @property
+ def _llm_type(self) -> str:
+ return "fixed-answer"
+
+ @property
+ def _identifying_params(self) -> dict[str, Any]:
+ return {
+ "fixed_question_id": self.fixed_question_id,
+ "tokens_per_second": self.tokens_per_second,
+ }
+
+ def _message(self) -> AIMessage:
+ return AIMessage(
+ content=self.answer,
+ id=self.message_id,
+ additional_kwargs={
+ "fixed_answer": True,
+ "fixed_question_id": self.fixed_question_id,
+ "tokens_per_second": self.tokens_per_second,
+ },
+ )
+
+ def _generate(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: CallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> ChatResult:
+ del messages, stop, run_manager, kwargs
+ return ChatResult(generations=[ChatGeneration(message=self._message())])
+
+ async def _astream(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: AsyncCallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> AsyncIterator[ChatGenerationChunk]:
+ del messages, stop, run_manager, kwargs
+ answer = self.answer
+ if not answer:
+ yield ChatGenerationChunk(message=AIMessageChunk(content="", id=self.message_id))
+ return
+ token_ids = _TOKEN_ENCODING.encode(answer, disallowed_special=())
+ tokens_per_second = max(
+ 1,
+ min(_MAX_TOKENS_PER_SECOND, self.tokens_per_second),
+ )
+ chunk_size = max(
+ 1,
+ min(
+ _MAX_TOKENS_PER_CHUNK,
+ math.ceil(tokens_per_second / _TARGET_CHUNKS_PER_SECOND),
+ ),
+ )
+ decoder = codecs.getincrementaldecoder("utf-8")()
+ token_groups = [token_ids[:1], *(token_ids[offset : offset + chunk_size] for offset in range(1, len(token_ids), chunk_size))]
+ for index, chunk_token_ids in enumerate(token_groups):
+ if index > 0:
+ await asyncio.sleep(len(chunk_token_ids) / tokens_per_second)
+ chunk_bytes = b"".join(_TOKEN_ENCODING.decode_single_token_bytes(token_id) for token_id in chunk_token_ids)
+ is_last = index == len(token_groups) - 1
+ content = decoder.decode(chunk_bytes, final=is_last)
+ chunk_kwargs = (
+ {
+ "fixed_answer": True,
+ "fixed_question_id": self.fixed_question_id,
+ "tokens_per_second": tokens_per_second,
+ }
+ if index == 0
+ else {}
+ )
+ yield ChatGenerationChunk(
+ message=AIMessageChunk(
+ content=content,
+ id=self.message_id,
+ additional_kwargs=chunk_kwargs,
+ )
+ )
+
+
+def _fallback_title(question: str) -> str:
+ if len(question) > 50:
+ return question[:50].rstrip() + "..."
+ return question or "New Conversation"
+
+
+def make_fixed_answer_agent(
+ *,
+ question: str,
+ answer: str,
+ fixed_question_id: str,
+ tokens_per_second: int = 100,
+):
+ """Build a one-node graph that streams and persists a fixed answer."""
+
+ tokens_per_second = max(
+ 1,
+ min(_MAX_TOKENS_PER_SECOND, int(tokens_per_second)),
+ )
+ message_id = f"fixed-answer-{uuid.uuid4()}"
+ model = FixedAnswerChatModel(
+ answer=answer,
+ message_id=message_id,
+ fixed_question_id=fixed_question_id,
+ tokens_per_second=tokens_per_second,
+ ).with_config(tags=["fixed-answer"])
+
+ async def respond(
+ state: ThreadState,
+ config: RunnableConfig,
+ ) -> dict[str, Any]:
+ messages: Sequence[BaseMessage] = state.get("messages", [])
+ # Calling the local model through ``astream`` makes the runtime publish
+ # normal messages-mode chunks. The full AIMessage returned below is the
+ # durable checkpoint value used on refresh/history reads.
+ async for _chunk in model.astream(list(messages), config=config):
+ pass
+ update: dict[str, Any] = {
+ "messages": [
+ AIMessage(
+ content=answer,
+ id=message_id,
+ additional_kwargs={
+ "fixed_answer": True,
+ "fixed_question_id": fixed_question_id,
+ "tokens_per_second": tokens_per_second,
+ },
+ )
+ ]
+ }
+ if not state.get("title"):
+ update["title"] = _fallback_title(question)
+ update["title_provisional"] = False
+ elif state.get("title_provisional") is True:
+ # A fixed-answer run must not trigger the worker's background LLM
+ # title finalizer either; keep the already-visible local title.
+ update["title_provisional"] = False
+ return update
+
+ graph = StateGraph(ThreadState)
+ graph.add_node("fixed_answer", respond)
+ graph.add_edge(START, "fixed_answer")
+ graph.add_edge("fixed_answer", END)
+ return graph.compile()
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/prompt.py b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/prompt.py
index 470e706f..5d46c25c 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/prompt.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/agents/lead_agent/prompt.py
@@ -271,7 +271,42 @@ answers — and never create a skill silently without user confirmation.
"""
-def _build_available_subagents_description(available_names: list[str], bash_available: bool, *, app_config: AppConfig | None = None) -> str:
+def _build_es_query_routing_section(
+ *,
+ app_config: AppConfig | None = None,
+ available_skills: set[str] | None = None,
+) -> str:
+ """Return the configurable ``es_query`` routing rule for the system prompt.
+
+ A custom agent with an explicit skill allowlist only receives the rule when
+ ``es_query`` is in that allowlist. The default lead agent has no allowlist,
+ so the operator-controlled config switch is authoritative there.
+ """
+ if available_skills is not None and "es_query" not in available_skills:
+ return ""
+
+ config = app_config
+ if config is None:
+ try:
+ from deerflow.config import get_app_config
+
+ config = get_app_config()
+ except Exception:
+ logger.debug("Failed to load es_query routing config", exc_info=True)
+ return ""
+
+ routing = getattr(getattr(config, "skills", None), "es_query_routing", None)
+ if routing is None or not routing.enabled:
+ return ""
+
+ prompt = (routing.prompt or "").strip()
+ if not prompt:
+ return ""
+
+ return f'\n{prompt}\n'
+
+
+def _build_available_subagents_description(available_names: list[str], bash_available: bool, *, app_config: AppConfig | None = None) -> str:
"""Dynamically build subagent type descriptions from registry.
Mirrors Codex's pattern where agent_type_description is dynamically generated
@@ -548,9 +583,11 @@ User: "staging"
You: "Deploying to staging..." [proceed]
-{skills_section}
-
-{bootstrap_section}
+{skills_section}
+
+{es_query_routing_section}
+
+{bootstrap_section}
{deferred_tools_section}
@@ -663,10 +700,12 @@ combined with a FastAPI gateway for REST API access [citation:FastAPI](https://f
# style, clarification system, citation rules, response-style / critical-reminder
# defaults) and no personal memory injection. Only the mechanical sections an
# agent still needs to actually use its configured skills/tools are kept.
-AGENT_ONLY_PROMPT_TEMPLATE = """{soul}
-{network_policy_section}{skills_section}
-
-{deferred_tools_section}
+AGENT_ONLY_PROMPT_TEMPLATE = """{soul}
+{network_policy_section}{skills_section}
+
+{es_query_routing_section}
+
+{deferred_tools_section}
{writing_mode_section}
@@ -719,12 +758,16 @@ def apply_agent_only_prompt_template(
soul = get_agent_soul(storage_agent_id)
- skills_section = get_skills_prompt_section(
- available_skills,
- app_config=app_config,
- display_enabled=rag_mode_enabled,
- )
- deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
+ skills_section = get_skills_prompt_section(
+ available_skills,
+ app_config=app_config,
+ display_enabled=rag_mode_enabled,
+ )
+ es_query_routing_section = _build_es_query_routing_section(
+ app_config=app_config,
+ available_skills=available_skills,
+ )
+ deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
n = max_concurrent_subagents
subagent_section = _build_subagent_section(n, app_config=app_config) if subagent_enabled else ""
@@ -766,9 +809,10 @@ RAG/reference display mode is enabled.
)
prompt = AGENT_ONLY_PROMPT_TEMPLATE.format(
- soul=soul,
- skills_section=skills_section,
- deferred_tools_section=deferred_tools_section,
+ soul=soul,
+ skills_section=skills_section,
+ es_query_routing_section=es_query_routing_section,
+ deferred_tools_section=deferred_tools_section,
writing_mode_section=writing_mode_section,
subagent_section=subagent_section,
network_policy_section=network_policy_section,
@@ -1377,11 +1421,15 @@ def apply_prompt_template(
bootstrap_section = _build_bootstrap_section() if is_bootstrap else ""
# Get skills section
- skills_section = get_skills_prompt_section(
- available_skills,
- app_config=app_config,
- display_enabled=rag_mode_enabled,
- )
+ skills_section = get_skills_prompt_section(
+ available_skills,
+ app_config=app_config,
+ display_enabled=rag_mode_enabled,
+ )
+ es_query_routing_section = _build_es_query_routing_section(
+ app_config=app_config,
+ available_skills=available_skills,
+ )
# Get deferred tools section (tool_search)
deferred_tools_section = get_deferred_tools_prompt_section(app_config=app_config)
@@ -1438,9 +1486,10 @@ When a search, knowledge, skill, or MCP tool returns a result list:
# Format the prompt with dynamic skills and memory
prompt = SYSTEM_PROMPT_TEMPLATE.format(
agent_name=agent_name or storage_agent_id or "CM 助手",
- soul=get_agent_soul(storage_agent_id),
- skills_section=skills_section,
- bootstrap_section=bootstrap_section,
+ soul=get_agent_soul(storage_agent_id),
+ skills_section=skills_section,
+ es_query_routing_section=es_query_routing_section,
+ bootstrap_section=bootstrap_section,
deferred_tools_section=deferred_tools_section,
writing_mode_section=writing_mode_section,
memory_context=memory_context,
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py
index 5c1afd4b..8e46dbce 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py
@@ -10,7 +10,7 @@ from __future__ import annotations
from dataclasses import dataclass
from urllib.parse import urlsplit, urlunsplit
-from pydantic import BaseModel, ConfigDict, Field, field_validator
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
def normalize_service_url(value: str | None) -> str:
@@ -37,18 +37,12 @@ class WeKnoraConnectionConfig(BaseModel):
web_base_url: str = Field(default="", description="Optional internal WeKnora administration URL")
admin_email: str = Field(
default="",
- description=(
- "Optional WeKnora administrator email used by DeerFlow's server-side "
- "WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
- ),
+ description=("Optional WeKnora administrator email used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
)
admin_password: str = Field(
default="",
repr=False,
- description=(
- "Optional WeKnora administrator password used by DeerFlow's server-side "
- "WeKnora API client and detail-page iframe proxy. Prefer $ENV references."
- ),
+ description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
)
model_config = ConfigDict(extra="ignore")
@@ -58,10 +52,107 @@ class WeKnoraConnectionConfig(BaseModel):
return normalize_service_url(value)
+class LocalWikiEmbeddingConfig(BaseModel):
+ enabled: bool = True
+ provider: str = "openai"
+ model: str = "bge-m3"
+ base_url: str = ""
+ api_key: str = Field(default="", repr=False)
+ dimensions: int | None = Field(default=1024, ge=1, le=65536)
+ batch_size: int = Field(default=32, ge=1, le=512)
+ timeout_seconds: float = Field(default=30, gt=0, le=300)
+ max_retries: int = Field(default=3, ge=0, le=10)
+ model_config = ConfigDict(extra="ignore")
+
+ @field_validator("provider")
+ @classmethod
+ def _validate_provider(cls, value: str) -> str:
+ normalized = value.strip().lower()
+ if normalized not in {"openai", "openai_compatible"}:
+ raise ValueError("local Wiki embedding provider must be openai or openai_compatible")
+ return normalized
+
+ @field_validator("base_url")
+ @classmethod
+ def _validate_url(cls, value: str) -> str:
+ return normalize_service_url(value)
+
+
+class LocalWikiExternalApiConfig(BaseModel):
+ enabled: bool = False
+ api_key: str = Field(default="", repr=False)
+ previous_api_key: str = Field(default="", repr=False)
+ max_query_chars: int = Field(default=1000, ge=1, le=10000)
+ max_knowledge_bases: int = Field(default=20, ge=1, le=100)
+ max_results: int = Field(default=50, ge=1, le=100)
+ default_results: int = Field(default=10, ge=1, le=100)
+ include_content_max_chars: int = Field(default=12000, ge=0, le=1_000_000)
+ requests_per_minute: int = Field(default=60, ge=1, le=100000)
+ model_config = ConfigDict(extra="ignore")
+
+ @model_validator(mode="after")
+ def _validate_enabled_key(self):
+ if self.enabled and not self.api_key.strip():
+ raise ValueError("llmwiki.local_wiki_index.external_api.api_key is required when external API is enabled")
+ if self.default_results > self.max_results:
+ raise ValueError("external_api.default_results must not exceed max_results")
+ return self
+
+
+class LocalWikiIndexConfig(BaseModel):
+ """DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
+
+ enabled: bool = False
+ sync_enabled: bool = False
+ internal_search_enabled: bool = False
+ reference_ui_enabled: bool = False
+ sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
+ sync_page_size: int = Field(default=100, ge=1, le=500)
+ sync_concurrency: int = Field(default=3, ge=1, le=32)
+ sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
+ sync_request_timeout_seconds: float = Field(default=30, gt=0, le=300)
+ sync_max_retries: int = Field(default=3, ge=0, le=10)
+ sync_retry_backoff_seconds: float = Field(default=5, ge=0, le=300)
+ chunk_max_chars: int = Field(default=1000, ge=100, le=20000)
+ chunk_overlap_chars: int = Field(default=120, ge=0, le=10000)
+ chunk_min_chars: int = Field(default=80, ge=1, le=10000)
+ title_prefix_enabled: bool = True
+ embedding: LocalWikiEmbeddingConfig = Field(default_factory=LocalWikiEmbeddingConfig)
+ engine: str = "numpy_exact"
+ strict_vector: bool = True
+ top_k_sections: int = Field(default=30, ge=1, le=1000)
+ top_k_pages: int = Field(default=8, ge=1, le=100)
+ max_sections_per_page: int = Field(default=2, ge=1, le=20)
+ min_similarity: float = Field(default=0.35, ge=-1, le=1)
+ keyword_boost_enabled: bool = False
+ keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
+ cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
+ cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
+ external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
+ model_config = ConfigDict(extra="ignore")
+
+ @model_validator(mode="after")
+ def _validate_index(self):
+ if self.chunk_overlap_chars >= self.chunk_max_chars:
+ raise ValueError("chunk_overlap_chars must be smaller than chunk_max_chars")
+ if self.chunk_min_chars > self.chunk_max_chars:
+ raise ValueError("chunk_min_chars must not exceed chunk_max_chars")
+ if self.top_k_pages > self.top_k_sections:
+ raise ValueError("top_k_pages must not exceed top_k_sections")
+ if self.engine != "numpy_exact":
+ raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
+ if self.enabled and not self.embedding.enabled:
+ raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
+ if self.enabled and (not self.embedding.base_url or not self.embedding.model.strip()):
+ raise ValueError("local Wiki index embedding base_url and model are required when enabled")
+ return self
+
+
class LlmWikiConfig(BaseModel):
"""Top-level ``config.yaml`` section for conditional WeKnora mode."""
weknora: WeKnoraConnectionConfig = Field(default_factory=WeKnoraConnectionConfig)
+ local_wiki_index: LocalWikiIndexConfig = Field(default_factory=LocalWikiIndexConfig)
model_config = ConfigDict(extra="ignore")
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/skills_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/skills_config.py
index de2f5112..f707ced8 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/config/skills_config.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/skills_config.py
@@ -5,8 +5,26 @@ from pydantic import BaseModel, Field
from deerflow.config.runtime_paths import project_root, resolve_path
-
-class SecurityScanConfig(BaseModel):
+DEFAULT_ES_QUERY_ROUTING_PROMPT = """当用户的问题需要基于 Elasticsearch 查询语句(Query DSL)灵活构造、调整或执行查询时,必须优先使用 `es_query` 技能。
+典型场景包括:根据用户条件动态组合 bool/must/filter/should、范围、聚合、排序或分页等 ES 查询;或者用户直接提供、要求修改 ES 语句并据此查询数据。
+执行前先读取 `es_query` 技能的 SKILL.md,严格按照技能中的流程和参数调用;不要用通用搜索替代,也不要凭记忆臆测查询结果。
+仅当问题确实需要 ES 数据查询时使用;纯概念解释、普通代码说明或与 ES 无关的问题不要调用。"""
+
+
+class EsQueryRoutingConfig(BaseModel):
+ """Prompt steering that routes flexible Elasticsearch queries to ``es_query``."""
+
+ enabled: bool = Field(
+ default=False,
+ description="Inject the es_query routing prompt into the agent system prompt.",
+ )
+ prompt: str = Field(
+ default=DEFAULT_ES_QUERY_ROUTING_PROMPT,
+ description="Editable system-prompt instructions for deciding when and how to use es_query.",
+ )
+
+
+class SecurityScanConfig(BaseModel):
"""Toggle for the LLM-based skill security scanner."""
enabled: bool = Field(
@@ -33,10 +51,14 @@ class SkillsConfig(BaseModel):
default="/mnt/skills",
description="Path where skills are mounted in the sandbox container",
)
- security_scan: SecurityScanConfig = Field(
- default_factory=SecurityScanConfig,
- description="LLM security-scan toggle for skill content.",
- )
+ security_scan: SecurityScanConfig = Field(
+ default_factory=SecurityScanConfig,
+ description="LLM security-scan toggle for skill content.",
+ )
+ es_query_routing: EsQueryRoutingConfig = Field(
+ default_factory=EsQueryRoutingConfig,
+ description="Optional configurable routing prompt for the es_query skill.",
+ )
def get_skills_path(self) -> Path:
"""
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/__init__.py
new file mode 100644
index 00000000..9c11c04e
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/__init__.py
@@ -0,0 +1,7 @@
+"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
+
+from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
+from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
+from deerflow.integrations.weknora.local_index.sync import WikiSyncService
+
+__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/chunking.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/chunking.py
new file mode 100644
index 00000000..b97147af
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/chunking.py
@@ -0,0 +1,108 @@
+"""Markdown-aware Wiki section chunking."""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from dataclasses import dataclass
+
+_FRONTMATTER_RE = re.compile(r"\A\s*---\s*\n.*?\n---\s*(?:\n|\Z)", re.DOTALL)
+_SCRIPT_STYLE_RE = re.compile(r"<\s*(script|style)\b[^>]*>.*?<\s*/\s*\1\s*>", re.IGNORECASE | re.DOTALL)
+_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\([^)]*\)")
+_HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
+
+
+@dataclass(frozen=True)
+class WikiSection:
+ section_index: int
+ heading: str | None
+ content: str
+ embedding_text: str
+ content_hash: str
+
+
+def _clean_markdown(value: str) -> str:
+ text = value.replace("\r\n", "\n")
+ text = _FRONTMATTER_RE.sub("", text)
+ text = _SCRIPT_STYLE_RE.sub("", text)
+ text = _IMAGE_RE.sub(lambda match: match.group(1), text)
+ return re.sub(r"\n{3,}", "\n\n", text).strip()
+
+
+def _windows(text: str, max_chars: int, overlap: int) -> list[str]:
+ paragraphs = [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
+ chunks: list[str] = []
+ buffer = ""
+ for paragraph in paragraphs:
+ if len(buffer) + len(paragraph) + 2 <= max_chars:
+ buffer = f"{buffer}\n\n{paragraph}".strip()
+ continue
+ if buffer:
+ chunks.append(buffer)
+ buffer = ""
+ if len(paragraph) <= max_chars:
+ buffer = paragraph
+ continue
+ step = max(1, max_chars - overlap)
+ chunks.extend(paragraph[start : start + max_chars] for start in range(0, len(paragraph), step))
+ if buffer:
+ chunks.append(buffer)
+ return chunks
+
+
+def chunk_wiki_page(
+ page: dict,
+ *,
+ max_chars: int,
+ overlap: int,
+ min_chars: int,
+ title_prefix_enabled: bool,
+) -> list[WikiSection]:
+ content = _clean_markdown(str(page.get("content_md") or ""))
+ heading_stack: list[str] = []
+ sections: list[tuple[str | None, str]] = []
+ current_heading: str | None = None
+ body: list[str] = []
+ for line in content.splitlines():
+ match = _HEADING_RE.match(line)
+ if not match:
+ body.append(line)
+ continue
+ if any(item.strip() for item in body):
+ sections.append((current_heading, "\n".join(body).strip()))
+ level = len(match.group(1))
+ heading_stack = heading_stack[: level - 1]
+ heading_stack.append(match.group(2).strip())
+ current_heading = " / ".join(heading_stack)
+ body = []
+ if any(item.strip() for item in body):
+ sections.append((current_heading, "\n".join(body).strip()))
+ if content and not sections:
+ sections.append((None, content))
+ summary = _clean_markdown(str(page.get("summary") or ""))
+ if summary:
+ sections.insert(0, ("页面摘要", summary))
+
+ prefix_lines = [f"页面:{page.get('title') or page.get('slug') or ''}"]
+ aliases = [str(item) for item in page.get("aliases") or [] if str(item).strip()]
+ if aliases:
+ prefix_lines.append(f"别名:{'、'.join(aliases)}")
+ if page.get("page_type"):
+ prefix_lines.append(f"类型:{page['page_type']}")
+ seen: set[str] = set()
+ result: list[WikiSection] = []
+ for heading, section_text in sections:
+ for chunk in _windows(section_text, max_chars, overlap):
+ normalized = re.sub(r"\s+", " ", chunk).strip()
+ if len(normalized) < min_chars and len(sections) > 1:
+ continue
+ digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
+ if digest in seen:
+ continue
+ seen.add(digest)
+ context = list(prefix_lines) if title_prefix_enabled else []
+ if heading:
+ context.append(f"章节:{heading}")
+ embedding_text = "\n".join([*context, "", chunk]).strip()
+ result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
+ return result
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py
new file mode 100644
index 00000000..65a13a2a
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py
@@ -0,0 +1,107 @@
+"""Strict OpenAI-compatible embedding client for the Wiki index."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import logging
+from time import monotonic
+
+import httpx
+
+from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig
+from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
+
+logger = logging.getLogger(__name__)
+
+
+class WikiEmbeddingError(RuntimeError):
+ def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
+ super().__init__(message)
+ self.category = category
+ self.retryable = retryable
+
+
+class StrictWikiEmbeddingClient:
+ def __init__(self, config: LocalWikiEmbeddingConfig, *, http_client: httpx.AsyncClient | None = None) -> None:
+ self.config = config
+ self._client = http_client
+ self._owns_client = http_client is None
+ self._resolved_dimensions = config.dimensions
+
+ @property
+ def dimensions(self) -> int | None:
+ return self._resolved_dimensions
+
+ @property
+ def fingerprint(self) -> str:
+ raw = "\0".join((self.config.provider.strip().lower(), self.config.base_url.rstrip("/").lower(), self.config.model.strip(), str(self._resolved_dimensions or "auto")))
+ return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+ async def close(self) -> None:
+ if self._owns_client and self._client is not None:
+ await self._client.aclose()
+ self._client = None
+
+ async def _get_client(self) -> httpx.AsyncClient:
+ if self._client is None:
+ self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
+ return self._client
+
+ async def embed_texts(self, texts: list[str]) -> list[list[float]]:
+ if not self.config.enabled or not self.config.base_url or not self.config.model:
+ raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
+ if not texts:
+ return []
+ vectors: list[list[float]] = []
+ batch_size = max(1, self.config.batch_size)
+ for start in range(0, len(texts), batch_size):
+ vectors.extend(await self._embed_batch(texts[start : start + batch_size]))
+ if len(vectors) != len(texts):
+ raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
+ return vectors
+
+ async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
+ url = self.config.base_url.rstrip("/") + "/embeddings"
+ headers = {"Content-Type": "application/json"}
+ if self.config.api_key:
+ headers["Authorization"] = f"Bearer {self.config.api_key}"
+ payload: dict = {"model": self.config.model, "input": texts}
+ if self.config.dimensions:
+ payload["dimensions"] = self.config.dimensions
+ last_error: WikiEmbeddingError | None = None
+ for attempt in range(self.config.max_retries + 1):
+ started = monotonic()
+ try:
+ response = await (await self._get_client()).post(url, headers=headers, json=payload)
+ if response.status_code in {401, 403}:
+ raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False)
+ if response.status_code == 429 or response.status_code >= 500:
+ raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True)
+ response.raise_for_status()
+ data = response.json()
+ items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
+ if len(items) != len(texts):
+ raise WikiEmbeddingError("Embedding response count mismatch", category="response", retryable=False)
+ result: list[list[float]] = []
+ for item in items:
+ vector = normalize_vector(item.get("embedding") or [], dimensions=self._resolved_dimensions)
+ if self._resolved_dimensions is None:
+ self._resolved_dimensions = int(vector.size)
+ result.append(vector.astype(float).tolist())
+ logger.info("Wiki embedding batch completed model=%s batch=%d duration_ms=%d", self.config.model, len(texts), int((monotonic() - started) * 1000))
+ return result
+ except WikiEmbeddingError as exc:
+ last_error = exc
+ except httpx.TimeoutException:
+ last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
+ except (httpx.HTTPError, ValueError, TypeError) as exc:
+ last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError))
+ if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
+ raise last_error
+ await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
+ raise last_error or WikiEmbeddingError("Embedding request failed")
+
+ async def embed_query(self, text: str) -> list[float]:
+ result = await self.embed_texts([text])
+ return result[0]
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/normalize.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/normalize.py
new file mode 100644
index 00000000..0c93a989
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/normalize.py
@@ -0,0 +1,86 @@
+"""Normalize unstable WeKnora Wiki response shapes into the local contract."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from datetime import datetime
+from typing import Any
+from urllib.parse import unquote
+
+
+def normalize_slug(value: Any) -> str:
+ slug = unquote(str(value or "")).replace("\\", "/").strip().strip("/")
+ slug = re.sub(r"/{2,}", "/", slug)
+ if not slug:
+ raise ValueError("Wiki page slug is empty")
+ return slug
+
+
+def slug_hash(slug: str) -> str:
+ return hashlib.sha256(normalize_slug(slug).encode("utf-8")).hexdigest()
+
+
+def _list(value: Any) -> list:
+ if isinstance(value, list):
+ return value
+ if isinstance(value, tuple):
+ return list(value)
+ if isinstance(value, str) and value.strip():
+ return [value.strip()]
+ return []
+
+
+def _datetime(value: Any) -> datetime | None:
+ if isinstance(value, datetime):
+ return value
+ raw = str(value or "").strip()
+ if not raw:
+ return None
+ try:
+ return datetime.fromisoformat(raw.replace("Z", "+00:00"))
+ except ValueError:
+ return None
+
+
+def normalize_wiki_page(mapping_id: str, remote: dict[str, Any]) -> dict[str, Any]:
+ slug = normalize_slug(remote.get("slug") or remote.get("wiki_slug") or remote.get("path"))
+ aliases = sorted({str(item).strip() for item in _list(remote.get("aliases")) if str(item).strip()})
+ category_path = [str(item).strip() for item in _list(remote.get("category_path") or remote.get("categories")) if str(item).strip()]
+ title = str(remote.get("title") or slug.rsplit("/", 1)[-1]).strip()
+ summary = str(remote.get("summary") or remote.get("description") or "").replace("\r\n", "\n").strip()
+ content = str(remote.get("content_md") or remote.get("content") or remote.get("markdown") or "").replace("\r\n", "\n").strip()
+ status = str(remote.get("status") or "published").strip().lower()
+ if status not in {"draft", "published", "archived"}:
+ status = "published"
+ hash_payload = {
+ "title": title,
+ "aliases": aliases,
+ "summary": summary,
+ "content_md": content,
+ "page_type": str(remote.get("page_type") or remote.get("type") or "article").strip().lower(),
+ "status": status,
+ }
+ content_hash = hashlib.sha256(json.dumps(hash_payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest()
+ return {
+ "knowledge_base_mapping_id": mapping_id,
+ "remote_page_id": str(remote.get("id") or remote.get("page_id") or "").strip() or None,
+ "slug": slug,
+ "slug_hash": slug_hash(slug),
+ "title": title,
+ "summary": summary,
+ "content_md": content,
+ "page_type": hash_payload["page_type"],
+ "status": status,
+ "aliases": aliases,
+ "category_path": category_path,
+ "parent_slug": str(remote.get("parent_slug") or "").strip() or None,
+ "wiki_path": str(remote.get("wiki_path") or remote.get("path") or "").strip() or None,
+ "source_refs": _list(remote.get("source_refs") or remote.get("sources")),
+ "chunk_refs": _list(remote.get("chunk_refs") or remote.get("chunks")),
+ "page_version": int(remote.get("page_version") or remote.get("version") or 0),
+ "content_hash": content_hash,
+ "remote_created_at": _datetime(remote.get("created_at") or remote.get("createdAt")),
+ "remote_updated_at": _datetime(remote.get("updated_at") or remote.get("updatedAt")),
+ }
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/retrieval.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/retrieval.py
new file mode 100644
index 00000000..c22e596e
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/retrieval.py
@@ -0,0 +1,248 @@
+"""Wiki-only retrieval with local-vector preference and remote Wiki fallback.
+
+The fallback deliberately calls WeKnora's ``/wiki/pages`` API. It never calls
+the raw knowledge/chunk search endpoint, so callers always receive processed
+Wiki pages even while a whole knowledge base is awaiting vectorization.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from urllib.parse import urlencode
+
+from deerflow.integrations.weknora.client import WeKnoraClient
+from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
+from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
+from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
+
+
+def is_fully_vectorized(state: dict[str, Any] | None, fingerprint: str | None) -> bool:
+ """Return whether a completed full scan produced one coherent active index."""
+
+ if not state or not fingerprint:
+ return False
+ if str(state.get("state") or "") != "idle" or not state.get("last_success_at"):
+ return False
+ if str(state.get("embedding_fingerprint") or "") != fingerprint:
+ return False
+ local_count = int(state.get("local_page_count") or 0)
+ ready_count = int(state.get("ready_page_count") or 0)
+ failed_count = int(state.get("failed_page_count") or 0)
+ vector_count = int(state.get("vector_count") or 0)
+ if failed_count or ready_count != local_count:
+ return False
+ return local_count == 0 or vector_count > 0
+
+
+class WikiRetrievalService:
+ """Search complete indexes locally and incomplete indexes through Wiki API."""
+
+ def __init__(
+ self,
+ store: LlmWikiIndexStore,
+ *,
+ vector_search: WikiVectorSearchService | None,
+ client: WeKnoraClient | None,
+ ) -> None:
+ self.store = store
+ self.vector_search = vector_search
+ self.client = client
+
+ async def search(
+ self,
+ query: str,
+ mappings: list[dict[str, Any]],
+ *,
+ top_k_pages: int = 8,
+ include_drafts: bool = False,
+ ) -> dict[str, Any]:
+ normalized_query = query.strip()
+ if not normalized_query:
+ raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
+ if not mappings:
+ return {
+ "query": normalized_query,
+ "mode": "wiki_api",
+ "results": [],
+ "knowledge_base_count": 0,
+ "partial": False,
+ "warnings": [],
+ "fallback_knowledge_base_ids": [],
+ }
+
+ vector_mappings, fallback_mappings = await self._partition_mappings(mappings)
+ results: list[dict[str, Any]] = []
+ warnings: list[str] = []
+ vector_used = False
+ fallback_used = False
+
+ if vector_mappings and self.vector_search is not None:
+ try:
+ payload = await self.vector_search.search(
+ normalized_query,
+ vector_mappings,
+ top_k_pages=max(top_k_pages, len(vector_mappings)),
+ include_drafts=include_drafts,
+ )
+ results.extend(payload.get("results") or [])
+ warnings.extend(str(item) for item in payload.get("warnings") or [])
+ vector_used = True
+ except WikiIndexError as exc:
+ # A status row can become stale between classification and
+ # matrix loading. Wiki API remains a safe Wiki-only fallback.
+ warnings.append(f"local_vector:{exc.code}")
+ fallback_mappings.extend(vector_mappings)
+ vector_mappings = []
+
+ fallback_ids = [str(mapping["id"]) for mapping in fallback_mappings]
+ if fallback_mappings:
+ if self.client is None:
+ warnings.extend(f"{mapping['id']}:WIKI_API_UNAVAILABLE" for mapping in fallback_mappings)
+ else:
+ batches = await asyncio.gather(
+ *(
+ self._search_remote_mapping(
+ normalized_query,
+ mapping,
+ limit=max(top_k_pages, 1),
+ include_drafts=include_drafts,
+ )
+ for mapping in fallback_mappings
+ ),
+ return_exceptions=True,
+ )
+ for mapping, batch in zip(fallback_mappings, batches, strict=True):
+ if isinstance(batch, Exception):
+ warnings.append(f"{mapping['id']}:WIKI_API_SEARCH_FAILED")
+ else:
+ results.extend(batch)
+ fallback_used = True
+
+ if warnings and not results and not vector_used and not fallback_used:
+ raise WikiIndexError(
+ "WIKI_API_UNAVAILABLE",
+ "Wiki page retrieval is unavailable",
+ status_code=502 if self.client is not None else 503,
+ )
+
+ # De-duplicate by the public Wiki citation identity. Vector scores and
+ # Wiki-API rank scores are both descending relevance hints.
+ unique: dict[tuple[str, str], dict[str, Any]] = {}
+ for item in results:
+ key = (str(item.get("knowledge_base_id") or ""), str(item.get("wiki_slug") or item.get("id") or ""))
+ current = unique.get(key)
+ if current is None or float(item.get("score") or 0) > float(current.get("score") or 0):
+ unique[key] = item
+ ranked = sorted(unique.values(), key=lambda item: float(item.get("score") or 0), reverse=True)
+ mode = "mixed" if vector_used and fallback_used else "vector" if vector_used else "wiki_api"
+ fingerprint = self.vector_search.embedding.fingerprint[:12] if vector_used and self.vector_search is not None else None
+ return {
+ "query": normalized_query,
+ "mode": mode,
+ "results": ranked[:top_k_pages],
+ "knowledge_base_count": len(mappings),
+ "index_fingerprint": fingerprint,
+ "partial": bool(warnings),
+ "warnings": warnings,
+ "fallback_knowledge_base_ids": fallback_ids,
+ }
+
+ async def _partition_mappings(self, mappings: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+ if self.vector_search is None:
+ return [], list(mappings)
+ mapping_ids = [str(mapping["id"]) for mapping in mappings]
+ states = {
+ str(state["knowledge_base_mapping_id"]): state
+ for state in await self.store.list_index_status(mapping_ids)
+ }
+ fingerprint = self.vector_search.embedding.fingerprint
+ vector_mappings = [
+ mapping
+ for mapping in mappings
+ if mapping.get("wiki_index_enabled", True)
+ and is_fully_vectorized(states.get(str(mapping["id"])), fingerprint)
+ ]
+ ready_ids = {str(mapping["id"]) for mapping in vector_mappings}
+ return vector_mappings, [mapping for mapping in mappings if str(mapping["id"]) not in ready_ids]
+
+ async def _search_remote_mapping(
+ self,
+ query: str,
+ mapping: dict[str, Any],
+ *,
+ limit: int,
+ include_drafts: bool,
+ ) -> list[dict[str, Any]]:
+ if self.client is None:
+ return []
+ remote_id = str(mapping["weknora_id"])
+ listing = await self.client.list_wiki_pages(
+ remote_id,
+ page=1,
+ page_size=min(max(limit * 3, 20), 100),
+ query=query,
+ )
+ summaries = [item for item in listing.get("pages", []) if isinstance(item, dict)]
+ slugs = list(
+ dict.fromkeys(
+ str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
+ for item in summaries
+ if str(item.get("slug") or item.get("wiki_slug") or item.get("path") or "").strip("/")
+ )
+ )[: max(limit * 2, limit)]
+ details = await asyncio.gather(
+ *(self.client.get_wiki_page(remote_id, slug) for slug in slugs),
+ return_exceptions=True,
+ )
+ results: list[dict[str, Any]] = []
+ for rank, (slug, detail) in enumerate(zip(slugs, details, strict=True)):
+ if isinstance(detail, Exception):
+ continue
+ try:
+ page = normalize_wiki_page(str(mapping["id"]), detail)
+ except (TypeError, ValueError):
+ continue
+ if page["status"] == "archived" or (page["status"] == "draft" and not include_drafts):
+ continue
+ params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": slug})
+ content = str(page.get("content_md") or "")
+ score = float(detail.get("score") or max(0.01, 1.0 - rank * 0.05))
+ results.append(
+ {
+ "kind": "wiki_page",
+ "id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
+ "knowledge_base_id": mapping["id"],
+ "knowledge_base_name": mapping.get("name") or "",
+ "wiki_page_id": page.get("remote_page_id") or f"{mapping['id']}:{slug}",
+ "wiki_slug": slug,
+ "title": page.get("title") or slug,
+ "summary": page.get("summary") or "",
+ "page_type": page.get("page_type") or "article",
+ "page_version": int(page.get("page_version") or 0),
+ "updated_at": page.get("remote_updated_at"),
+ "score": score,
+ "content": content,
+ "snippet": content[:360],
+ "matched_heading": None,
+ "matched_sections": [
+ {
+ "section_index": 0,
+ "heading": None,
+ "content": content,
+ "score": score,
+ }
+ ],
+ "url": f"/weknora-source-preview?{params}",
+ "source": mapping.get("name") or "LLMWiki",
+ "type": "llmwiki_wiki",
+ "mode": "citation",
+ "retrieval_mode": "wiki_api",
+ }
+ )
+ if len(results) >= limit:
+ break
+ return results
+
+
+__all__ = ["WikiRetrievalService", "is_fully_vectorized"]
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/runtime.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/runtime.py
new file mode 100644
index 00000000..fa3f2838
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/runtime.py
@@ -0,0 +1,48 @@
+"""App-to-harness runtime injection for local Wiki index services."""
+
+from __future__ import annotations
+
+from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
+from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
+from deerflow.integrations.weknora.local_index.sync import WikiSyncService
+from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
+
+_store: LlmWikiIndexStore | None = None
+_search: WikiVectorSearchService | None = None
+_sync: WikiSyncService | None = None
+_retrieval: WikiRetrievalService | None = None
+
+
+def register_local_wiki_index_runtime(
+ *,
+ store: LlmWikiIndexStore | None,
+ search: WikiVectorSearchService | None,
+ sync: WikiSyncService | None,
+ retrieval: WikiRetrievalService | None = None,
+) -> None:
+ global _store, _search, _sync, _retrieval
+ _store, _search, _sync, _retrieval = store, search, sync, retrieval
+
+
+def get_local_wiki_index_store() -> LlmWikiIndexStore:
+ if _store is None:
+ raise RuntimeError("Local Wiki index store is not initialized")
+ return _store
+
+
+def get_local_wiki_search_service() -> WikiVectorSearchService:
+ if _search is None:
+ raise RuntimeError("Local Wiki vector search is not initialized")
+ return _search
+
+
+def get_local_wiki_sync_service() -> WikiSyncService:
+ if _sync is None:
+ raise RuntimeError("Local Wiki sync service is not initialized")
+ return _sync
+
+
+def get_local_wiki_retrieval_service() -> WikiRetrievalService:
+ if _retrieval is None:
+ raise RuntimeError("Wiki-only retrieval service is not initialized")
+ return _retrieval
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/search.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/search.py
new file mode 100644
index 00000000..5e893b12
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/search.py
@@ -0,0 +1,176 @@
+"""Unified exact-vector search over DeerFlow-local Wiki section snapshots."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import re
+from collections import OrderedDict, defaultdict
+from time import monotonic
+from typing import Any
+from urllib.parse import urlencode
+
+import numpy as np
+
+from deerflow.config.llmwiki_config import LocalWikiIndexConfig
+from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient, WikiEmbeddingError
+from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache, WikiVectorSnapshot
+from deerflow.integrations.weknora.local_index.vector_codec import normalize_vector
+
+
+class WikiIndexError(RuntimeError):
+ def __init__(self, code: str, message: str, *, status_code: int) -> None:
+ super().__init__(message)
+ self.code = code
+ self.status_code = status_code
+
+
+class WikiVectorSearchService:
+ def __init__(self, config: LocalWikiIndexConfig, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache) -> None:
+ self.config = config
+ self.embedding = embedding
+ self.cache = cache
+ self._query_vectors: OrderedDict[str, tuple[float, np.ndarray]] = OrderedDict()
+
+ async def search(
+ self,
+ query: str,
+ mappings: list[dict[str, Any]],
+ *,
+ top_k_pages: int | None = None,
+ include_drafts: bool = False,
+ external: bool = False,
+ ) -> dict[str, Any]:
+ normalized_query = query.strip()
+ if not normalized_query:
+ raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400)
+ if not mappings:
+ return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []}
+ try:
+ query_vector = await self._embed_query(normalized_query)
+ except (WikiEmbeddingError, ValueError) as exc:
+ raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None
+ fingerprint = self.embedding.fingerprint
+ tasks = [self._search_mapping(normalized_query, query_vector, mapping, fingerprint, include_drafts=include_drafts, external=external) for mapping in mappings]
+ batches = await asyncio.gather(*tasks, return_exceptions=True)
+ candidates: list[dict[str, Any]] = []
+ warnings: list[str] = []
+ for mapping, batch in zip(mappings, batches, strict=True):
+ if isinstance(batch, Exception):
+ warnings.append(f"{mapping.get('id')}: {batch}")
+ else:
+ candidates.extend(batch)
+ if warnings and not candidates and len(warnings) == len(mappings):
+ if any("FINGERPRINT" in item for item in warnings):
+ code = "WIKI_INDEX_FINGERPRINT_MISMATCH"
+ elif any("rebuild" in item.lower() for item in warnings):
+ code = "WIKI_INDEX_REBUILDING"
+ else:
+ code = "WIKI_INDEX_NOT_READY"
+ raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409)
+ candidates.sort(key=lambda item: item["score"], reverse=True)
+ grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list)
+ for candidate in candidates:
+ page = candidate["page"]
+ key = (str(page["knowledge_base_mapping_id"]), str(page["id"]))
+ if len(grouped[key]) < self.config.max_sections_per_page:
+ grouped[key].append(candidate)
+ results: list[dict[str, Any]] = []
+ for sections in grouped.values():
+ first = sections[0]
+ page = first["page"]
+ mapping = first["mapping"]
+ second_score = sections[1]["score"] if len(sections) > 1 else 0.0
+ page_score = first["score"] + (0.05 * second_score if second_score else 0)
+ params = urlencode({"knowledge_base_id": mapping["id"], "wiki_slug": page["slug"]})
+ results.append(
+ {
+ "kind": "wiki_page",
+ "id": page["id"],
+ "knowledge_base_id": mapping["id"],
+ "knowledge_base_name": mapping.get("name") or "",
+ "wiki_page_id": page["id"],
+ "wiki_slug": page["slug"],
+ "title": page.get("title") or page["slug"],
+ "summary": page.get("summary") or "",
+ "page_type": page.get("page_type") or "article",
+ "page_version": int(page.get("page_version") or 0),
+ "updated_at": page.get("remote_updated_at") or page.get("updated_at"),
+ "score": float(page_score),
+ "content": "\n\n".join(str(item["section_content"]) for item in sections),
+ "snippet": str(sections[0]["section_content"])[:360],
+ "matched_heading": sections[0].get("heading"),
+ "matched_sections": [{"section_index": int(item["section_index"]), "heading": item.get("heading"), "content": item["section_content"], "score": float(item["score"])} for item in sections],
+ "url": f"/weknora-source-preview?{params}",
+ "source": mapping.get("name") or "LLMWiki",
+ "type": "llmwiki_wiki",
+ "mode": "citation",
+ }
+ )
+ results.sort(key=lambda item: item["score"], reverse=True)
+ limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50)
+ return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings}
+
+ async def _embed_query(self, query: str) -> np.ndarray:
+ digest = hashlib.sha256(query.encode("utf-8")).hexdigest()
+ key = f"{self.embedding.fingerprint}:{digest}"
+ cached = self._query_vectors.get(key)
+ if cached is not None and monotonic() - cached[0] <= 300:
+ self._query_vectors.move_to_end(key)
+ return cached[1]
+ vector = normalize_vector(await self.embedding.embed_query(query), dimensions=self.embedding.dimensions)
+ key = f"{self.embedding.fingerprint}:{digest}"
+ self._query_vectors[key] = (monotonic(), vector)
+ self._query_vectors.move_to_end(key)
+ while len(self._query_vectors) > 256:
+ self._query_vectors.popitem(last=False)
+ return vector
+
+ async def _search_mapping(self, query: str, vector: np.ndarray, mapping: dict[str, Any], fingerprint: str, *, include_drafts: bool, external: bool) -> list[dict[str, Any]]:
+ mapping_id = str(mapping["id"])
+ try:
+ snapshot = await self.cache.get(mapping_id, fingerprint)
+ except ValueError as exc:
+ await self.cache.store.mark_rebuild_required([mapping_id], fingerprint)
+ self.cache.invalidate(mapping_id)
+ raise RuntimeError("WIKI_INDEX_REBUILD_REQUIRED") from exc
+ if snapshot.matrix.size == 0:
+ return []
+ if snapshot.dimensions != vector.size:
+ raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
+ eligible = np.asarray(
+ [
+ index
+ for index, row in enumerate(snapshot.rows)
+ if row["page"].get("status") != "draft" or (include_drafts and not external)
+ ],
+ dtype=int,
+ )
+ if eligible.size == 0:
+ return []
+ indices, scores = await asyncio.to_thread(self._top_indices, snapshot, vector, eligible)
+ terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)}
+ candidates = []
+ for index, score in zip(indices, scores, strict=True):
+ row = snapshot.rows[int(index)]
+ page = row["page"]
+ vector_score = float(score)
+ if vector_score < self.config.min_similarity:
+ continue
+ final_score = vector_score
+ if self.config.keyword_boost_enabled and terms:
+ haystack = f"{page.get('title', '')} {row.get('heading', '')} {row.get('section_content', '')}".lower()
+ keyword_score = sum(term in haystack for term in terms) / len(terms)
+ weight = self.config.keyword_boost_weight
+ final_score = (1 - weight) * vector_score + weight * keyword_score
+ candidates.append({**row, "page": page, "mapping": mapping, "score": final_score})
+ return candidates
+
+ def _top_indices(self, snapshot: WikiVectorSnapshot, vector: np.ndarray, eligible: np.ndarray) -> tuple[np.ndarray, np.ndarray]:
+ scores = snapshot.matrix[eligible] @ vector
+ count = min(self.config.top_k_sections, scores.size)
+ if count <= 0:
+ return np.asarray([], dtype=int), np.asarray([], dtype=np.float32)
+ positions = np.argpartition(scores, -count)[-count:]
+ positions = positions[np.argsort(scores[positions])[::-1]]
+ return eligible[positions], scores[positions]
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/sync.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/sync.py
new file mode 100644
index 00000000..5805c5e4
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/sync.py
@@ -0,0 +1,173 @@
+"""Lease-protected full Wiki mirror scan and page-level vector replacement."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import platform
+from typing import Any
+from uuid import uuid4
+
+from deerflow.config.llmwiki_config import LocalWikiIndexConfig
+from deerflow.integrations.weknora.client import WeKnoraClient
+from deerflow.integrations.weknora.local_index.chunking import chunk_wiki_page
+from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
+from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
+from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
+from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
+from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
+
+logger = logging.getLogger(__name__)
+
+
+class WikiSyncLeaseLost(RuntimeError):
+ pass
+
+
+def _safe_error(exc: BaseException) -> str:
+ return f"{type(exc).__name__}: {str(exc)[:1000]}"
+
+
+class WikiSyncService:
+ def __init__(self, config: LocalWikiIndexConfig, store: LlmWikiIndexStore, embedding: StrictWikiEmbeddingClient, cache: WikiVectorCache, client: WeKnoraClient) -> None:
+ self.config = config
+ self.store = store
+ self.embedding = embedding
+ self.cache = cache
+ self.client = client
+ self.owner = f"{platform.node()}:{uuid4()}"
+
+ async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
+ mapping_id = str(mapping["id"])
+ sync_id = str(uuid4())
+ if self.embedding.dimensions is None:
+ # Auto-dimension configurations need one successful response to
+ # produce a stable fingerprint before lease/rebuild decisions.
+ await self.embedding.embed_query("DeerFlow local Wiki embedding dimension probe")
+ fingerprint = self.embedding.fingerprint
+ revision, indexed_fingerprint, prior_state = await self.store.get_index_revision(mapping_id)
+ if revision > 0 and indexed_fingerprint and indexed_fingerprint != fingerprint and prior_state not in {"rebuild_required", "rebuilding"}:
+ await self.store.mark_rebuild_required([mapping_id], fingerprint)
+ self.cache.invalidate(mapping_id)
+ prior_state = "rebuild_required"
+ rebuilding = prior_state == "rebuild_required"
+ acquired = await self.store.try_acquire_sync_lease(mapping_id, owner=self.owner, sync_id=sync_id, lease_seconds=self.config.sync_lease_seconds, fingerprint=fingerprint)
+ if not acquired:
+ return {"mapping_id": mapping_id, "status": "already_running"}
+ lease_lost = asyncio.Event()
+
+ async def _lease_heartbeat() -> None:
+ interval = max(5.0, min(60.0, self.config.sync_lease_seconds / 3))
+ while True:
+ await asyncio.sleep(interval)
+ try:
+ renewed = await self.store.renew_sync_lease(
+ mapping_id,
+ owner=self.owner,
+ lease_seconds=self.config.sync_lease_seconds,
+ )
+ except Exception:
+ renewed = False
+ logger.exception("Local Wiki sync lease heartbeat failed mapping_id=%s", mapping_id)
+ if not renewed:
+ lease_lost.set()
+ return
+
+ heartbeat = asyncio.create_task(_lease_heartbeat())
+
+ def _ensure_lease() -> None:
+ if lease_lost.is_set():
+ raise WikiSyncLeaseLost("Local Wiki sync lease was lost")
+
+ remote_count = 0
+ changed = 0
+ failed = 0
+ try:
+ remote_id = str(mapping["weknora_id"])
+ await self.client.get_knowledge_base(remote_id)
+ page_number = 1
+ while True:
+ batch = await self.client.list_wiki_pages(remote_id, page=page_number, page_size=self.config.sync_page_size)
+ pages = [page for page in batch.get("pages", []) if isinstance(page, dict)]
+ remote_count += len(pages)
+ for summary in pages:
+ try:
+ _ensure_lease()
+ slug = str(summary.get("slug") or summary.get("wiki_slug") or summary.get("path") or "")
+ remote_page = await self.client.get_wiki_page(remote_id, slug)
+ page = normalize_wiki_page(mapping_id, {**summary, **remote_page})
+ existing = await self.store.get_page_by_slug_hash(mapping_id, page["slug_hash"])
+ active_fingerprint = self.embedding.fingerprint
+ if (
+ not force
+ and existing
+ and existing.get("content_hash") == page["content_hash"]
+ and existing.get("indexed_fingerprint") == active_fingerprint
+ and existing.get("index_status") == "ready"
+ and not existing.get("is_remote_deleted")
+ ):
+ await self.store.touch_page(str(existing["id"]), sync_id)
+ continue
+ sections = (
+ []
+ if page["status"] == "archived"
+ else chunk_wiki_page(page, max_chars=self.config.chunk_max_chars, overlap=self.config.chunk_overlap_chars, min_chars=self.config.chunk_min_chars, title_prefix_enabled=self.config.title_prefix_enabled)
+ )
+ embeddings = await self.embedding.embed_texts([section.embedding_text for section in sections]) if sections else []
+ fingerprint = self.embedding.fingerprint
+ vectors = []
+ for section, vector in zip(sections, embeddings, strict=True):
+ blob, dimensions = encode_vector(vector, dimensions=self.embedding.dimensions)
+ vectors.append(
+ {"section_index": section.section_index, "heading": section.heading, "section_content": section.content, "content_hash": section.content_hash, "embedding_dimensions": dimensions, "vector_blob": blob}
+ )
+ _ensure_lease()
+ await self.store.replace_page_vectors(page=page, vectors=vectors, fingerprint=fingerprint, sync_id=sync_id)
+ self.cache.invalidate(mapping_id)
+ changed += 1
+ except Exception as exc:
+ if isinstance(exc, WikiSyncLeaseLost):
+ raise
+ failed += 1
+ try:
+ fallback = normalize_wiki_page(mapping_id, summary)
+ await self.store.record_page_failure(page=fallback, sync_id=sync_id, error=_safe_error(exc))
+ except Exception as record_exc:
+ logger.exception("Could not record local Wiki page failure mapping_id=%s", mapping_id)
+ raise RuntimeError("Could not safely retain a failed Wiki page during the full scan") from record_exc
+ logger.warning("Local Wiki page sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
+ _ensure_lease()
+ total_pages = int(batch.get("total_pages") or 1)
+ if page_number >= total_pages:
+ break
+ page_number += 1
+ _ensure_lease()
+ deleted = await self.store.mark_missing_pages_deleted(mapping_id, sync_id)
+ if deleted:
+ self.cache.invalidate(mapping_id)
+ await self.store.finish_sync(
+ mapping_id,
+ owner=self.owner,
+ sync_id=sync_id,
+ success=not (rebuilding and failed),
+ remote_page_count=remote_count,
+ error=f"{failed} page(s) failed" if failed else None,
+ )
+ return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
+ except asyncio.CancelledError:
+ await self.store.finish_sync(
+ mapping_id,
+ owner=self.owner,
+ sync_id=sync_id,
+ success=False,
+ remote_page_count=remote_count,
+ error="Sync cancelled",
+ )
+ raise
+ except Exception as exc:
+ await self.store.finish_sync(mapping_id, owner=self.owner, sync_id=sync_id, success=False, remote_page_count=remote_count, error=_safe_error(exc))
+ logger.warning("Local Wiki sync failed mapping_id=%s: %s", mapping_id, _safe_error(exc))
+ raise
+ finally:
+ heartbeat.cancel()
+ await asyncio.gather(heartbeat, return_exceptions=True)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/vector_cache.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/vector_cache.py
new file mode 100644
index 00000000..1a098a78
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/vector_cache.py
@@ -0,0 +1,86 @@
+"""Per-knowledge-base LRU snapshots for NumPy exact search."""
+
+from __future__ import annotations
+
+import asyncio
+from collections import OrderedDict
+from dataclasses import dataclass
+from time import monotonic
+from typing import Any
+
+import numpy as np
+
+from deerflow.integrations.weknora.local_index.vector_codec import decode_vector
+from deerflow.persistence.llmwiki_index import LlmWikiIndexStore
+
+
+@dataclass(frozen=True)
+class WikiVectorSnapshot:
+ mapping_id: str
+ index_revision: int
+ embedding_fingerprint: str
+ dimensions: int
+ matrix: np.ndarray
+ rows: list[dict[str, Any]]
+
+
+class WikiVectorCache:
+ def __init__(self, store: LlmWikiIndexStore, *, max_knowledge_bases: int = 20, revision_check_seconds: float = 5) -> None:
+ self.store = store
+ self.max_knowledge_bases = max_knowledge_bases
+ self.revision_check_seconds = revision_check_seconds
+ self._snapshots: OrderedDict[str, WikiVectorSnapshot] = OrderedDict()
+ self._checked_at: dict[str, float] = {}
+ self._locks: dict[str, asyncio.Lock] = {}
+
+ def invalidate(self, mapping_id: str) -> None:
+ self._snapshots.pop(mapping_id, None)
+ self._checked_at.pop(mapping_id, None)
+
+ def clear(self) -> None:
+ self._snapshots.clear()
+ self._checked_at.clear()
+
+ async def get(self, mapping_id: str, fingerprint: str) -> WikiVectorSnapshot:
+ cached = self._snapshots.get(mapping_id)
+ now = monotonic()
+ if cached is not None and cached.embedding_fingerprint == fingerprint and now - self._checked_at.get(mapping_id, 0) < self.revision_check_seconds:
+ self._snapshots.move_to_end(mapping_id)
+ return cached
+ lock = self._locks.setdefault(mapping_id, asyncio.Lock())
+ async with lock:
+ revision, active_fingerprint, state = await self.store.get_index_revision(mapping_id)
+ self._checked_at[mapping_id] = monotonic()
+ cached = self._snapshots.get(mapping_id)
+ if state in {"not_synced", "rebuild_required", "rebuilding", "disabled"} or revision <= 0:
+ raise RuntimeError(f"WIKI_INDEX_NOT_READY:{state}")
+ if active_fingerprint != fingerprint:
+ raise RuntimeError("WIKI_INDEX_FINGERPRINT_MISMATCH")
+ if cached is not None and cached.index_revision == revision and cached.embedding_fingerprint == fingerprint:
+ self._snapshots.move_to_end(mapping_id)
+ return cached
+ raw = await self.store.load_vector_snapshot(mapping_id, fingerprint)
+ snapshot = await asyncio.to_thread(self._decode_snapshot, mapping_id, raw)
+ self._snapshots[mapping_id] = snapshot
+ self._snapshots.move_to_end(mapping_id)
+ while len(self._snapshots) > self.max_knowledge_bases:
+ evicted, _ = self._snapshots.popitem(last=False)
+ self._checked_at.pop(evicted, None)
+ return snapshot
+
+ @staticmethod
+ def _decode_snapshot(mapping_id: str, raw: dict[str, Any]) -> WikiVectorSnapshot:
+ rows = list(raw.get("rows") or [])
+ if not rows:
+ return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), 0, np.empty((0, 0), dtype=np.float32), [])
+ dimensions = int(rows[0]["embedding_dimensions"])
+ vectors = []
+ valid_rows = []
+ for row in rows:
+ if int(row["embedding_dimensions"]) != dimensions:
+ raise ValueError("Mixed vector dimensions in one Wiki snapshot")
+ vectors.append(decode_vector(row["vector_blob"], dimensions))
+ valid_rows.append(row)
+ matrix = np.ascontiguousarray(np.vstack(vectors), dtype=np.float32)
+ matrix.setflags(write=False)
+ return WikiVectorSnapshot(mapping_id, int(raw.get("revision") or 0), str(raw.get("fingerprint") or ""), dimensions, matrix, valid_rows)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/vector_codec.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/vector_codec.py
new file mode 100644
index 00000000..9b1af0e3
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/vector_codec.py
@@ -0,0 +1,33 @@
+"""Validated little-endian normalized float32 Wiki vector codec."""
+
+from __future__ import annotations
+
+import numpy as np
+
+
+def normalize_vector(values, *, dimensions: int | None = None) -> np.ndarray:
+ vector = np.asarray(values, dtype=np.float32)
+ if vector.ndim != 1 or vector.size == 0:
+ raise ValueError("Embedding vector must be a non-empty one-dimensional array")
+ if dimensions is not None and vector.size != dimensions:
+ raise ValueError(f"Embedding dimension mismatch: expected {dimensions}, got {vector.size}")
+ if not np.isfinite(vector).all():
+ raise ValueError("Embedding vector contains NaN or infinity")
+ norm = float(np.linalg.norm(vector))
+ if not np.isfinite(norm) or norm <= 0:
+ raise ValueError("Embedding vector has zero norm")
+ return np.asarray(vector / norm, dtype=" tuple[bytes, int]:
+ vector = normalize_vector(values, dimensions=dimensions)
+ return vector.tobytes(order="C"), int(vector.size)
+
+
+def decode_vector(blob: bytes, dimensions: int) -> np.ndarray:
+ if len(blob) != dimensions * 4:
+ raise ValueError(f"Corrupt vector BLOB: expected {dimensions * 4} bytes, got {len(blob)}")
+ vector = np.frombuffer(blob, dtype=" list[dict[str, Any]]:
+ """Return every configured question in display order."""
+ raise NotImplementedError
+
+ @abstractmethod
+ async def find_enabled(self, question: str) -> dict[str, Any] | None:
+ """Return the enabled row whose question exactly equals ``question``."""
+ raise NotImplementedError
+
+ @abstractmethod
+ async def replace_all(
+ self,
+ rows: list[dict[str, Any]],
+ *,
+ updated_by: str | None = None,
+ ) -> list[dict[str, Any]]:
+ """Atomically replace the complete fixed-question configuration."""
+ raise NotImplementedError
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/memory.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/memory.py
new file mode 100644
index 00000000..4c128efa
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/memory.py
@@ -0,0 +1,56 @@
+"""In-memory fixed-question store used by memory deployments and tests."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from deerflow.persistence.fixed_questions.base import FixedQuestionStore
+
+
+def _normalize(data: dict[str, Any], updated_by: str | None) -> dict[str, Any]:
+ return {
+ "id": str(data["id"]),
+ "question": str(data.get("question") or ""),
+ "answer": str(data.get("answer") or ""),
+ "enabled": bool(data.get("enabled", True)),
+ "tokens_per_second": int(data.get("tokens_per_second", 100)),
+ "sort_order": int(data.get("sort_order", 0)),
+ "updated_by": updated_by,
+ "updated_at": datetime.now(UTC).isoformat(),
+ }
+
+
+class MemoryFixedQuestionStore(FixedQuestionStore):
+ def __init__(self) -> None:
+ self._rows: dict[str, dict[str, Any]] = {}
+
+ async def list_questions(self) -> list[dict[str, Any]]:
+ rows = sorted(
+ self._rows.values(),
+ key=lambda row: (row.get("sort_order", 0), row.get("question", "")),
+ )
+ return [dict(row) for row in rows]
+
+ async def find_enabled(self, question: str) -> dict[str, Any] | None:
+ for row in self._rows.values():
+ if row["enabled"] and row["question"] == question:
+ return dict(row)
+ return None
+
+ async def replace_all(
+ self,
+ rows: list[dict[str, Any]],
+ *,
+ updated_by: str | None = None,
+ ) -> list[dict[str, Any]]:
+ # Last duplicate question wins, mirroring the SQL repository.
+ by_id: dict[str, dict[str, Any]] = {}
+ for row in rows:
+ question = row.get("question")
+ row_id = row.get("id")
+ if row_id and question not in (None, ""):
+ by_id[str(row_id)] = row
+ by_question = {str(row["question"]): row for row in by_id.values()}
+ self._rows = {str(row["id"]): _normalize(row, updated_by) for row in by_question.values()}
+ return await self.list_questions()
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/model.py
new file mode 100644
index 00000000..265b5e3e
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/model.py
@@ -0,0 +1,41 @@
+"""ORM model for exact-match, administrator-managed fixed questions."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column
+
+from deerflow.persistence.base import Base
+from deerflow.persistence.types import BeijingDateTime, PortableLongText
+
+
+class FixedQuestionRow(Base):
+ """One globally shared question/answer pair.
+
+ ``question_hash`` provides a portable indexed lookup for questions stored in
+ ``Text``. The repository always compares the original question as well, so a
+ theoretical SHA-256 collision cannot cause a false match.
+ """
+
+ __tablename__ = "fixed_questions"
+ __table_args__ = (UniqueConstraint("question_hash", name="uq_fixed_questions_question_hash"),)
+
+ id: Mapped[str] = mapped_column(String(64), primary_key=True)
+ question: Mapped[str] = mapped_column(Text, nullable=False)
+ question_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+ answer: Mapped[str] = mapped_column(PortableLongText, nullable=False)
+ enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="1")
+ tokens_per_second: Mapped[int] = mapped_column(
+ Integer,
+ default=100,
+ server_default="100",
+ )
+ sort_order: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
+ updated_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
+ updated_at: Mapped[datetime] = mapped_column(
+ BeijingDateTime(),
+ default=lambda: datetime.now(UTC),
+ onupdate=lambda: datetime.now(UTC),
+ )
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/sql.py
new file mode 100644
index 00000000..5d8098d5
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/fixed_questions/sql.py
@@ -0,0 +1,88 @@
+"""SQLAlchemy-backed fixed-question store."""
+
+from __future__ import annotations
+
+import hashlib
+from datetime import datetime
+from typing import Any
+
+from sqlalchemy import delete, select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from deerflow.persistence.fixed_questions.base import FixedQuestionStore
+from deerflow.persistence.fixed_questions.model import FixedQuestionRow
+
+
+def question_hash(question: str) -> str:
+ return hashlib.sha256(question.encode("utf-8")).hexdigest()
+
+
+def _to_dict(row: FixedQuestionRow) -> dict[str, Any]:
+ data = row.to_dict()
+ data.pop("question_hash", None)
+ value = data.get("updated_at")
+ if isinstance(value, datetime):
+ data["updated_at"] = value.isoformat()
+ return data
+
+
+def _row_from(data: dict[str, Any], updated_by: str | None) -> FixedQuestionRow:
+ question = str(data.get("question") or "")
+ return FixedQuestionRow(
+ id=str(data["id"]),
+ question=question,
+ question_hash=question_hash(question),
+ answer=str(data.get("answer") or ""),
+ enabled=bool(data.get("enabled", True)),
+ tokens_per_second=int(data.get("tokens_per_second", 100)),
+ sort_order=int(data.get("sort_order", 0)),
+ updated_by=updated_by,
+ )
+
+
+class FixedQuestionRepository(FixedQuestionStore):
+ def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
+ self._sf = session_factory
+
+ async def list_questions(self) -> list[dict[str, Any]]:
+ stmt = select(FixedQuestionRow).order_by(
+ FixedQuestionRow.sort_order.asc(),
+ FixedQuestionRow.question.asc(),
+ )
+ async with self._sf() as session:
+ result = await session.execute(stmt)
+ return [_to_dict(row) for row in result.scalars()]
+
+ async def find_enabled(self, question: str) -> dict[str, Any] | None:
+ stmt = select(FixedQuestionRow).where(
+ FixedQuestionRow.question_hash == question_hash(question),
+ FixedQuestionRow.enabled.is_(True),
+ )
+ async with self._sf() as session:
+ row = (await session.execute(stmt)).scalar_one_or_none()
+ if row is None or row.question != question:
+ return None
+ return _to_dict(row)
+
+ async def replace_all(
+ self,
+ rows: list[dict[str, Any]],
+ *,
+ updated_by: str | None = None,
+ ) -> list[dict[str, Any]]:
+ # Last duplicate question wins, preventing a unique-hash violation even
+ # if a non-UI caller submits malformed data.
+ by_id: dict[str, dict[str, Any]] = {}
+ for row in rows:
+ question = row.get("question")
+ row_id = row.get("id")
+ if row_id and question not in (None, ""):
+ by_id[str(row_id)] = row
+ by_question = {str(row["question"]): row for row in by_id.values()}
+
+ async with self._sf() as session:
+ await session.execute(delete(FixedQuestionRow))
+ new_rows = [_row_from(row, updated_by) for row in by_question.values()]
+ session.add_all(new_rows)
+ await session.commit()
+ return [_to_dict(row) for row in new_rows]
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/base.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/base.py
index aca5d293..a0bf66cf 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/base.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/base.py
@@ -22,7 +22,15 @@ class LlmWikiStore(ABC):
async def get_by_weknora_id(self, weknora_id: str) -> dict[str, Any] | None: ...
@abstractmethod
- async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None: ...
+ async def update_mapping(
+ self,
+ mapping_id: str,
+ *,
+ name: str | None = None,
+ description: str | None = None,
+ wiki_index_enabled: bool | None = None,
+ external_search_enabled: bool | None = None,
+ ) -> dict[str, Any] | None: ...
@abstractmethod
async def adopt_conversation_deposit_mapping(
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/memory.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/memory.py
index ee732223..78b6265e 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/memory.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/memory.py
@@ -33,6 +33,9 @@ class MemoryLlmWikiStore(LlmWikiStore):
"published_by": None,
"reviewed_by": None,
"published_at": None,
+ "wiki_index_enabled": True,
+ "external_search_enabled": False,
+ "external_search_updated_at": None,
"created_at": now,
"updated_at": now,
}
@@ -65,7 +68,15 @@ class MemoryLlmWikiStore(LlmWikiStore):
return dict(row)
return None
- async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
+ async def update_mapping(
+ self,
+ mapping_id: str,
+ *,
+ name: str | None = None,
+ description: str | None = None,
+ wiki_index_enabled: bool | None = None,
+ external_search_enabled: bool | None = None,
+ ) -> dict[str, Any] | None:
row = self._rows.get(mapping_id)
if row is None:
return None
@@ -73,6 +84,11 @@ class MemoryLlmWikiStore(LlmWikiStore):
row["name"] = name
if description is not None:
row["description"] = description
+ if wiki_index_enabled is not None:
+ row["wiki_index_enabled"] = wiki_index_enabled
+ if external_search_enabled is not None:
+ row["external_search_enabled"] = external_search_enabled
+ row["external_search_updated_at"] = _now()
row["updated_at"] = _now()
return dict(row)
@@ -143,11 +159,7 @@ class MemoryLlmWikiStore(LlmWikiStore):
async def get_conversation_deposit_mapping(self) -> dict[str, Any] | None:
for row in self._rows.values():
- if (
- row.get("owner_user_id") == "system"
- and row.get("name") == "对话沉淀"
- and row.get("publication_status") == "published"
- ):
+ if row.get("owner_user_id") == "system" and row.get("name") == "对话沉淀" and row.get("publication_status") == "published":
return dict(row)
return None
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/model.py
index 65950a0b..728bc6b1 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/model.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/model.py
@@ -4,7 +4,7 @@ from __future__ import annotations
from datetime import UTC, datetime
-from sqlalchemy import String, UniqueConstraint
+from sqlalchemy import Boolean, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
@@ -24,6 +24,9 @@ class LlmWikiKnowledgeBaseRow(Base):
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+ wiki_index_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default=true(), index=True)
+ external_search_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
+ external_search_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC), onupdate=lambda: datetime.now(UTC))
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/sql.py
index 83e4ccef..30c9b0af 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/sql.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki/sql.py
@@ -19,7 +19,7 @@ def _now() -> datetime:
def _to_dict(row: LlmWikiKnowledgeBaseRow | LlmWikiConversationDepositRow) -> dict[str, Any]:
data = row.to_dict()
- for key in ("published_at", "created_at", "updated_at"):
+ for key in ("published_at", "external_search_updated_at", "created_at", "updated_at"):
value = data.get(key)
if isinstance(value, datetime):
data[key] = value.isoformat()
@@ -82,7 +82,15 @@ class LlmWikiRepository(LlmWikiStore):
row = (await session.execute(stmt)).scalar_one_or_none()
return _to_dict(row) if row else None
- async def update_mapping(self, mapping_id: str, *, name: str | None = None, description: str | None = None) -> dict[str, Any] | None:
+ async def update_mapping(
+ self,
+ mapping_id: str,
+ *,
+ name: str | None = None,
+ description: str | None = None,
+ wiki_index_enabled: bool | None = None,
+ external_search_enabled: bool | None = None,
+ ) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(LlmWikiKnowledgeBaseRow, mapping_id)
if row is None:
@@ -91,6 +99,11 @@ class LlmWikiRepository(LlmWikiStore):
row.name = name
if description is not None:
row.description = description
+ if wiki_index_enabled is not None:
+ row.wiki_index_enabled = wiki_index_enabled
+ if external_search_enabled is not None:
+ row.external_search_enabled = external_search_enabled
+ row.external_search_updated_at = _now()
row.updated_at = _now()
await session.commit()
return _to_dict(row)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py
new file mode 100644
index 00000000..71f2b522
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py
@@ -0,0 +1,21 @@
+"""Persistence facade for the DeerFlow-local Wiki vector index."""
+
+from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
+from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
+from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
+from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
+
+
+def make_llmwiki_index_store(session_factory):
+ return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory)
+
+
+__all__ = [
+ "LlmWikiIndexStore",
+ "MemoryLlmWikiIndexStore",
+ "SqlLlmWikiIndexStore",
+ "LlmWikiWikiPageRow",
+ "LlmWikiWikiVectorRow",
+ "LlmWikiWikiSyncStateRow",
+ "make_llmwiki_index_store",
+]
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/base.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/base.py
new file mode 100644
index 00000000..044740d4
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/base.py
@@ -0,0 +1,54 @@
+"""Storage contract for local Wiki pages, vectors, revisions and sync leases."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class LlmWikiIndexStore(ABC):
+ @abstractmethod
+ async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None: ...
+
+ @abstractmethod
+ async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None: ...
+
+ @abstractmethod
+ async def touch_page(self, page_id: str, sync_id: str) -> None: ...
+
+ @abstractmethod
+ async def replace_page_vectors(
+ self,
+ *,
+ page: dict[str, Any],
+ vectors: list[dict[str, Any]],
+ fingerprint: str,
+ sync_id: str,
+ ) -> dict[str, Any]: ...
+
+ @abstractmethod
+ async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None: ...
+
+ @abstractmethod
+ async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int: ...
+
+ @abstractmethod
+ async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
+
+ @abstractmethod
+ async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
+
+ @abstractmethod
+ async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool: ...
+
+ @abstractmethod
+ async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool: ...
+
+ @abstractmethod
+ async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None: ...
+
+ @abstractmethod
+ async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]: ...
+
+ @abstractmethod
+ async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int: ...
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/memory.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/memory.py
new file mode 100644
index 00000000..51e9e512
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/memory.py
@@ -0,0 +1,160 @@
+"""In-memory local Wiki index store used by tests and DB-less mode."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from uuid import uuid4
+
+from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
+
+
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
+class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
+ def __init__(self) -> None:
+ self.pages: dict[str, dict[str, Any]] = {}
+ self.vectors: dict[str, list[dict[str, Any]]] = {}
+ self.states: dict[str, dict[str, Any]] = {}
+
+ def _state(self, mapping_id: str) -> dict[str, Any]:
+ return self.states.setdefault(mapping_id, {"knowledge_base_mapping_id": mapping_id, "state": "idle", "index_revision": 0, "embedding_fingerprint": None})
+
+ async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
+ row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
+ return dict(row) if row else None
+
+ async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
+ row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
+ return dict(row) if row else None
+
+ async def touch_page(self, page_id: str, sync_id: str) -> None:
+ if page_id in self.pages:
+ self.pages[page_id]["last_seen_sync_id"] = sync_id
+ self.pages[page_id]["last_error"] = None
+
+ async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
+ existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
+ page_id = existing["id"] if existing else str(uuid4())
+ now = _now().isoformat()
+ row = {
+ **(existing or {}),
+ **page,
+ "id": page_id,
+ "index_status": "ready",
+ "indexed_fingerprint": fingerprint,
+ "indexed_at": now,
+ "last_error": None,
+ "is_remote_deleted": False,
+ "last_seen_sync_id": sync_id,
+ "created_at": (existing or {}).get("created_at", now),
+ "updated_at": now,
+ }
+ self.pages[page_id] = row
+ self.vectors[page_id] = [{**item, "page_id": page_id, "knowledge_base_mapping_id": page["knowledge_base_mapping_id"], "embedding_fingerprint": fingerprint} for item in vectors]
+ state = self._state(page["knowledge_base_mapping_id"])
+ state["index_revision"] += 1
+ state["embedding_fingerprint"] = fingerprint
+ return dict(row)
+
+ async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
+ existing = await self.get_page_by_slug_hash(page["knowledge_base_mapping_id"], page["slug_hash"])
+ if existing:
+ self.pages[existing["id"]].update(
+ {
+ "last_seen_sync_id": sync_id,
+ "last_error": error,
+ # Preserve the last atomically completed generation.
+ "index_status": "ready" if self.vectors.get(existing["id"]) else "failed",
+ }
+ )
+ return
+ page_id = str(uuid4())
+ self.pages[page_id] = {**page, "id": page_id, "index_status": "failed", "last_error": error, "last_seen_sync_id": sync_id, "is_remote_deleted": False}
+
+ async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
+ changed = 0
+ for page in self.pages.values():
+ if page["knowledge_base_mapping_id"] == mapping_id and page.get("last_seen_sync_id") != sync_id and not page.get("is_remote_deleted"):
+ page.update({"is_remote_deleted": True, "index_status": "deleted"})
+ self.vectors.pop(page["id"], None)
+ changed += 1
+ if changed:
+ self._state(mapping_id)["index_revision"] += 1
+ return changed
+
+ async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
+ rows = []
+ for page_id, vectors in self.vectors.items():
+ page = self.pages[page_id]
+ if page["knowledge_base_mapping_id"] != mapping_id or page.get("is_remote_deleted") or page.get("status") not in {"published", "draft"} or page.get("index_status") != "ready":
+ continue
+ rows.extend({**vector, "page": dict(page)} for vector in vectors if vector["embedding_fingerprint"] == fingerprint)
+ state = self._state(mapping_id)
+ return {"revision": state["index_revision"], "fingerprint": fingerprint, "rows": rows}
+
+ async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
+ state = self._state(mapping_id)
+ return int(state["index_revision"]), state.get("embedding_fingerprint"), str(state.get("state") or "idle")
+
+ async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
+ state = self._state(mapping_id)
+ now = _now()
+ expires = state.get("lease_expires_at")
+ if state.get("lease_owner") is not None and isinstance(expires, datetime) and expires >= now:
+ return False
+ state.update(
+ {
+ "state": "rebuilding" if state.get("state") == "rebuild_required" else "running",
+ "active_sync_id": sync_id,
+ "lease_owner": owner,
+ "lease_expires_at": now + timedelta(seconds=lease_seconds),
+ "last_started_at": now,
+ }
+ )
+ return True
+
+ async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
+ state = self._state(mapping_id)
+ if state.get("lease_owner") != owner:
+ return False
+ state["lease_expires_at"] = _now() + timedelta(seconds=lease_seconds)
+ return True
+
+ async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
+ state = self._state(mapping_id)
+ if state.get("lease_owner") != owner or state.get("active_sync_id") != sync_id:
+ return
+ pages = [p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and not p.get("is_remote_deleted")]
+ was_rebuilding = state.get("state") == "rebuilding"
+ state.update(
+ {
+ "state": "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed",
+ "lease_owner": None,
+ "lease_expires_at": None,
+ "active_sync_id": None,
+ "last_error": error,
+ "remote_page_count": remote_page_count,
+ "local_page_count": len(pages),
+ "ready_page_count": sum(p.get("index_status") == "ready" for p in pages),
+ "failed_page_count": sum(p.get("index_status") in {"failed", "stale"} or p.get("last_error") is not None for p in pages),
+ "vector_count": sum(len(self.vectors.get(p["id"], [])) for p in pages),
+ "last_completed_at": _now() if success and not error else state.get("last_completed_at"),
+ "last_success_at": _now() if success else state.get("last_success_at"),
+ }
+ )
+
+ async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
+ wanted = set(mapping_ids or [])
+ return [dict(value) for key, value in self.states.items() if not wanted or key in wanted]
+
+ async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
+ for mapping_id in mapping_ids:
+ state = self._state(mapping_id)
+ state.update({"state": "rebuild_required", "embedding_fingerprint": fingerprint})
+ for page in self.pages.values():
+ if page["knowledge_base_mapping_id"] == mapping_id and not page.get("is_remote_deleted"):
+ page["index_status"] = "stale"
+ return len(mapping_ids)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/model.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/model.py
new file mode 100644
index 00000000..5a6f2c66
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/model.py
@@ -0,0 +1,94 @@
+"""Portable ORM rows for the DeerFlow-local WeKnora Wiki mirror/index."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from sqlalchemy import BigInteger, Boolean, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, false
+from sqlalchemy.orm import Mapped, mapped_column
+
+from deerflow.persistence.base import Base
+from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
+
+
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
+class LlmWikiWikiPageRow(Base):
+ __tablename__ = "llmwiki_wiki_pages"
+ __table_args__ = (
+ UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
+ Index("ix_llmwiki_wiki_page_mapping_visibility", "knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"),
+ )
+
+ id: Mapped[str] = mapped_column(String(36), primary_key=True)
+ knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False, index=True)
+ remote_page_id: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
+ slug: Mapped[str] = mapped_column(String(1024), nullable=False)
+ slug_hash: Mapped[str] = mapped_column(String(64), nullable=False)
+ title: Mapped[str] = mapped_column(String(512), nullable=False, default="", index=True)
+ summary: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
+ content_md: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
+ page_type: Mapped[str] = mapped_column(String(32), nullable=False, default="article", index=True)
+ status: Mapped[str] = mapped_column(String(32), nullable=False, default="published", index=True)
+ aliases: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
+ category_path: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
+ parent_slug: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+ wiki_path: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+ source_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
+ chunk_refs: Mapped[list] = mapped_column(PortableJSON(), nullable=False, default=list)
+ page_version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, index=True)
+ content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+ remote_created_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+ remote_updated_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
+ index_status: Mapped[str] = mapped_column(String(24), nullable=False, default="pending", index=True)
+ indexed_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
+ indexed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+ last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+ is_remote_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default=false(), index=True)
+ last_seen_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True, index=True)
+ created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
+ updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now, index=True)
+
+
+class LlmWikiWikiVectorRow(Base):
+ __tablename__ = "llmwiki_wiki_vectors"
+ __table_args__ = (
+ UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
+ Index("ix_llmwiki_wiki_vector_snapshot", "knowledge_base_mapping_id", "embedding_fingerprint"),
+ )
+
+ id: Mapped[str] = mapped_column(String(36), primary_key=True)
+ page_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False, index=True)
+ knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
+ section_index: Mapped[int] = mapped_column(Integer, nullable=False)
+ heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
+ section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
+ content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+ embedding_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
+ embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
+ vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
+ created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now)
+
+
+class LlmWikiWikiSyncStateRow(Base):
+ __tablename__ = "llmwiki_wiki_sync_states"
+
+ knowledge_base_mapping_id: Mapped[str] = mapped_column(String(36), ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True)
+ state: Mapped[str] = mapped_column(String(24), nullable=False, default="idle")
+ active_sync_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
+ lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True)
+ lease_expires_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+ last_started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+ last_completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+ last_success_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
+ last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+ remote_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ local_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ ready_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ vector_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ failed_page_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+ index_revision: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
+ embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True)
+ updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), nullable=False, default=_now, onupdate=_now)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py
new file mode 100644
index 00000000..c9d87b27
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py
@@ -0,0 +1,295 @@
+"""SQLAlchemy implementation of the local Wiki index store."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from uuid import uuid4
+
+from sqlalchemy import case, delete, func, or_, select, update
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from deerflow.persistence.llmwiki_index.base import LlmWikiIndexStore
+from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
+
+
+def _now() -> datetime:
+ return datetime.now(UTC)
+
+
+def _dict(row) -> dict[str, Any]:
+ value = row.to_dict()
+ for key, item in tuple(value.items()):
+ if isinstance(item, datetime):
+ value[key] = item.isoformat()
+ return value
+
+
+class SqlLlmWikiIndexStore(LlmWikiIndexStore):
+ def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
+ self._sf = session_factory
+
+ async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
+ row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
+ if row is None:
+ row = LlmWikiWikiSyncStateRow(knowledge_base_mapping_id=mapping_id, state="idle", index_revision=0, updated_at=_now())
+ session.add(row)
+ await session.flush()
+ return row
+
+ async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
+ stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
+ async with self._sf() as session:
+ row = (await session.execute(stmt)).scalar_one_or_none()
+ return _dict(row) if row else None
+
+ async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
+ stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
+ async with self._sf() as session:
+ row = (await session.execute(stmt)).scalar_one_or_none()
+ return _dict(row) if row else None
+
+ async def touch_page(self, page_id: str, sync_id: str) -> None:
+ async with self._sf() as session:
+ await session.execute(
+ update(LlmWikiWikiPageRow)
+ .where(LlmWikiWikiPageRow.id == page_id)
+ .values(last_seen_sync_id=sync_id, last_error=None, updated_at=_now())
+ )
+ await session.commit()
+
+ async def replace_page_vectors(self, *, page: dict[str, Any], vectors: list[dict[str, Any]], fingerprint: str, sync_id: str) -> dict[str, Any]:
+ mapping_id = str(page["knowledge_base_mapping_id"])
+ async with self._sf() as session:
+ stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"]).with_for_update()
+ row = (await session.execute(stmt)).scalar_one_or_none()
+ if row is not None and row.slug != page["slug"]:
+ raise ValueError("Wiki slug hash collision detected")
+ now = _now()
+ if row is None:
+ row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
+ session.add(row)
+ else:
+ for key, value in page.items():
+ setattr(row, key, value)
+ row.index_status = "ready"
+ row.indexed_fingerprint = fingerprint
+ row.indexed_at = now
+ row.last_error = None
+ row.is_remote_deleted = False
+ row.last_seen_sync_id = sync_id
+ row.updated_at = now
+ await session.flush()
+ await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
+ session.add_all(
+ [
+ LlmWikiWikiVectorRow(
+ id=str(uuid4()),
+ page_id=row.id,
+ knowledge_base_mapping_id=mapping_id,
+ section_index=int(item["section_index"]),
+ heading=item.get("heading"),
+ section_content=str(item["section_content"]),
+ content_hash=str(item["content_hash"]),
+ embedding_fingerprint=fingerprint,
+ embedding_dimensions=int(item["embedding_dimensions"]),
+ vector_blob=bytes(item["vector_blob"]),
+ created_at=now,
+ )
+ for item in vectors
+ ]
+ )
+ state = await self._ensure_state(session, mapping_id)
+ state.index_revision = int(state.index_revision or 0) + 1
+ state.embedding_fingerprint = fingerprint
+ state.updated_at = now
+ await session.commit()
+ return _dict(row)
+
+ async def record_page_failure(self, *, page: dict[str, Any], sync_id: str, error: str) -> None:
+ mapping_id = str(page["knowledge_base_mapping_id"])
+ safe_error = error[:4000]
+ async with self._sf() as session:
+ stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == page["slug_hash"])
+ row = (await session.execute(stmt)).scalar_one_or_none()
+ now = _now()
+ if row is None:
+ row = LlmWikiWikiPageRow(id=str(uuid4()), created_at=now, **page)
+ session.add(row)
+ row.index_status = "failed"
+ else:
+ vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))).scalar_one())
+ # Preserve the last atomically completed generation while the
+ # newer remote version remains pending after an embedding error.
+ row.index_status = "ready" if vector_count else "failed"
+ row.last_seen_sync_id = sync_id
+ row.last_error = safe_error
+ row.updated_at = now
+ await self._ensure_state(session, mapping_id)
+ await session.commit()
+
+ async def mark_missing_pages_deleted(self, mapping_id: str, sync_id: str) -> int:
+ async with self._sf() as session:
+ stmt = (
+ select(LlmWikiWikiPageRow)
+ .where(
+ LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id,
+ LlmWikiWikiPageRow.is_remote_deleted.is_(False),
+ or_(LlmWikiWikiPageRow.last_seen_sync_id.is_(None), LlmWikiWikiPageRow.last_seen_sync_id != sync_id),
+ )
+ .with_for_update()
+ )
+ rows = list((await session.execute(stmt)).scalars())
+ if not rows:
+ return 0
+ now = _now()
+ page_ids = [row.id for row in rows]
+ await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id.in_(page_ids)))
+ for row in rows:
+ row.is_remote_deleted = True
+ row.index_status = "deleted"
+ row.updated_at = now
+ state = await self._ensure_state(session, mapping_id)
+ state.index_revision = int(state.index_revision or 0) + 1
+ state.updated_at = now
+ await session.commit()
+ return len(rows)
+
+ async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]:
+ stmt = (
+ select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
+ .join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
+ .where(
+ LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
+ LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
+ LlmWikiWikiPageRow.index_status == "ready",
+ LlmWikiWikiPageRow.is_remote_deleted.is_(False),
+ LlmWikiWikiPageRow.status.in_(["published", "draft"]),
+ )
+ .order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
+ )
+ async with self._sf() as session:
+ state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
+ pairs = list((await session.execute(stmt)).all())
+ rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
+ return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
+
+ async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
+ async with self._sf() as session:
+ state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
+ if state is None:
+ return 0, None, "not_synced"
+ return int(state.index_revision or 0), state.embedding_fingerprint, state.state
+
+ async def try_acquire_sync_lease(self, mapping_id: str, *, owner: str, sync_id: str, lease_seconds: int, fingerprint: str) -> bool:
+ now = _now()
+ async with self._sf() as session:
+ try:
+ await self._ensure_state(session, mapping_id)
+ await session.commit()
+ except IntegrityError:
+ # Another worker created the one-per-KB state row between our
+ # read and insert. Roll back this isolated bootstrap session;
+ # the conditional UPDATE below is the actual lease election.
+ await session.rollback()
+ async with self._sf() as session:
+ stmt = (
+ update(LlmWikiWikiSyncStateRow)
+ .where(
+ LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id,
+ or_(
+ LlmWikiWikiSyncStateRow.lease_owner.is_(None),
+ LlmWikiWikiSyncStateRow.lease_expires_at.is_(None),
+ LlmWikiWikiSyncStateRow.lease_expires_at < now,
+ ),
+ )
+ .values(
+ state=case((LlmWikiWikiSyncStateRow.state == "rebuild_required", "rebuilding"), else_="running"),
+ active_sync_id=sync_id,
+ lease_owner=owner,
+ lease_expires_at=now + timedelta(seconds=lease_seconds),
+ last_started_at=now,
+ last_error=None,
+ updated_at=now,
+ )
+ )
+ result = await session.execute(stmt)
+ await session.commit()
+ return bool(result.rowcount)
+
+ async def renew_sync_lease(self, mapping_id: str, *, owner: str, lease_seconds: int) -> bool:
+ now = _now()
+ async with self._sf() as session:
+ result = await session.execute(
+ update(LlmWikiWikiSyncStateRow)
+ .where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiSyncStateRow.lease_owner == owner)
+ .values(lease_expires_at=now + timedelta(seconds=lease_seconds), updated_at=now)
+ )
+ await session.commit()
+ return bool(result.rowcount)
+
+ async def finish_sync(self, mapping_id: str, *, owner: str, sync_id: str, success: bool, remote_page_count: int, error: str | None = None) -> None:
+ async with self._sf() as session:
+ state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
+ if state is None or state.lease_owner != owner or state.active_sync_id != sync_id:
+ return
+ visible = [LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.is_remote_deleted.is_(False)]
+ local_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible))).scalar_one())
+ ready_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiPageRow).where(*visible, LlmWikiWikiPageRow.index_status == "ready"))).scalar_one())
+ failed_count = int(
+ (
+ await session.execute(
+ select(func.count())
+ .select_from(LlmWikiWikiPageRow)
+ .where(
+ *visible,
+ or_(
+ LlmWikiWikiPageRow.index_status.in_(["failed", "stale"]),
+ LlmWikiWikiPageRow.last_error.is_not(None),
+ ),
+ )
+ )
+ ).scalar_one()
+ )
+ vector_count = int((await session.execute(select(func.count()).select_from(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id))).scalar_one())
+ now = _now()
+ was_rebuilding = state.state == "rebuilding"
+ state.state = "partial" if success and error else "idle" if success else "rebuild_required" if was_rebuilding else "failed"
+ state.active_sync_id = None
+ state.lease_owner = None
+ state.lease_expires_at = None
+ state.last_error = (error or "")[:4000] or None
+ state.remote_page_count = remote_page_count
+ state.local_page_count = local_count
+ state.ready_page_count = ready_count
+ state.failed_page_count = failed_count
+ state.vector_count = vector_count
+ if success and not error:
+ state.last_completed_at = now
+ if success:
+ state.last_success_at = now
+ state.updated_at = now
+ await session.commit()
+
+ async def list_index_status(self, mapping_ids: list[str] | None = None) -> list[dict[str, Any]]:
+ stmt = select(LlmWikiWikiSyncStateRow)
+ if mapping_ids:
+ stmt = stmt.where(LlmWikiWikiSyncStateRow.knowledge_base_mapping_id.in_(mapping_ids))
+ stmt = stmt.order_by(LlmWikiWikiSyncStateRow.updated_at.desc())
+ async with self._sf() as session:
+ return [_dict(row) for row in (await session.execute(stmt)).scalars()]
+
+ async def mark_rebuild_required(self, mapping_ids: list[str], fingerprint: str) -> int:
+ if not mapping_ids:
+ return 0
+ now = _now()
+ async with self._sf() as session:
+ for mapping_id in mapping_ids:
+ state = await self._ensure_state(session, mapping_id)
+ state.state = "rebuild_required"
+ state.embedding_fingerprint = fingerprint
+ state.updated_at = now
+ await session.execute(update(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id.in_(mapping_ids), LlmWikiWikiPageRow.is_remote_deleted.is_(False)).values(index_status="stale", updated_at=now))
+ await session.commit()
+ return len(mapping_ids)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_01_fixed_questions.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_01_fixed_questions.py
new file mode 100644
index 00000000..dfeca77f
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_01_fixed_questions.py
@@ -0,0 +1,51 @@
+"""Create administrator-managed exact-match fixed questions.
+
+Revision ID: 20260821_01
+Revises: 20260816_01
+Create Date: 2026-08-21
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import mysql
+
+revision = "20260821_01"
+down_revision = "20260816_01"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ bind = op.get_bind()
+ if sa.inspect(bind).has_table("fixed_questions"):
+ return
+ long_text = sa.Text().with_variant(mysql.LONGTEXT(), "mysql")
+ op.create_table(
+ "fixed_questions",
+ sa.Column("id", sa.String(length=64), primary_key=True),
+ sa.Column("question", sa.Text(), nullable=False),
+ sa.Column("question_hash", sa.String(length=64), nullable=False),
+ sa.Column("answer", long_text, nullable=False),
+ sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
+ sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"),
+ sa.Column("updated_by", sa.String(length=64), nullable=True),
+ sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+ sa.UniqueConstraint(
+ "question_hash",
+ name="uq_fixed_questions_question_hash",
+ ),
+ )
+ op.create_index(
+ "ix_fixed_questions_question_hash",
+ "fixed_questions",
+ ["question_hash"],
+ unique=False,
+ )
+
+
+def downgrade() -> None:
+ bind = op.get_bind()
+ if sa.inspect(bind).has_table("fixed_questions"):
+ op.drop_table("fixed_questions")
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_02_fixed_question_stream_speed.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_02_fixed_question_stream_speed.py
new file mode 100644
index 00000000..b26747b7
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260821_02_fixed_question_stream_speed.py
@@ -0,0 +1,44 @@
+"""Add configurable fixed-answer streaming speed.
+
+Revision ID: 20260821_02
+Revises: 20260821_01
+Create Date: 2026-08-21
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "20260821_02"
+down_revision = "20260821_01"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ bind = op.get_bind()
+ inspector = sa.inspect(bind)
+ if not inspector.has_table("fixed_questions"):
+ return
+ columns = {column["name"] for column in inspector.get_columns("fixed_questions")}
+ if "tokens_per_second" not in columns:
+ op.add_column(
+ "fixed_questions",
+ sa.Column(
+ "tokens_per_second",
+ sa.Integer(),
+ nullable=False,
+ server_default="100",
+ ),
+ )
+
+
+def downgrade() -> None:
+ bind = op.get_bind()
+ inspector = sa.inspect(bind)
+ if not inspector.has_table("fixed_questions"):
+ return
+ columns = {column["name"] for column in inspector.get_columns("fixed_questions")}
+ if "tokens_per_second" in columns:
+ op.drop_column("fixed_questions", "tokens_per_second")
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260823_01_llmwiki_local_wiki_index.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260823_01_llmwiki_local_wiki_index.py
new file mode 100644
index 00000000..e32dce19
--- /dev/null
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260823_01_llmwiki_local_wiki_index.py
@@ -0,0 +1,143 @@
+"""Add DeerFlow-local WeKnora Wiki mirror and vector index.
+
+Revision ID: 20260823_01
+Revises: 20260821_02
+Create Date: 2026-08-23
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import mysql
+
+revision = "20260823_01"
+down_revision = "20260821_02"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ bind = op.get_bind()
+ inspector = sa.inspect(bind)
+ columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
+ if "wiki_index_enabled" not in columns:
+ op.add_column("llmwiki_knowledge_bases", sa.Column("wiki_index_enabled", sa.Boolean(), nullable=False, server_default=sa.true()))
+ if "external_search_enabled" not in columns:
+ op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_enabled", sa.Boolean(), nullable=False, server_default=sa.false()))
+ if "external_search_updated_at" not in columns:
+ op.add_column("llmwiki_knowledge_bases", sa.Column("external_search_updated_at", sa.DateTime(timezone=True), nullable=True))
+ mapping_indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
+ if "ix_llmwiki_knowledge_bases_wiki_index_enabled" not in mapping_indexes:
+ op.create_index("ix_llmwiki_knowledge_bases_wiki_index_enabled", "llmwiki_knowledge_bases", ["wiki_index_enabled"])
+ if "ix_llmwiki_knowledge_bases_external_search_enabled" not in mapping_indexes:
+ op.create_index("ix_llmwiki_knowledge_bases_external_search_enabled", "llmwiki_knowledge_bases", ["external_search_enabled"])
+
+ long_text = sa.Text().with_variant(mysql.LONGTEXT(), "mysql")
+ portable_json = sa.JSON().with_variant(sa.Text(), "mysql")
+ if not inspector.has_table("llmwiki_wiki_pages"):
+ op.create_table(
+ "llmwiki_wiki_pages",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), nullable=False),
+ sa.Column("remote_page_id", sa.String(128), nullable=True),
+ sa.Column("slug", sa.String(1024), nullable=False),
+ sa.Column("slug_hash", sa.String(64), nullable=False),
+ sa.Column("title", sa.String(512), nullable=False, server_default=""),
+ sa.Column("summary", long_text, nullable=False),
+ sa.Column("content_md", long_text, nullable=False),
+ sa.Column("page_type", sa.String(32), nullable=False, server_default="article"),
+ sa.Column("status", sa.String(32), nullable=False, server_default="published"),
+ sa.Column("aliases", portable_json, nullable=False),
+ sa.Column("category_path", portable_json, nullable=False),
+ sa.Column("parent_slug", sa.String(1024), nullable=True),
+ sa.Column("wiki_path", sa.String(1024), nullable=True),
+ sa.Column("source_refs", portable_json, nullable=False),
+ sa.Column("chunk_refs", portable_json, nullable=False),
+ sa.Column("page_version", sa.Integer(), nullable=False, server_default="0"),
+ sa.Column("content_hash", sa.String(64), nullable=False),
+ sa.Column("remote_created_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("remote_updated_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("index_status", sa.String(24), nullable=False, server_default="pending"),
+ sa.Column("indexed_fingerprint", sa.String(64), nullable=True),
+ sa.Column("indexed_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("last_error", sa.Text(), nullable=True),
+ sa.Column("is_remote_deleted", sa.Boolean(), nullable=False, server_default=sa.false()),
+ sa.Column("last_seen_sync_id", sa.String(36), nullable=True),
+ sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+ sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+ sa.UniqueConstraint("knowledge_base_mapping_id", "slug_hash", name="uq_llmwiki_wiki_page_mapping_slug_hash"),
+ )
+ op.create_index("ix_llmwiki_wiki_pages_knowledge_base_mapping_id", "llmwiki_wiki_pages", ["knowledge_base_mapping_id"])
+ op.create_index("ix_llmwiki_wiki_page_mapping_visibility", "llmwiki_wiki_pages", ["knowledge_base_mapping_id", "status", "index_status", "is_remote_deleted"])
+ op.create_index("ix_llmwiki_wiki_pages_remote_page_id", "llmwiki_wiki_pages", ["remote_page_id"])
+ op.create_index("ix_llmwiki_wiki_pages_title", "llmwiki_wiki_pages", ["title"])
+ op.create_index("ix_llmwiki_wiki_pages_page_type", "llmwiki_wiki_pages", ["page_type"])
+ op.create_index("ix_llmwiki_wiki_pages_status", "llmwiki_wiki_pages", ["status"])
+ op.create_index("ix_llmwiki_wiki_pages_page_version", "llmwiki_wiki_pages", ["page_version"])
+ op.create_index("ix_llmwiki_wiki_pages_content_hash", "llmwiki_wiki_pages", ["content_hash"])
+ op.create_index("ix_llmwiki_wiki_pages_remote_updated_at", "llmwiki_wiki_pages", ["remote_updated_at"])
+ op.create_index("ix_llmwiki_wiki_pages_index_status", "llmwiki_wiki_pages", ["index_status"])
+ op.create_index("ix_llmwiki_wiki_pages_is_remote_deleted", "llmwiki_wiki_pages", ["is_remote_deleted"])
+ op.create_index("ix_llmwiki_wiki_pages_last_seen_sync_id", "llmwiki_wiki_pages", ["last_seen_sync_id"])
+ op.create_index("ix_llmwiki_wiki_pages_updated_at", "llmwiki_wiki_pages", ["updated_at"])
+
+ if not inspector.has_table("llmwiki_wiki_vectors"):
+ op.create_table(
+ "llmwiki_wiki_vectors",
+ sa.Column("id", sa.String(36), primary_key=True),
+ sa.Column("page_id", sa.String(36), sa.ForeignKey("llmwiki_wiki_pages.id", ondelete="CASCADE"), nullable=False),
+ sa.Column("knowledge_base_mapping_id", sa.String(36), nullable=False),
+ sa.Column("section_index", sa.Integer(), nullable=False),
+ sa.Column("heading", sa.String(1024), nullable=True),
+ sa.Column("section_content", long_text, nullable=False),
+ sa.Column("content_hash", sa.String(64), nullable=False),
+ sa.Column("embedding_fingerprint", sa.String(64), nullable=False),
+ sa.Column("embedding_dimensions", sa.Integer(), nullable=False),
+ sa.Column("vector_blob", sa.LargeBinary(), nullable=False),
+ sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+ sa.UniqueConstraint("page_id", "section_index", "embedding_fingerprint", name="uq_llmwiki_wiki_vector_page_section_fingerprint"),
+ )
+ op.create_index("ix_llmwiki_wiki_vector_snapshot", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id", "embedding_fingerprint"])
+ op.create_index("ix_llmwiki_wiki_vectors_page_id", "llmwiki_wiki_vectors", ["page_id"])
+ op.create_index("ix_llmwiki_wiki_vectors_knowledge_base_mapping_id", "llmwiki_wiki_vectors", ["knowledge_base_mapping_id"])
+ op.create_index("ix_llmwiki_wiki_vectors_content_hash", "llmwiki_wiki_vectors", ["content_hash"])
+ op.create_index("ix_llmwiki_wiki_vectors_embedding_fingerprint", "llmwiki_wiki_vectors", ["embedding_fingerprint"])
+
+ if not inspector.has_table("llmwiki_wiki_sync_states"):
+ op.create_table(
+ "llmwiki_wiki_sync_states",
+ sa.Column("knowledge_base_mapping_id", sa.String(36), sa.ForeignKey("llmwiki_knowledge_bases.id", ondelete="CASCADE"), primary_key=True),
+ sa.Column("state", sa.String(24), nullable=False, server_default="idle"),
+ sa.Column("active_sync_id", sa.String(36), nullable=True),
+ sa.Column("lease_owner", sa.String(128), nullable=True),
+ sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("last_started_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("last_completed_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("last_error", sa.Text(), nullable=True),
+ sa.Column("remote_page_count", sa.Integer(), nullable=False, server_default="0"),
+ sa.Column("local_page_count", sa.Integer(), nullable=False, server_default="0"),
+ sa.Column("ready_page_count", sa.Integer(), nullable=False, server_default="0"),
+ sa.Column("vector_count", sa.Integer(), nullable=False, server_default="0"),
+ sa.Column("failed_page_count", sa.Integer(), nullable=False, server_default="0"),
+ sa.Column("index_revision", sa.BigInteger(), nullable=False, server_default="0"),
+ sa.Column("embedding_fingerprint", sa.String(64), nullable=True),
+ sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+ )
+
+
+def downgrade() -> None:
+ bind = op.get_bind()
+ inspector = sa.inspect(bind)
+ for table in ("llmwiki_wiki_vectors", "llmwiki_wiki_pages", "llmwiki_wiki_sync_states"):
+ if inspector.has_table(table):
+ op.drop_table(table)
+ columns = {column["name"] for column in inspector.get_columns("llmwiki_knowledge_bases")}
+ indexes = {index["name"] for index in inspector.get_indexes("llmwiki_knowledge_bases")}
+ for index in ("ix_llmwiki_knowledge_bases_external_search_enabled", "ix_llmwiki_knowledge_bases_wiki_index_enabled"):
+ if index in indexes:
+ op.drop_index(index, table_name="llmwiki_knowledge_bases")
+ for column in ("external_search_updated_at", "external_search_enabled", "wiki_index_enabled"):
+ if column in columns:
+ op.drop_column("llmwiki_knowledge_bases", column)
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py
index 69eb7971..8c5b4b32 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/models/__init__.py
@@ -47,10 +47,12 @@ from deerflow.persistence.deep_research_sources.model import DeepResearchSourceR
from deerflow.persistence.document_rewrite_versions.model import DocumentRewriteVersionRow
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
from deerflow.persistence.feedback.model import FeedbackRow
+from deerflow.persistence.fixed_questions.model import FixedQuestionRow
from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
+from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
from deerflow.persistence.memory_config.model import UserMemoryConfigRow
from deerflow.persistence.menu_overrides.model import MenuOverrideRow
from deerflow.persistence.models.run_event import RunEventRow
@@ -98,6 +100,7 @@ from deerflow.persistence.user_prompts.model import UserCustomPromptRow
__all__ = [
"FeedbackRow",
+ "FixedQuestionRow",
"AgentExtrasRow",
"AgentFavoriteRow",
"AgentRow",
@@ -128,6 +131,9 @@ __all__ = [
"LlmCallMetricRow",
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
+ "LlmWikiWikiPageRow",
+ "LlmWikiWikiVectorRow",
+ "LlmWikiWikiSyncStateRow",
"MenuOverrideRow",
"NotificationRow",
"ParallelAgentTaskRow",
diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/llmwiki_search_tool.py b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/llmwiki_search_tool.py
index 45f26202..35735168 100644
--- a/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/llmwiki_search_tool.py
+++ b/offline-backend-20260512/backend/packages/harness/deerflow/tools/builtins/llmwiki_search_tool.py
@@ -6,28 +6,25 @@ import json
import re
from collections.abc import Mapping
from typing import Any
-from urllib.parse import urlencode
from langchain.tools import ToolRuntime, tool
from langgraph.typing import ContextT
from deerflow.agents.thread_state import ThreadState
+from deerflow.config import get_app_config
from deerflow.config.agents_config import load_agent_config
-from deerflow.integrations.weknora.client import WeKnoraError
-from deerflow.integrations.weknora.runtime import build_weknora_client, get_llmwiki_store, get_resolved_llmwiki_runtime
+from deerflow.integrations.weknora.local_index.runtime import get_local_wiki_retrieval_service
+from deerflow.integrations.weknora.local_index.search import WikiIndexError
+from deerflow.integrations.weknora.runtime import get_llmwiki_store, get_resolved_llmwiki_runtime
from deerflow.runtime.user_context import get_current_user, get_effective_user_id
-
_CONVERSATION_DEPOSIT_NAME = "对话沉淀"
_CONVERSATION_DEPOSIT_OWNER_USER_ID = "system"
_LEGACY_BRAND_PATTERN = re.compile(r"deer[\s_-]*flow", re.IGNORECASE)
def _is_conversation_deposit_mapping(row: Mapping[str, Any]) -> bool:
- return (
- str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID
- and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
- )
+ return str(row.get("owner_user_id") or "") == _CONVERSATION_DEPOSIT_OWNER_USER_ID and str(row.get("name") or "").strip() == _CONVERSATION_DEPOSIT_NAME
def _sanitize_conversation_deposit_value(value: Any) -> Any:
@@ -79,9 +76,10 @@ async def llmwiki_search_tool(
query: Focused natural-language retrieval query.
"""
+ local_config = get_app_config().llmwiki.local_wiki_index
resolved = get_resolved_llmwiki_runtime()
- if not resolved.weknora_enabled:
- return "LLMWiki retrieval is unavailable because WeKnora mode is not enabled."
+ if not (local_config.enabled and local_config.internal_search_enabled) and not resolved.weknora_enabled:
+ return "LLMWiki retrieval is unavailable because no retrieval provider is enabled."
store = get_llmwiki_store()
user_id = get_effective_user_id()
@@ -111,52 +109,26 @@ async def llmwiki_search_tool(
return "No authorized LLMWiki knowledge bases are available."
try:
- results = await build_weknora_client(resolved).search(
- query.strip(),
- [str(row["weknora_id"]) for row in rows],
- )
- except (RuntimeError, WeKnoraError) as exc:
- return f"LLMWiki retrieval failed: {exc}"
-
- local_by_remote = {str(row["weknora_id"]): row for row in rows}
- citations = []
- for result in results:
- mapping = local_by_remote.get(str(result.get("knowledge_base_id") or ""))
- if mapping is None:
- continue
- visible_result = (
- _sanitize_conversation_deposit_value(result)
- if _is_conversation_deposit_mapping(mapping)
- else result
- )
- params = urlencode(
- {
- "knowledge_base_id": mapping["id"],
- "chunk_id": visible_result.get("chunk_id") or "",
- "knowledge_id": visible_result.get("knowledge_id") or "",
- "document_id": visible_result.get("knowledge_id") or "",
- "wiki_slug": visible_result.get("wiki_slug") or visible_result.get("slug") or "",
- }
- )
- citations.append(
- {
- "title": visible_result.get("title") or visible_result.get("filename") or mapping.get("name") or "LLMWiki source",
- "content": visible_result.get("content") or "",
- "score": visible_result.get("score") or 0,
- "knowledge_base_id": mapping["id"],
- "knowledge_base_name": mapping.get("name") or "",
- "source": mapping.get("name") or "LLMWiki",
- "knowledge_id": visible_result.get("knowledge_id") or "",
- "chunk_id": visible_result.get("chunk_id") or "",
- "url": f"/weknora-source-preview?{params}",
- }
- )
+ payload = await get_local_wiki_retrieval_service().search(query.strip(), rows)
+ except (RuntimeError, WikiIndexError) as exc:
+ code = exc.code if isinstance(exc, WikiIndexError) else "WIKI_RETRIEVAL_UNAVAILABLE"
+ return f"LLMWiki Wiki-only retrieval failed ({code}); retrieval is unavailable, not empty."
+ mapping_by_id = {str(row["id"]): row for row in rows}
+ results = []
+ for item in payload["results"]:
+ mapping = mapping_by_id.get(str(item.get("knowledge_base_id") or ""))
+ visible = _sanitize_conversation_deposit_value(item) if mapping and _is_conversation_deposit_mapping(mapping) else item
+ results.append(visible)
return json.dumps(
{
"query": query,
"selection": [row["id"] for row in rows],
"knowledge_base_count": len(rows),
- "results": citations,
+ "mode": payload.get("mode") or "wiki_api",
+ "results": results,
+ "partial": payload.get("partial", False),
+ "warnings": payload.get("warnings", []),
+ "fallback_knowledge_base_ids": payload.get("fallback_knowledge_base_ids", []),
},
ensure_ascii=False,
)
diff --git a/offline-backend-20260512/backend/packages/harness/pyproject.toml b/offline-backend-20260512/backend/packages/harness/pyproject.toml
index 2c47c1a5..3325f550 100644
--- a/offline-backend-20260512/backend/packages/harness/pyproject.toml
+++ b/offline-backend-20260512/backend/packages/harness/pyproject.toml
@@ -21,7 +21,8 @@ dependencies = [
"langgraph-cli>=0.4.24",
"langgraph-runtime-inmem>=0.28.0",
"markdownify>=1.2.2",
- "markitdown[all,xlsx]>=0.0.1a2",
+ "markitdown[all,xlsx]>=0.0.1a2",
+ "numpy>=2.0.0",
"pydantic>=2.12.5",
"pyyaml>=6.0.3",
"readabilipy>=0.3.0",
diff --git a/offline-backend-20260512/backend/pyproject.toml b/offline-backend-20260512/backend/pyproject.toml
index 45a21bbb..2f9f2e87 100644
--- a/offline-backend-20260512/backend/pyproject.toml
+++ b/offline-backend-20260512/backend/pyproject.toml
@@ -24,6 +24,7 @@ dependencies = [
"croniter>=6.2.2",
"hindsight-client>=0.4.22",
"asyncmy>=0.2.11",
+ "openpyxl>=3.1.5",
]
[project.optional-dependencies]
diff --git a/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py b/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py
new file mode 100644
index 00000000..4837f9a0
--- /dev/null
+++ b/offline-backend-20260512/backend/tests/test_es_query_routing_prompt.py
@@ -0,0 +1,70 @@
+from types import SimpleNamespace
+
+from deerflow.agents.lead_agent import prompt as prompt_module
+from deerflow.config.skills_config import EsQueryRoutingConfig, SkillsConfig
+
+
+def _app_config(*, enabled: bool, prompt: str = "自定义 ES 路由规则") -> SimpleNamespace:
+ return SimpleNamespace(
+ skills=SkillsConfig(
+ es_query_routing=EsQueryRoutingConfig(enabled=enabled, prompt=prompt),
+ )
+ )
+
+
+def test_es_query_routing_config_is_disabled_by_default() -> None:
+ config = SkillsConfig()
+
+ assert config.es_query_routing.enabled is False
+ assert "es_query" in config.es_query_routing.prompt
+
+
+def test_es_query_routing_section_honors_switch_prompt_and_allowlist() -> None:
+ enabled = _app_config(enabled=True, prompt=" 自定义 ES 路由规则 ")
+
+ assert prompt_module._build_es_query_routing_section(app_config=enabled) == (
+ '\n自定义 ES 路由规则\n'
+ )
+ assert (
+ prompt_module._build_es_query_routing_section(
+ app_config=enabled,
+ available_skills={"other_skill"},
+ )
+ == ""
+ )
+ assert "自定义 ES 路由规则" in prompt_module._build_es_query_routing_section(
+ app_config=enabled,
+ available_skills={"es_query"},
+ )
+ assert prompt_module._build_es_query_routing_section(app_config=_app_config(enabled=False)) == ""
+ assert prompt_module._build_es_query_routing_section(app_config=_app_config(enabled=True, prompt=" ")) == ""
+
+
+def test_es_query_routing_is_injected_into_lead_and_allowed_custom_agent_prompts(monkeypatch) -> None:
+ config = _app_config(enabled=True)
+ monkeypatch.setattr(prompt_module, "_get_memory_context", lambda *args, **kwargs: "")
+ monkeypatch.setattr(prompt_module, "get_agent_soul", lambda *args, **kwargs: "")
+ monkeypatch.setattr(prompt_module, "get_skills_prompt_section", lambda *args, **kwargs: "")
+ monkeypatch.setattr(prompt_module, "get_deferred_tools_prompt_section", lambda *args, **kwargs: "")
+ monkeypatch.setattr(prompt_module, "_build_acp_section", lambda *args, **kwargs: "")
+ monkeypatch.setattr(prompt_module, "_build_custom_mounts_section", lambda *args, **kwargs: "")
+ monkeypatch.setattr(prompt_module, "_current_time_context", lambda: "")
+
+ lead_prompt = prompt_module.apply_prompt_template(app_config=config)
+ allowed_custom_prompt = prompt_module.apply_agent_only_prompt_template(
+ agent_id="agent-with-es",
+ agent_name="Agent with ES",
+ available_skills={"es_query"},
+ app_config=config,
+ )
+ denied_custom_prompt = prompt_module.apply_agent_only_prompt_template(
+ agent_id="agent-without-es",
+ agent_name="Agent without ES",
+ available_skills={"other_skill"},
+ app_config=config,
+ )
+
+ assert '' in lead_prompt
+ assert "自定义 ES 路由规则" in lead_prompt
+ assert '' in allowed_custom_prompt
+ assert '' not in denied_custom_prompt
diff --git a/offline-backend-20260512/backend/tests/test_external_llmwiki_search.py b/offline-backend-20260512/backend/tests/test_external_llmwiki_search.py
new file mode 100644
index 00000000..2e748a13
--- /dev/null
+++ b/offline-backend-20260512/backend/tests/test_external_llmwiki_search.py
@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from fastapi import HTTPException
+from starlette.requests import Request
+
+from app.gateway.auth_middleware import _authorize_external_llmwiki
+from app.gateway.routers.external_llmwiki import _external_mappings
+
+
+def _request(key: str) -> Request:
+ return Request({"type": "http", "method": "POST", "path": "/api/external/llmwiki/wiki/vector-search", "headers": [(b"x-api-key", key.encode())]})
+
+
+def test_external_api_key_uses_isolated_fail_closed_auth(monkeypatch) -> None:
+ config = SimpleNamespace(enabled=True, api_key="current", previous_api_key="previous")
+ monkeypatch.setattr("deerflow.config.get_app_config", lambda: SimpleNamespace(llmwiki=SimpleNamespace(local_wiki_index=SimpleNamespace(external_api=config))))
+ assert _authorize_external_llmwiki(_request("current")) is None
+ assert _authorize_external_llmwiki(_request("previous")) is None
+ rejected = _authorize_external_llmwiki(_request("wrong"))
+ assert rejected is not None
+ assert rejected.status_code == 401
+
+
+@pytest.mark.asyncio
+async def test_external_mappings_require_explicit_public_external_permission() -> None:
+ rows = [
+ {"id": "allowed", "publication_status": "published", "external_search_enabled": True, "wiki_index_enabled": True},
+ {"id": "private", "publication_status": "private", "external_search_enabled": True, "wiki_index_enabled": True},
+ {"id": "not-opted-in", "publication_status": "published", "external_search_enabled": False, "wiki_index_enabled": True},
+ {
+ "id": "deposit",
+ "name": "对话沉淀",
+ "owner_user_id": "system",
+ "publication_status": "published",
+ "external_search_enabled": True,
+ "wiki_index_enabled": True,
+ },
+ ]
+
+ class Store:
+ async def list_visible(self, *_args, **_kwargs):
+ return rows
+
+ request = Request({"type": "http", "method": "POST", "path": "/", "headers": [], "app": SimpleNamespace(state=SimpleNamespace(llmwiki_store=Store()))})
+ assert [row["id"] for row in await _external_mappings(request, [])] == ["allowed"]
+ with pytest.raises(HTTPException) as exc_info:
+ await _external_mappings(request, ["not-opted-in"])
+ assert exc_info.value.status_code == 404
diff --git a/offline-backend-20260512/backend/tests/test_fixed_questions.py b/offline-backend-20260512/backend/tests/test_fixed_questions.py
new file mode 100644
index 00000000..04a7af2b
--- /dev/null
+++ b/offline-backend-20260512/backend/tests/test_fixed_questions.py
@@ -0,0 +1,288 @@
+"""Regression tests for administrator-managed exact-match fixed answers."""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from langchain_core.messages import HumanMessage
+from pydantic import ValidationError
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+from app.gateway.routers.fixed_questions import (
+ FixedQuestion,
+ FixedQuestionReplaceRequest,
+)
+from app.gateway.services import (
+ RUN_SCOPE_EXTERNAL_ROOT,
+ RUN_SCOPE_ORCHESTRATION_CHILD,
+ _find_fixed_answer,
+ normalize_input,
+)
+from deerflow.agents import fixed_answer_agent as fixed_answer_module
+from deerflow.agents.fixed_answer_agent import make_fixed_answer_agent
+from deerflow.persistence.fixed_questions import MemoryFixedQuestionStore
+from deerflow.persistence.fixed_questions.model import FixedQuestionRow
+from deerflow.persistence.fixed_questions.sql import FixedQuestionRepository
+
+
+def _body(**overrides):
+ data = {
+ "assistant_id": "lead_agent",
+ "command": None,
+ "context": {},
+ }
+ data.update(overrides)
+ return SimpleNamespace(**data)
+
+
+def _request(store: MemoryFixedQuestionStore):
+ return SimpleNamespace(
+ app=SimpleNamespace(
+ state=SimpleNamespace(fixed_question_store=store),
+ )
+ )
+
+
+@pytest.mark.asyncio
+async def test_memory_store_matches_only_enabled_exact_question() -> None:
+ store = MemoryFixedQuestionStore()
+ await store.replace_all(
+ [
+ {
+ "id": "enabled",
+ "question": "系统开放时间?",
+ "answer": "工作日 09:00—18:00。",
+ "enabled": True,
+ "tokens_per_second": 80,
+ "sort_order": 0,
+ },
+ {
+ "id": "disabled",
+ "question": "停用问题",
+ "answer": "不应返回",
+ "enabled": False,
+ "sort_order": 1,
+ },
+ ]
+ )
+
+ match = await store.find_enabled("系统开放时间?")
+ assert match is not None
+ assert match["answer"] == "工作日 09:00—18:00。"
+ assert match["tokens_per_second"] == 80
+ assert await store.find_enabled("系统开放时间?") is None
+ assert await store.find_enabled(" 系统开放时间?") is None
+ assert await store.find_enabled("停用问题") is None
+
+
+@pytest.mark.asyncio
+async def test_sql_store_round_trips_and_exact_matches() -> None:
+ engine = create_async_engine("sqlite+aiosqlite:///:memory:")
+ try:
+ async with engine.begin() as connection:
+ await connection.run_sync(FixedQuestionRow.__table__.create)
+ repository = FixedQuestionRepository(async_sessionmaker(engine, expire_on_commit=False))
+ saved = await repository.replace_all(
+ [
+ {
+ "id": "sql-q1",
+ "question": "SQL 精确问题",
+ "answer": "SQL 固定回答",
+ "enabled": True,
+ "tokens_per_second": 120,
+ "sort_order": 0,
+ }
+ ],
+ updated_by="admin",
+ )
+ assert saved[0]["updated_by"] == "admin"
+ assert saved[0]["tokens_per_second"] == 120
+ assert (await repository.find_enabled("SQL 精确问题"))["answer"] == "SQL 固定回答"
+ assert await repository.find_enabled("sql 精确问题") is None
+ finally:
+ await engine.dispose()
+
+
+@pytest.mark.asyncio
+async def test_lookup_skips_attachments_and_internal_runs() -> None:
+ store = MemoryFixedQuestionStore()
+ await store.replace_all(
+ [
+ {
+ "id": "q1",
+ "question": "固定问题",
+ "answer": "固定回答",
+ "enabled": True,
+ "sort_order": 0,
+ }
+ ]
+ )
+ request = _request(store)
+
+ plain_input = normalize_input({"messages": [{"content": "固定问题"}]})
+ match = await _find_fixed_answer(
+ body=_body(),
+ request=request,
+ graph_input=plain_input,
+ run_scope=RUN_SCOPE_EXTERNAL_ROOT,
+ )
+ assert match is not None
+ assert match["id"] == "q1"
+
+ attachment_input = normalize_input(
+ {
+ "messages": [
+ {
+ "content": "固定问题",
+ "additional_kwargs": {"files": [{"name": "context.pdf"}]},
+ }
+ ]
+ }
+ )
+ assert (
+ await _find_fixed_answer(
+ body=_body(),
+ request=request,
+ graph_input=attachment_input,
+ run_scope=RUN_SCOPE_EXTERNAL_ROOT,
+ )
+ is None
+ )
+ contextual_input = normalize_input(
+ {
+ "messages": [
+ {
+ "content": "固定问题",
+ "additional_kwargs": {"prompt_prefix": "网页正文上下文"},
+ }
+ ]
+ }
+ )
+ assert (
+ await _find_fixed_answer(
+ body=_body(),
+ request=request,
+ graph_input=contextual_input,
+ run_scope=RUN_SCOPE_EXTERNAL_ROOT,
+ )
+ is None
+ )
+ assert (
+ await _find_fixed_answer(
+ body=_body(),
+ request=request,
+ graph_input=plain_input,
+ run_scope=RUN_SCOPE_ORCHESTRATION_CHILD,
+ )
+ is None
+ )
+
+
+@pytest.mark.asyncio
+async def test_fixed_answer_agent_streams_chunks_and_persists_full_answer() -> None:
+ question = "固定问题"
+ answer = "这是一个足够长的固定回答,用于验证返回内容会拆成多个流式消息块,同时完整写入会话状态。"
+ agent = make_fixed_answer_agent(
+ question=question,
+ answer=answer,
+ fixed_question_id="q1",
+ tokens_per_second=1000,
+ )
+
+ chunks: list[str] = []
+ final_state = None
+ async for mode, data in agent.astream(
+ {"messages": [HumanMessage(content=question)]},
+ stream_mode=["values", "messages"],
+ ):
+ if mode == "messages":
+ message, _metadata = data
+ if isinstance(message.content, str):
+ chunks.append(message.content)
+ elif mode == "values":
+ final_state = data
+
+ assert "".join(chunks) == answer
+ assert len([chunk for chunk in chunks if chunk]) >= 2
+ assert final_state is not None
+ assert final_state["messages"][-1].content == answer
+ assert final_state["messages"][-1].additional_kwargs["fixed_answer"] is True
+ assert final_state["messages"][-1].additional_kwargs["tokens_per_second"] == 1000
+ assert final_state["title"] == question
+ assert final_state["title_provisional"] is False
+
+
+@pytest.mark.asyncio
+async def test_fixed_answer_default_speed_schedules_100_tokens_per_second(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ answer = "one two three four five six seven eight nine ten eleven twelve thirteen"
+ sleeps: list[float] = []
+
+ async def record_sleep(delay: float) -> None:
+ sleeps.append(delay)
+
+ monkeypatch.setattr(fixed_answer_module.asyncio, "sleep", record_sleep)
+ model = fixed_answer_module.FixedAnswerChatModel(
+ answer=answer,
+ message_id="speed-test",
+ fixed_question_id="q-speed",
+ )
+
+ chunks: list[str] = []
+ async for generation in model._astream([]):
+ chunks.append(str(generation.message.content))
+
+ token_count = len(fixed_answer_module._TOKEN_ENCODING.encode(answer))
+ assert "".join(chunks) == answer
+ assert model.tokens_per_second == 100
+ assert sum(sleeps) == pytest.approx((token_count - 1) / 100)
+
+
+def test_replace_request_rejects_duplicate_exact_questions() -> None:
+ row = {
+ "question": "重复问题",
+ "answer": "回答",
+ "enabled": True,
+ "tokens_per_second": 100,
+ "sort_order": 0,
+ }
+ with pytest.raises(ValidationError):
+ FixedQuestionReplaceRequest(
+ questions=[
+ FixedQuestion(id="q1", **row),
+ FixedQuestion(id="q2", **row),
+ ]
+ )
+ with pytest.raises(ValidationError):
+ FixedQuestionReplaceRequest(
+ questions=[
+ FixedQuestion(id="same-id", **row),
+ FixedQuestion(
+ id="same-id",
+ question="另一个问题",
+ answer="另一个回答",
+ enabled=True,
+ sort_order=1,
+ ),
+ ]
+ )
+
+
+def test_fixed_question_stream_speed_defaults_and_bounds() -> None:
+ default_row = FixedQuestion(
+ id="default-speed",
+ question="默认速度",
+ answer="回答",
+ )
+ assert default_row.tokens_per_second == 100
+
+ for invalid_speed in (0, 1001):
+ with pytest.raises(ValidationError):
+ FixedQuestion(
+ id=f"invalid-{invalid_speed}",
+ question="非法速度",
+ answer="回答",
+ tokens_per_second=invalid_speed,
+ )
diff --git a/offline-backend-20260512/backend/tests/test_health_auth_bypass.py b/offline-backend-20260512/backend/tests/test_health_auth_bypass.py
index f027799c..9428949d 100644
--- a/offline-backend-20260512/backend/tests/test_health_auth_bypass.py
+++ b/offline-backend-20260512/backend/tests/test_health_auth_bypass.py
@@ -2,12 +2,15 @@
from __future__ import annotations
+from types import SimpleNamespace
+
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from app.gateway import auth_middleware
from app.gateway.app import app as gateway_app
+from app.gateway.auth.token_source import get_explicit_or_session_token
from app.gateway.auth_middleware import PUBLIC_HEALTH_PATHS, AuthMiddleware
@@ -98,3 +101,28 @@ def test_non_health_route_still_requires_auth(monkeypatch: pytest.MonkeyPatch) -
response = client.get("/private")
assert response.status_code == 401
+
+
+def _auth_request(*, authorization: str = "", cookie: str = "") -> SimpleNamespace:
+ headers = {"authorization": authorization} if authorization else {}
+ cookies = {"access_token": cookie} if cookie else {}
+ return SimpleNamespace(headers=headers, cookies=cookies)
+
+
+def test_explicit_bearer_token_wins_over_shared_cookie() -> None:
+ """Another tab's cookie must not override this tab's explicit token."""
+ request = _auth_request(authorization="Bearer current-tab", cookie="other-tab")
+
+ assert get_explicit_or_session_token(request) == "current-tab"
+
+
+def test_cookie_remains_the_fallback_for_cookie_only_clients() -> None:
+ request = _auth_request(cookie="browser-session")
+
+ assert get_explicit_or_session_token(request) == "browser-session"
+
+
+def test_malformed_authorization_header_does_not_hide_valid_cookie() -> None:
+ request = _auth_request(authorization="Basic abc", cookie="browser-session")
+
+ assert get_explicit_or_session_token(request) == "browser-session"
diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py b/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py
new file mode 100644
index 00000000..d33a1309
--- /dev/null
+++ b/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py
@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+import pytest
+from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+
+from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
+from deerflow.integrations.weknora.local_index.vector_codec import decode_vector, encode_vector
+from deerflow.persistence.base import Base
+from deerflow.persistence.llmwiki.model import LlmWikiKnowledgeBaseRow
+from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
+from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
+
+
+@pytest.mark.asyncio
+async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(tmp_path) -> None:
+ engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'wiki.db'}")
+ async with engine.begin() as connection:
+ await connection.run_sync(
+ lambda sync_connection: Base.metadata.create_all(
+ sync_connection,
+ tables=[LlmWikiKnowledgeBaseRow.__table__, LlmWikiWikiPageRow.__table__, LlmWikiWikiVectorRow.__table__, LlmWikiWikiSyncStateRow.__table__],
+ )
+ )
+ session_factory = async_sessionmaker(engine, expire_on_commit=False)
+ async with session_factory() as session:
+ session.add(
+ LlmWikiKnowledgeBaseRow(
+ id="mapping-1",
+ weknora_id="remote-1",
+ owner_user_id="u1",
+ name="Wiki",
+ description="",
+ kb_type="document",
+ publication_status="private",
+ created_at=datetime.now(UTC),
+ updated_at=datetime.now(UTC),
+ )
+ )
+ await session.commit()
+
+ store = SqlLlmWikiIndexStore(session_factory)
+ assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp")
+ assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp")
+ page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"})
+ blob, dimensions = encode_vector([3.0, 4.0], dimensions=2)
+ saved = await store.replace_page_vectors(
+ page=page,
+ vectors=[{"section_index": 0, "heading": "History", "section_content": "Alice founded Acme.", "content_hash": "section", "embedding_dimensions": dimensions, "vector_blob": blob}],
+ fingerprint="fp",
+ sync_id="sync-1",
+ )
+ snapshot = await store.load_vector_snapshot("mapping-1", "fp")
+ assert saved["slug"] == "people/alice"
+ assert len(snapshot["rows"]) == 1
+ assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
+
+ assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
+ assert (await store.load_vector_snapshot("mapping-1", "fp"))["rows"] == []
+ await engine.dispose()
+
+
+def test_vector_codec_rejects_corrupt_and_non_finite_vectors() -> None:
+ with pytest.raises(ValueError, match="NaN"):
+ encode_vector([1.0, float("nan")], dimensions=2)
+ with pytest.raises(ValueError, match="Corrupt"):
+ decode_vector(b"123", 2)
diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py b/offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py
new file mode 100644
index 00000000..b27cbbf5
--- /dev/null
+++ b/offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py
@@ -0,0 +1,89 @@
+from __future__ import annotations
+
+import pytest
+from pydantic import ValidationError
+
+from deerflow.config.llmwiki_config import LocalWikiIndexConfig
+from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
+from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
+from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
+from deerflow.integrations.weknora.local_index.vector_codec import encode_vector
+from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
+
+
+class FakeEmbedding:
+ dimensions = 2
+ fingerprint = "fp"
+
+ async def embed_query(self, text: str) -> list[float]:
+ return [1.0, 0.0]
+
+
+def test_enabled_index_requires_supported_enabled_embedding() -> None:
+ with pytest.raises(ValidationError, match="embedding must be enabled"):
+ LocalWikiIndexConfig(enabled=True, embedding={"enabled": False})
+ with pytest.raises(ValidationError, match="provider must be openai"):
+ LocalWikiIndexConfig(embedding={"provider": "custom"})
+
+
+@pytest.mark.asyncio
+async def test_exact_search_groups_sections_by_wiki_page_and_filters_drafts() -> None:
+ store = MemoryLlmWikiIndexStore()
+ await store.try_acquire_sync_lease("kb", owner="w", sync_id="s", lease_seconds=60, fingerprint="fp")
+ published = normalize_wiki_page("kb", {"slug": "alice", "title": "Alice", "summary": "Founder", "content": "Alpha", "status": "published"})
+ draft = normalize_wiki_page("kb", {"slug": "secret", "title": "Secret", "content": "Draft", "status": "draft"})
+ a, _ = encode_vector([1.0, 0.0], dimensions=2)
+ b, _ = encode_vector([0.9, 0.1], dimensions=2)
+ await store.replace_page_vectors(
+ page=published,
+ vectors=[
+ {"section_index": 0, "heading": "A", "section_content": "Alpha", "content_hash": "a", "embedding_dimensions": 2, "vector_blob": a},
+ {"section_index": 1, "heading": "B", "section_content": "Beta", "content_hash": "b", "embedding_dimensions": 2, "vector_blob": b},
+ ],
+ fingerprint="fp",
+ sync_id="s",
+ )
+ await store.replace_page_vectors(page=draft, vectors=[{"section_index": 0, "heading": None, "section_content": "Draft", "content_hash": "d", "embedding_dimensions": 2, "vector_blob": a}], fingerprint="fp", sync_id="s")
+ config = LocalWikiIndexConfig(min_similarity=0, top_k_sections=10, top_k_pages=8, max_sections_per_page=2)
+ service = WikiVectorSearchService(config, FakeEmbedding(), WikiVectorCache(store, revision_check_seconds=0))
+ mapping = {"id": "kb", "name": "People"}
+
+ result = await service.search("alice", [mapping])
+ assert [item["wiki_slug"] for item in result["results"]] == ["alice"]
+ assert len(result["results"][0]["matched_sections"]) == 2
+ assert "wiki_slug=alice" in result["results"][0]["url"]
+
+ with_drafts = await service.search("alice", [mapping], include_drafts=True)
+ assert {item["wiki_slug"] for item in with_drafts["results"]} == {"alice", "secret"}
+
+ narrow_service = WikiVectorSearchService(
+ LocalWikiIndexConfig(min_similarity=0, top_k_sections=1, top_k_pages=1),
+ FakeEmbedding(),
+ WikiVectorCache(store, revision_check_seconds=0),
+ )
+ assert [item["wiki_slug"] for item in (await narrow_service.search("alice", [mapping]))["results"]] == ["alice"]
+
+
+@pytest.mark.asyncio
+async def test_corrupt_blob_marks_index_for_rebuild() -> None:
+ store = MemoryLlmWikiIndexStore()
+ await store.try_acquire_sync_lease("kb", owner="w", sync_id="s", lease_seconds=60, fingerprint="fp")
+ page = normalize_wiki_page("kb", {"slug": "broken", "title": "Broken", "content": "Broken", "status": "published"})
+ blob, _ = encode_vector([1.0, 0.0], dimensions=2)
+ saved = await store.replace_page_vectors(
+ page=page,
+ vectors=[{"section_index": 0, "heading": None, "section_content": "Broken", "content_hash": "x", "embedding_dimensions": 2, "vector_blob": blob}],
+ fingerprint="fp",
+ sync_id="s",
+ )
+ store.vectors[saved["id"]][0]["vector_blob"] = b"broken"
+ service = WikiVectorSearchService(
+ LocalWikiIndexConfig(min_similarity=0, top_k_sections=10, top_k_pages=8),
+ FakeEmbedding(),
+ WikiVectorCache(store, revision_check_seconds=0),
+ )
+
+ with pytest.raises(WikiIndexError) as exc_info:
+ await service.search("broken", [{"id": "kb", "name": "Broken"}])
+ assert exc_info.value.code == "WIKI_INDEX_REBUILDING"
+ assert store.states["kb"]["state"] == "rebuild_required"
diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_wiki_retrieval.py b/offline-backend-20260512/backend/tests/test_llmwiki_wiki_retrieval.py
new file mode 100644
index 00000000..9a0fca53
--- /dev/null
+++ b/offline-backend-20260512/backend/tests/test_llmwiki_wiki_retrieval.py
@@ -0,0 +1,154 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from openpyxl import load_workbook
+
+from app.gateway.routers.llmwiki import _build_wiki_excel
+from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
+
+
+class _Store:
+ def __init__(self, states):
+ self.states = states
+
+ async def list_index_status(self, mapping_ids=None):
+ wanted = set(mapping_ids or [])
+ return [state for state in self.states if not wanted or state["knowledge_base_mapping_id"] in wanted]
+
+
+class _WikiClient:
+ def __init__(self):
+ self.list_calls = []
+ self.raw_search_called = False
+
+ async def list_wiki_pages(self, knowledge_base_id, *, page=1, page_size=50, query=""):
+ self.list_calls.append((knowledge_base_id, query))
+ return {"pages": [{"slug": "guides/install", "title": "安装"}], "total": 1, "total_pages": 1}
+
+ async def get_wiki_page(self, knowledge_base_id, slug):
+ return {
+ "id": f"{knowledge_base_id}:{slug}",
+ "slug": slug,
+ "title": "安装",
+ "summary": "Wiki 摘要",
+ "content": "# 安装\n这是处理后的 Wiki 正文。",
+ "status": "published",
+ "category_path": ["指南"],
+ }
+
+ async def search(self, query, knowledge_base_ids):
+ self.raw_search_called = True
+ raise AssertionError("raw knowledge search must never be called")
+
+
+class _VectorSearch:
+ def __init__(self):
+ self.embedding = SimpleNamespace(fingerprint="fp-current")
+ self.calls = []
+
+ async def search(self, query, mappings, **kwargs):
+ self.calls.append([mapping["id"] for mapping in mappings])
+ mapping = mappings[0]
+ return {
+ "results": [
+ {
+ "kind": "wiki_page",
+ "id": "local-page",
+ "wiki_page_id": "local-page",
+ "wiki_slug": "local/page",
+ "knowledge_base_id": mapping["id"],
+ "title": "本地页面",
+ "content": "本地 Wiki",
+ "matched_sections": [],
+ "score": 0.95,
+ }
+ ],
+ "warnings": [],
+ }
+
+
+def _ready_state(mapping_id: str):
+ return {
+ "knowledge_base_mapping_id": mapping_id,
+ "state": "idle",
+ "last_success_at": "2026-08-24T00:00:00Z",
+ "embedding_fingerprint": "fp-current",
+ "local_page_count": 1,
+ "ready_page_count": 1,
+ "failed_page_count": 0,
+ "vector_count": 3,
+ }
+
+
+def test_fully_vectorized_requires_completed_full_consistent_scan() -> None:
+ state = _ready_state("kb")
+ assert is_fully_vectorized(state, "fp-current")
+ assert not is_fully_vectorized({**state, "state": "partial"}, "fp-current")
+ assert not is_fully_vectorized({**state, "ready_page_count": 0}, "fp-current")
+ assert not is_fully_vectorized(state, "different-fingerprint")
+
+
+@pytest.mark.asyncio
+async def test_unvectorized_library_uses_only_wiki_page_search_api() -> None:
+ client = _WikiClient()
+ service = WikiRetrievalService(_Store([]), vector_search=_VectorSearch(), client=client)
+ mapping = {"id": "local-kb", "weknora_id": "remote-kb", "name": "测试库"}
+
+ result = await service.search("安装", [mapping])
+
+ assert result["mode"] == "wiki_api"
+ assert result["fallback_knowledge_base_ids"] == ["local-kb"]
+ assert result["results"][0]["kind"] == "wiki_page"
+ assert result["results"][0]["content"] == "# 安装\n这是处理后的 Wiki 正文。"
+ assert "chunk_id" not in result["results"][0]
+ assert client.list_calls == [("remote-kb", "安装")]
+ assert not client.raw_search_called
+
+
+@pytest.mark.asyncio
+async def test_mixed_search_vectors_complete_library_and_falls_back_incomplete_library() -> None:
+ client = _WikiClient()
+ vector = _VectorSearch()
+ service = WikiRetrievalService(_Store([_ready_state("ready")]), vector_search=vector, client=client)
+ mappings = [
+ {"id": "ready", "weknora_id": "remote-ready", "name": "已完成"},
+ {"id": "pending", "weknora_id": "remote-pending", "name": "未完成"},
+ ]
+
+ result = await service.search("安装", mappings, top_k_pages=8)
+
+ assert result["mode"] == "mixed"
+ assert vector.calls == [["ready"]]
+ assert client.list_calls == [("remote-pending", "安装")]
+ assert {item["knowledge_base_id"] for item in result["results"]} == {"ready", "pending"}
+
+
+def test_excel_export_contains_current_directory_and_every_parent_directory() -> None:
+ long_content = "正文" * 20_000
+ pages = [
+ {"id": "1", "slug": "root", "title": "根专题", "content": "根", "status": "published"},
+ {"id": "2", "slug": "root/child", "title": "子专题", "parent_slug": "root", "content": "子", "status": "published"},
+ {
+ "id": "3",
+ "slug": "root/child/article",
+ "title": "文章",
+ "parent_slug": "root/child",
+ "content": long_content,
+ "status": "published",
+ },
+ ]
+
+ workbook = load_workbook(_build_wiki_excel("知识库", pages), read_only=True)
+ sheet = workbook["Wiki 全量导出"]
+ rows = list(sheet.iter_rows(values_only=True))
+ headings = list(rows[0])
+ article = dict(zip(headings, rows[3], strict=True))
+
+ assert article["目录完整路径"] == "根专题 / 子专题"
+ assert article["所在目录"] == "子专题"
+ assert article["父级目录完整路径"] == "根专题"
+ assert article["第1级目录"] == "根专题"
+ assert article["第2级目录"] == "子专题"
+ assert article["正文"] + article["正文(续1)"] == long_content
diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py b/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py
new file mode 100644
index 00000000..d13ceaaf
--- /dev/null
+++ b/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py
@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+import pytest
+
+from deerflow.config.llmwiki_config import LocalWikiIndexConfig
+from deerflow.integrations.weknora.local_index.sync import WikiSyncService
+from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
+from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore
+
+
+class FakeEmbedding:
+ dimensions = 2
+ fingerprint = "fp"
+
+ def __init__(self) -> None:
+ self.calls = 0
+
+ async def embed_texts(self, texts: list[str]) -> list[list[float]]:
+ self.calls += 1
+ return [[1.0, 0.0] for _ in texts]
+
+
+class FailingEmbedding(FakeEmbedding):
+ async def embed_texts(self, texts: list[str]) -> list[list[float]]:
+ raise RuntimeError("embedding unavailable")
+
+
+class NewFingerprintEmbedding(FakeEmbedding):
+ fingerprint = "fp-new"
+
+
+class FakeWeKnora:
+ def __init__(self) -> None:
+ self.pages = {
+ "one": {
+ "id": "p1",
+ "slug": "one",
+ "title": "One",
+ "summary": "A long enough summary for indexing and retrieval.",
+ "content": "# Intro\nA long enough Wiki paragraph for indexing and semantic retrieval.",
+ "status": "published",
+ "version": 1,
+ },
+ }
+
+ async def get_knowledge_base(self, knowledge_base_id: str):
+ return {"id": knowledge_base_id, "indexing_strategy": {"wiki_enabled": True}}
+
+ async def list_wiki_pages(self, knowledge_base_id: str, *, page: int, page_size: int):
+ return {"pages": [{"slug": item["slug"], "title": item["title"]} for item in self.pages.values()], "total_pages": 1}
+
+ async def get_wiki_page(self, knowledge_base_id: str, slug: str):
+ return dict(self.pages[slug])
+
+
+@pytest.mark.asyncio
+async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None:
+ config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
+ store = MemoryLlmWikiIndexStore()
+ embedding = FakeEmbedding()
+ client = FakeWeKnora()
+ service = WikiSyncService(config, store, embedding, WikiVectorCache(store), client)
+ mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
+
+ first = await service.sync_mapping(mapping)
+ assert first["changed_pages"] == 1
+ assert embedding.calls == 1
+ second = await service.sync_mapping(mapping)
+ assert second["changed_pages"] == 0
+ assert embedding.calls == 1
+
+ client.pages.clear()
+ deleted = await service.sync_mapping(mapping)
+ assert deleted["deleted_pages"] == 1
+ assert next(iter(store.pages.values()))["is_remote_deleted"] is True
+
+
+@pytest.mark.asyncio
+async def test_page_failure_keeps_previous_generation_and_marks_partial() -> None:
+ config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
+ store = MemoryLlmWikiIndexStore()
+ client = FakeWeKnora()
+ mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
+ await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
+ page = next(iter(store.pages.values()))
+ old_hash = page["content_hash"]
+ old_vectors = list(store.vectors[page["id"]])
+ client.pages["one"]["content"] += " Changed."
+
+ result = await WikiSyncService(config, store, FailingEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
+
+ assert result["status"] == "partial"
+ assert store.states["kb"]["state"] == "partial"
+ assert store.states["kb"]["failed_page_count"] == 1
+ assert store.pages[page["id"]]["content_hash"] == old_hash
+ assert store.pages[page["id"]]["index_status"] == "ready"
+ assert store.vectors[page["id"]] == old_vectors
+
+
+@pytest.mark.asyncio
+async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> None:
+ config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0)
+ store = MemoryLlmWikiIndexStore()
+ client = FakeWeKnora()
+ mapping = {"id": "kb", "weknora_id": "remote", "name": "Wiki"}
+ await WikiSyncService(config, store, FakeEmbedding(), WikiVectorCache(store), client).sync_mapping(mapping)
+
+ embedding = NewFingerprintEmbedding()
+ result = await WikiSyncService(config, store, embedding, WikiVectorCache(store), client).sync_mapping(mapping)
+
+ assert result["status"] == "completed"
+ assert result["changed_pages"] == 1
+ assert embedding.calls == 1
+ assert store.states["kb"]["state"] == "idle"
+ assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
+ assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}
diff --git a/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py b/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py
new file mode 100644
index 00000000..5c43a851
--- /dev/null
+++ b/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py
@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+from fastapi import HTTPException
+from starlette.requests import Request
+
+from app.gateway.routers import llmwiki
+
+
+def _request(store: object) -> Request:
+ return Request(
+ {
+ "type": "http",
+ "method": "GET",
+ "path": "/api/public/llmwiki/knowledge-bases/kb-one",
+ "headers": [],
+ "app": SimpleNamespace(state=SimpleNamespace(llmwiki_store=store)),
+ }
+ )
+
+
+class PublicStore:
+ def __init__(self, row: dict[str, object] | None) -> None:
+ self.row = row
+ self.calls: list[tuple[str, str, bool, bool]] = []
+
+ async def get_authorized(
+ self,
+ mapping_id: str,
+ user_id: str,
+ *,
+ write: bool,
+ is_admin: bool,
+ ) -> dict[str, object] | None:
+ self.calls.append((mapping_id, user_id, write, is_admin))
+ return self.row
+
+
+def _published_row() -> dict[str, object]:
+ return {
+ "id": "kb-one",
+ "weknora_id": "remote-one",
+ "owner_user_id": "owner-one",
+ "name": "公开知识库",
+ "description": "公开说明",
+ "kb_type": "document",
+ "publication_status": "published",
+ }
+
+
+@pytest.mark.asyncio
+async def test_public_wiki_reader_uses_anonymous_non_admin_authorization(monkeypatch) -> None:
+ store = PublicStore(_published_row())
+
+ class Client:
+ async def get_knowledge_base(self, knowledge_base_id: str):
+ assert knowledge_base_id == "remote-one"
+ return {"id": knowledge_base_id, "name": "公开知识库", "knowledge_count": 2}
+
+ monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
+
+ payload = await llmwiki.get_public_knowledge_base(_request(store), "kb-one")
+
+ assert store.calls == [("kb-one", "__public_wiki__", False, False)]
+ assert payload["publication_status"] == "published"
+ assert payload["owner_user_id"] is None
+ assert payload["can_write"] is False
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "row",
+ [
+ {**_published_row(), "publication_status": "private"},
+ {
+ **_published_row(),
+ "owner_user_id": "system",
+ "name": "对话沉淀",
+ },
+ None,
+ ],
+)
+async def test_public_wiki_reader_rejects_non_public_resources(row) -> None:
+ with pytest.raises(HTTPException) as exc_info:
+ await llmwiki._public_mapping(_request(PublicStore(row)), "kb-one")
+
+ assert exc_info.value.status_code == 404
+
+
+@pytest.mark.asyncio
+async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None:
+ store = PublicStore(_published_row())
+
+ class Client:
+ async def list_wiki_pages(self, knowledge_base_id: str, **kwargs):
+ assert knowledge_base_id == "remote-one"
+ assert kwargs == {"page": 1, "page_size": 50, "query": "目录"}
+ return {
+ "pages": [
+ {
+ "id": "page-one",
+ "slug": "guide/start",
+ "title": "开始使用",
+ "page_type": "article",
+ "status": "completed",
+ "content": "# 开始使用",
+ }
+ ],
+ "total": 1,
+ "page": 1,
+ "page_size": 50,
+ "total_pages": 1,
+ }
+
+ monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
+
+ payload = await llmwiki.list_public_wiki_pages(
+ _request(store),
+ "kb-one",
+ page=1,
+ page_size=50,
+ query=" 目录 ",
+ )
+
+ assert payload["pages"][0]["slug"] == "guide/start"
+ assert payload["pages"][0]["aliases"] == []
diff --git a/offline-backend-20260512/backend/tests/test_username_login_domain.py b/offline-backend-20260512/backend/tests/test_username_login_domain.py
index 85fa26c5..f2f5cd3f 100644
--- a/offline-backend-20260512/backend/tests/test_username_login_domain.py
+++ b/offline-backend-20260512/backend/tests/test_username_login_domain.py
@@ -9,15 +9,18 @@ Verifies that:
from __future__ import annotations
-import os
from datetime import UTC, datetime
-from uuid import UUID, uuid4
+from uuid import uuid4
import pytest
+from app.gateway.auth.local_provider import LocalAuthProvider
from app.gateway.auth.models import User
from app.gateway.auth.repositories.base import UserRepository
+from app.gateway.routers import auth as auth_module
from app.gateway.routers.auth import (
+ _get_or_create_user_by_username,
+ _lookup_user_by_username,
_normalize_username_local_part,
_username_to_email,
)
@@ -74,6 +77,9 @@ class FakeUserRepository(UserRepository):
async def count_admin_users(self) -> int:
return sum(1 for u in self.rows.values() if u.system_role == "admin")
+ async def list_users(self) -> list[User]:
+ return sorted(self.rows.values(), key=lambda user: user.created_at)
+
async def get_user_by_oauth(self, provider: str, oauth_id: str) -> User | None:
return None
@@ -146,3 +152,38 @@ async def test_local_part_lookup_misses_unknown_username() -> None:
repo = FakeUserRepository()
_seed(repo, "alice@cm.com")
assert await repo.find_user_by_email_local_part("bob") is None
+
+
+@pytest.mark.asyncio
+async def test_username_login_uses_oldest_account_when_canonical_duplicate_exists(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A domain migration must not make one username alternate user ids."""
+ repo = FakeUserRepository()
+ older = _seed(repo, "alice@local.deerflow")
+ older.created_at = datetime(2024, 1, 1, tzinfo=UTC)
+ newer = _seed(repo, "alice@cm.com")
+ newer.created_at = datetime(2026, 1, 1, tzinfo=UTC)
+ monkeypatch.setattr(auth_module, "get_local_provider", lambda: LocalAuthProvider(repo))
+
+ user, created = await _get_or_create_user_by_username("alice")
+
+ assert created is False
+ assert user.id == older.id
+
+
+@pytest.mark.asyncio
+async def test_password_lookup_uses_same_stable_username_account(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ repo = FakeUserRepository()
+ older = _seed(repo, "alice@local.deerflow")
+ older.created_at = datetime(2024, 1, 1, tzinfo=UTC)
+ newer = _seed(repo, "alice@cm.com")
+ newer.created_at = datetime(2026, 1, 1, tzinfo=UTC)
+ monkeypatch.setattr(auth_module, "get_local_provider", lambda: LocalAuthProvider(repo))
+
+ user = await _lookup_user_by_username("alice")
+
+ assert user is not None
+ assert user.id == older.id
diff --git a/offline-backend-20260512/config.yaml b/offline-backend-20260512/config.yaml
index 3360a25a..0a69220d 100644
--- a/offline-backend-20260512/config.yaml
+++ b/offline-backend-20260512/config.yaml
@@ -190,6 +190,15 @@ skills:
path: /app/skills
security_scan:
enabled: false
+ # ES 灵活查询技能路由提示:开启后注入整体 system prompt;修改 prompt 可调整
+ # 触发边界和调用要求。关闭后该提示完全不注入。
+ es_query_routing:
+ enabled: true
+ prompt: |
+ 当用户的问题需要基于 Elasticsearch 查询语句(Query DSL)灵活构造、调整或执行查询时,必须优先使用 `es_query` 技能。
+ 典型场景包括:根据用户条件动态组合 bool/must/filter/should、范围、聚合、排序或分页等 ES 查询;或者用户直接提供、要求修改 ES 语句并据此查询数据。
+ 执行前先读取 `es_query` 技能的 SKILL.md,严格按照技能中的流程和参数调用;不要用通用搜索替代,也不要凭记忆臆测查询结果。
+ 仅当问题确实需要 ES 数据查询时使用;纯概念解释、普通代码说明或与 ES 无关的问题不要调用。
agents_api:
enabled: true
# 登录鉴权扩展:在基础登录之上叠加两种自定义模式。