From 30943159bfdcc5a3e475734d490fab966ebf1431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=89=E9=91=AB=E5=8D=87?= Date: Mon, 24 Aug 2026 20:40:44 +0800 Subject: [PATCH 1/4] feat: complete public wiki and embedded previews --- .../workspace/messages/markdown-content.tsx | 20 +- .../src/core/llmwiki/wiki-page-markdown.ts | 208 +++++++++++++++++- frontend-web/src/core/llmwiki/wiki-tree.ts | 73 +++++- .../src/strategy-components/api/llmwiki.ts | 44 ++++ .../llmwiki/WeKnoraKnowledgeDetailPage.tsx | 1 - .../llmwiki/WeKnoraSourceDrawer.tsx | 60 ++++- .../llmwiki/WeKnoraWikiPageView.tsx | 72 ++++-- frontend-web/vite.config.ts | 19 +- offline-backend-20260512/backend/CLAUDE.md | 42 +++- offline-backend-20260512/backend/README.md | 28 ++- .../backend/app/gateway/routers/llmwiki.py | 132 ++++++++++- offline-backend-20260512/backend/config.yaml | 2 + .../harness/deerflow/config/llmwiki_config.py | 7 +- .../deerflow/integrations/weknora/client.py | 25 +++ .../weknora/local_index/embedding.py | 6 +- .../backend/tests/test_llmwiki_weknora.py | 53 +++++ .../backend/tests/test_llmwiki_wiki_sync.py | 55 ++++- .../tests/test_public_llmwiki_embed.py | 127 +++++++++++ 18 files changed, 926 insertions(+), 48 deletions(-) diff --git a/frontend-web/src/components/workspace/messages/markdown-content.tsx b/frontend-web/src/components/workspace/messages/markdown-content.tsx index e7b6380e..b83053f2 100644 --- a/frontend-web/src/components/workspace/messages/markdown-content.tsx +++ b/frontend-web/src/components/workspace/messages/markdown-content.tsx @@ -1,7 +1,12 @@ "use client"; import { useMemo } from "react"; -import type { AnchorHTMLAttributes, HTMLAttributes, ReactNode } from "react"; +import type { + AnchorHTMLAttributes, + HTMLAttributes, + MouseEvent as ReactMouseEvent, + ReactNode, +} from "react"; import { MessageResponse, @@ -34,6 +39,12 @@ export type MarkdownContentProps = { className?: string; remarkPlugins?: MessageResponseProps["remarkPlugins"]; components?: MessageResponseProps["components"]; + linkClassName?: (href: string, label: string) => string | undefined; + onLinkClick?: ( + event: ReactMouseEvent, + href: string, + label: string, + ) => void; }; /** Renders markdown content. */ @@ -43,6 +54,8 @@ export function MarkdownContent({ className, remarkPlugins = streamdownPlugins.remarkPlugins, components: componentsFromProps, + linkClassName, + onLinkClick, }: MarkdownContentProps) { const anchorResolver = useMemo(() => createMarkdownAnchorResolver(content), [content]); const components = useMemo(() => { @@ -93,6 +106,7 @@ export function MarkdownContent({ ); } const finalHref = resolvedHref ?? props.href; + const label = extractTextFromChildren(props.children); return ( @@ -117,7 +133,7 @@ export function MarkdownContent({ }, ...componentsFromProps, }; - }, [anchorResolver, componentsFromProps]); + }, [anchorResolver, componentsFromProps, linkClassName, onLinkClick]); if (!content) return null; diff --git a/frontend-web/src/core/llmwiki/wiki-page-markdown.ts b/frontend-web/src/core/llmwiki/wiki-page-markdown.ts index d4a24078..4ee93185 100644 --- a/frontend-web/src/core/llmwiki/wiki-page-markdown.ts +++ b/frontend-web/src/core/llmwiki/wiki-page-markdown.ts @@ -5,6 +5,38 @@ export type WikiPageUrlBuilder = ( wikiSlug: string, ) => string; +export type WikiPageLinkTarget = { + slug: string; + title: string; + aliases?: string[]; + page_type?: string; + summary?: string; + category_path?: string[]; +}; + +export type WikiIndexGroup = { + type: string; + total: number; + items: WikiPageLinkTarget[]; +}; + +type WikiIndexCategoryNode = { + label: string; + pages: WikiPageLinkTarget[]; + children: Map; +}; + +const WIKI_INDEX_TYPE_LABELS: Record = { + summary: "摘要", + entity: "实体", + concept: "概念", + overview: "概览", + directory: "目录", + article: "文章", +}; + +const WIKI_INDEX_TYPE_ORDER = ["summary", "entity", "concept"]; + export function stripWikiFrontmatter(value: string) { return value.replace(/^\s*---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, "").trim(); } @@ -38,18 +70,192 @@ export function wikiLinkMarkdown( .replace(/\[\[([^\]]+)\]\]/g, (_match, slug) => link(slug)); } +function normalizeWikiLinkLabel(value: string): string { + return value + .replace(/\\([\\`*_[\]{}()#+\-.!])/g, "$1") + .replace(/[*_`~]/g, "") + .replace(/\s+/g, " ") + .trim() + .toLocaleLowerCase("zh-CN"); +} + +function isLegacyWikiPlaceholderUrl(value: string): boolean { + return /\/platform\/knowledge-bases\/[^/?#)]+/i.test(value); +} + +function wikiIndexTypeRank(pageType: string): number { + const index = WIKI_INDEX_TYPE_ORDER.indexOf(pageType); + return index >= 0 ? index : WIKI_INDEX_TYPE_ORDER.length; +} + +function wikiIndexPageLine( + page: WikiPageLinkTarget, + knowledgeBaseId: string, + buildPageUrl: WikiPageUrlBuilder, +): string { + const label = (page.title || page.slug).replace(/\]/g, "\\]"); + const summary = (page.summary || "").replace(/\s+/g, " ").trim(); + return `[${label}](${buildPageUrl(knowledgeBaseId, page.slug)})${summary ? ` — ${summary}` : ""}`; +} + +function renderWikiIndexCategory( + node: WikiIndexCategoryNode, + knowledgeBaseId: string, + buildPageUrl: WikiPageUrlBuilder, +): string[] { + const lines = node.label ? [`**${node.label}**`] : []; + lines.push(...node.pages.map((page) => wikiIndexPageLine(page, knowledgeBaseId, buildPageUrl))); + const children = Array.from(node.children.values()).sort((left, right) => + left.label.localeCompare(right.label, "zh-Hans-CN"), + ); + for (const child of children) { + lines.push(...renderWikiIndexCategory(child, knowledgeBaseId, buildPageUrl)); + } + return lines; +} + +/** Build the category catalogue that WeKnora renders dynamically for its index page. */ +export function buildWikiIndexCatalogMarkdown( + pages: WikiPageLinkTarget[], + knowledgeBaseId: string, + buildPageUrl: WikiPageUrlBuilder = buildWeKnoraWikiPageUrl, + indexGroups: WikiIndexGroup[] = [], +): string { + if (indexGroups.length > 0) { + const sections: string[] = []; + for (const group of indexGroups) { + if (group.items.length === 0) continue; + const seenCategories = new Set(); + const lines: string[] = []; + for (const page of group.items) { + const categories = (page.category_path || []).map((item) => item.trim()).filter(Boolean); + for (let depth = 0; depth < categories.length; depth += 1) { + const path = categories.slice(0, depth + 1).join("/"); + if (seenCategories.has(path)) continue; + seenCategories.add(path); + lines.push(`${" ".repeat(depth)}**${categories[depth]}**`); + } + lines.push( + `${" ".repeat(categories.length)}${wikiIndexPageLine(page, knowledgeBaseId, buildPageUrl)}`, + ); + } + const label = WIKI_INDEX_TYPE_LABELS[group.type] || group.type; + sections.push(`## ${label} (${group.total})`, lines.join("\n")); + } + return sections.join("\n\n"); + } + + const groups = new Map(); + for (const page of pages) { + const pageType = (page.page_type || "article").trim().toLowerCase(); + const slugName = page.slug.split("/").filter(Boolean).at(-1)?.toLowerCase(); + if (pageType === "index" || ["index", "home", "readme"].includes(slugName || "")) continue; + const group = groups.get(pageType) || []; + group.push(page); + groups.set(pageType, group); + } + + const pageTypes = Array.from(groups.keys()).sort((left, right) => { + const rankDifference = wikiIndexTypeRank(left) - wikiIndexTypeRank(right); + if (rankDifference !== 0) return rankDifference; + return (WIKI_INDEX_TYPE_LABELS[left] || left).localeCompare( + WIKI_INDEX_TYPE_LABELS[right] || right, + "zh-Hans-CN", + ); + }); + + const sections: string[] = []; + for (const pageType of pageTypes) { + const groupPages = groups.get(pageType) || []; + const root: WikiIndexCategoryNode = { label: "", pages: [], children: new Map() }; + for (const page of groupPages) { + let node = root; + for (const category of page.category_path || []) { + const label = category.trim(); + if (!label) continue; + let child = node.children.get(label); + if (!child) { + child = { label, pages: [], children: new Map() }; + node.children.set(label, child); + } + node = child; + } + node.pages.push(page); + } + const label = WIKI_INDEX_TYPE_LABELS[pageType] || pageType; + sections.push( + `## ${label} (${groupPages.length})`, + ...renderWikiIndexCategory(root, knowledgeBaseId, buildPageUrl), + ); + } + return sections.join("\n\n"); +} + +/** + * WeKnora-generated index pages may contain Markdown links whose href only + * points at the knowledge-base Wiki shell. Resolve those placeholders by the + * visible link label so the native/public reader can open the exact page. + */ +export function resolveWikiIndexPageLinks( + markdown: string, + knowledgeBaseId: string, + pages: WikiPageLinkTarget[], + buildPageUrl: WikiPageUrlBuilder = buildWeKnoraWikiPageUrl, +): string { + const pagesByLabel = new Map(); + for (const page of pages) { + const labels = [page.title, page.slug, page.slug.split("/").filter(Boolean).at(-1), ...(page.aliases || [])]; + for (const label of labels) { + const normalized = normalizeWikiLinkLabel(label || ""); + if (normalized && !pagesByLabel.has(normalized)) pagesByLabel.set(normalized, page); + } + } + + return markdown.replace( + /(? { + if (!isLegacyWikiPlaceholderUrl(rawUrl)) return original; + const normalizedLabel = normalizeWikiLinkLabel(rawLabel); + const withoutSummarySuffix = normalizedLabel.replace(/\s+-\s+summary$/i, ""); + const target = pagesByLabel.get(normalizedLabel) || pagesByLabel.get(withoutSummarySuffix); + if (!target) return original; + return `[${rawLabel}](${buildPageUrl(knowledgeBaseId, target.slug)})`; + }, + ); +} + export function renderableWikiMarkdown( title: string, content: string, knowledgeBaseId: string, buildPageUrl?: WikiPageUrlBuilder, + pages: WikiPageLinkTarget[] = [], + indexGroups: WikiIndexGroup[] = [], ) { const lines = content.trim().split("\n"); if (lines[0]?.replace(/^#+\s*/, "").trim() === title.trim()) { lines[0] = `# ${title}`; } + const baseMarkdown = exportableWikiMarkdown(title, "", lines.join("\n")); + const isIndex = pages.some((page) => { + const slugName = page.slug.split("/").filter(Boolean).at(-1)?.toLowerCase(); + return page.title === title && ( + page.page_type?.toLowerCase() === "index" || + ["index", "home", "readme"].includes(slugName || "") + ); + }); + const existingMarkdownLinkCount = (baseMarkdown.match(/(?; }; +export type BuildWikiTreeOptions = { + preferredRootPageSlug?: string; +}; + +export function isWikiIndexPage(page: LlmWikiWikiPage): boolean { + const pageType = (page.page_type || "").trim().toLowerCase(); + const title = (page.title || "").trim().toLowerCase(); + const slug = (page.slug || "").trim().replace(/^\/+|\/+$/g, ""); + const slugName = slug.split("/").pop()?.toLowerCase() || ""; + return ( + pageType === "index" || + title === "索引" || + title === "index" || + ["index", "home", "readme"].includes(slugName) + ); +} + +export function selectDefaultWikiPage( + pages: LlmWikiWikiPage[], +): LlmWikiWikiPage | undefined { + return ( + pages.find(isWikiIndexPage) || + pages.find((page) => page.page_type?.toLowerCase() === "overview") || + pages.find((page) => !page.parent_slug && (page.depth ?? 0) === 0) || + pages[0] + ); +} + function pageDirectoryParts(page: LlmWikiWikiPage): string[] { const wikiPath = (page.wiki_path || "").split("/").map((item) => item.trim()).filter(Boolean); if (wikiPath.length > 2) return wikiPath.slice(1, -1); @@ -62,20 +90,49 @@ function attachNode(parent: WikiTreeNode | null, roots: WikiTreeNode[], node: Wi if (!siblings.some((item) => item.key === node.key)) siblings.push(node); } -function sortWikiNodes(nodes: WikiTreeNode[]) { +function sortWikiNodes( + nodes: WikiTreeNode[], + preferredRootPageSlug?: string, + isRoot = false, +) { nodes.sort((left, right) => { + if (isRoot && preferredRootPageSlug) { + const leftPreferred = left.page?.slug === preferredRootPageSlug; + const rightPreferred = right.page?.slug === preferredRootPageSlug; + if (leftPreferred !== rightPreferred) return leftPreferred ? -1 : 1; + } if (!!left.children.length !== !!right.children.length) return left.children.length ? -1 : 1; return left.label.localeCompare(right.label, "zh-Hans-CN"); }); nodes.forEach((node) => sortWikiNodes(node.children)); } +function detachPageNode(nodes: WikiTreeNode[], slug: string): WikiTreeNode | null { + const index = nodes.findIndex((node) => node.page?.slug === slug); + if (index >= 0) return nodes.splice(index, 1)[0] || null; + for (const node of nodes) { + const detached = detachPageNode(node.children, slug); + if (detached) return detached; + } + return null; +} + +function pruneEmptyDirectories(nodes: WikiTreeNode[]): WikiTreeNode[] { + return nodes.filter((node) => { + node.children = pruneEmptyDirectories(node.children); + return node.page !== null || node.children.length > 0; + }); +} + function countPages(node: WikiTreeNode): number { node.count = (node.page ? 1 : 0) + node.children.reduce((sum, child) => sum + countPages(child), 0); return node.count; } -export function buildWikiTree(pages: LlmWikiWikiPage[]): WikiTree { +export function buildWikiTree( + pages: LlmWikiWikiPage[], + options: BuildWikiTreeOptions = {}, +): WikiTree { const roots: WikiTreeNode[] = []; const pageNodes = new Map(); const directoryNodes = new Map(); @@ -140,7 +197,17 @@ export function buildWikiTree(pages: LlmWikiWikiPage[]): WikiTree { parentKeysBySlug.set(page.slug, ancestorKeys); } + const preferredSlug = options.preferredRootPageSlug?.trim(); + if (preferredSlug) { + const preferredNode = detachPageNode(roots, preferredSlug); + if (preferredNode) { + roots.splice(0, roots.length, ...pruneEmptyDirectories(roots)); + roots.unshift(preferredNode); + parentKeysBySlug.set(preferredSlug, []); + } + } + roots.forEach(countPages); - sortWikiNodes(roots); + sortWikiNodes(roots, preferredSlug, true); return { roots, parentKeysBySlug }; } diff --git a/frontend-web/src/strategy-components/api/llmwiki.ts b/frontend-web/src/strategy-components/api/llmwiki.ts index 481b2967..d7df1d9f 100644 --- a/frontend-web/src/strategy-components/api/llmwiki.ts +++ b/frontend-web/src/strategy-components/api/llmwiki.ts @@ -144,6 +144,27 @@ export interface LlmWikiWikiPageList { total_pages: number; } +export interface LlmWikiWikiIndexItem { + slug: string; + title: string; + summary: string; + category_path: string[]; + wiki_path: string; + depth: number; +} + +export interface LlmWikiWikiIndexGroup { + type: string; + total: number; + items: LlmWikiWikiIndexItem[]; +} + +export interface LlmWikiWikiIndex { + intro: string; + version: number; + groups: LlmWikiWikiIndexGroup[]; +} + export interface LlmWikiWikiIndexStatus { knowledge_base_id: string; state: "not_synced" | "idle" | "running" | "partial" | "failed" | "rebuilding" | "rebuild_required"; @@ -710,6 +731,17 @@ export async function getPublicLlmWikiWikiPage( ); } +export async function getPublicLlmWikiWikiIndex( + knowledgeBaseId: string, +): Promise { + return readJson( + await apiFetch( + `${PUBLIC_API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/wiki/index`, + ), + "读取公开 Wiki 索引", + ); +} + export async function createLlmWikiConversationDeposit(input: { thread_id: string; question: string; @@ -939,6 +971,18 @@ export function usePublicLlmWikiWikiPage( }); } +export function usePublicLlmWikiWikiIndex( + knowledgeBaseId: string | null, + enabled = true, +) { + return useQuery({ + queryKey: ["public-llmwiki", "wiki-index", knowledgeBaseId], + queryFn: () => getPublicLlmWikiWikiIndex(knowledgeBaseId!), + enabled: enabled && !!knowledgeBaseId, + staleTime: 60_000, + }); +} + export function useLlmWikiWikiIndexStatus( knowledgeBaseId: string | null, enabled = true, diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeDetailPage.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeDetailPage.tsx index 8def6a95..136f79c9 100644 --- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeDetailPage.tsx +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeDetailPage.tsx @@ -273,7 +273,6 @@ export function WeKnoraKnowledgeDetailPage() { title={`${details.data.name || "知识库"} 详情`} src={frameSrc} className="h-full min-h-0 w-full flex-1 border-0 bg-background" - sandbox="allow-same-origin allow-scripts allow-forms allow-downloads allow-popups allow-popups-to-escape-sandbox" referrerPolicy="no-referrer" onLoad={() => postThemeToFrame(frameRef.current)} /> diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx index 61a63d6e..272457d2 100644 --- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx @@ -21,12 +21,15 @@ import { } from "@/components/ui/sheet"; import { apiFetch } from "@/core/api/fetch-client"; import { getBackendBaseURL } from "@/core/config"; -import type { WeKnoraSourceTarget } from "@/core/llmwiki/source-links"; +import { + buildWeKnoraWikiPagePath, + parseWeKnoraPreviewUrl, + type WeKnoraSourceTarget, +} from "@/core/llmwiki/source-links"; import { SpreadsheetPreview } from "@/components/workspace/artifacts/spreadsheet-preview"; import { MarkdownContent } from "@/components/workspace/messages/markdown-content"; import { cn } from "@/lib/utils"; import { reasoningPlugins } from "@/core/streamdown"; -import { buildWeKnoraWikiPagePath } from "@/core/llmwiki/source-links"; import { exportableWikiMarkdown, renderableWikiMarkdown, @@ -81,6 +84,14 @@ type CachedSourceDetail = { const sourceDetailCache = new Map(); +function buildDrawerWikiPageUrl(knowledgeBaseId: string, wikiSlug: string) { + const params = new URLSearchParams({ + knowledge_base_id: knowledgeBaseId, + wiki_slug: wikiSlug, + }); + return `/weknora-source-preview?${params.toString()}`; +} + async function getAvailableWikiPage( knowledgeBaseId: string, wikiSlug: string, @@ -539,6 +550,7 @@ export function WeKnoraSourceDrawer({ const [downloading, setDownloading] = useState(false); const [wikiDownload, setWikiDownload] = useState<"md" | "word" | null>(null); const [resolvedDocumentId, setResolvedDocumentId] = useState(""); + const wikiNavigationRequestRef = useRef(0); const targetKey = useMemo(() => { if (!target) return ""; @@ -690,7 +702,12 @@ export function WeKnoraSourceDrawer({ const summary = cleanText(wikiPage?.summary || document?.description); const wikiContent = stripWikiFrontmatter(cleanText(wikiPage?.content)); const wikiDisplayContent = wikiPage && target - ? renderableWikiMarkdown(resolvedTitle, wikiContent, target.knowledgeBaseId) + ? renderableWikiMarkdown( + resolvedTitle, + wikiContent, + target.knowledgeBaseId, + buildDrawerWikiPageUrl, + ) : ""; const wikiMarkdown = wikiPage ? wikiLinkMarkdown( @@ -703,6 +720,32 @@ export function WeKnoraSourceDrawer({ ? buildWeKnoraWikiPagePath(target.knowledgeBaseId, resolvedWikiSlug) : ""; + const openWikiPageInDrawer = async (linkedTarget: WeKnoraSourceTarget) => { + if (!linkedTarget.wikiSlug) return; + const requestId = ++wikiNavigationRequestRef.current; + setLoading(true); + setError(null); + try { + const page = await getAvailableWikiPage( + linkedTarget.knowledgeBaseId, + linkedTarget.wikiSlug, + ); + if (requestId === wikiNavigationRequestRef.current) { + setWikiPage(page); + setDocument(null); + setResolvedDocumentId(""); + setPreview({ kind: "none" }); + setPreviewError(""); + } + } catch (error) { + if (requestId === wikiNavigationRequestRef.current) { + setError(error instanceof Error ? error.message : "无法打开 Wiki 页面"); + } + } finally { + if (requestId === wikiNavigationRequestRef.current) setLoading(false); + } + }; + const downloadOriginal = async () => { if (!target || !resolvedDocumentId) return; setDownloading(true); @@ -925,6 +968,17 @@ export function WeKnoraSourceDrawer({ isLoading={false} remarkPlugins={reasoningPlugins.remarkPlugins} rehypePlugins={reasoningPlugins.rehypePlugins} + linkClassName={(href) => + parseWeKnoraPreviewUrl(href)?.wikiSlug + ? "wiki-content-link font-medium text-[#be2323] decoration-[#be2323]/55 decoration-dashed hover:decoration-solid" + : undefined + } + onLinkClick={(event, href) => { + const linkedTarget = parseWeKnoraPreviewUrl(href); + if (!linkedTarget?.wikiSlug) return; + event.preventDefault(); + void openWikiPageInDrawer(linkedTarget); + }} /> diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx index 489ff7d6..1a6bd740 100644 --- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx @@ -25,6 +25,8 @@ import { import { buildWikiTree, filterWikiPagesWithAncestors, + isWikiIndexPage, + selectDefaultWikiPage, type WikiTreeNode, } from "@/core/llmwiki/wiki-tree"; import { usePublicBackground } from "@/core/embed/background"; @@ -43,6 +45,7 @@ import { useLlmWikiWikiPage, useLlmWikiWikiPages, usePublicLlmWikiKnowledgeBase, + usePublicLlmWikiWikiIndex, usePublicLlmWikiWikiPage, usePublicLlmWikiWikiPages, } from "@/strategy-components/api/llmwiki"; @@ -92,13 +95,13 @@ function downloadBlob(blob: Blob, fileName: string) { window.setTimeout(() => URL.revokeObjectURL(url), 30_000); } -function selectLandingPage(pages: LlmWikiWikiPage[]): LlmWikiWikiPage | undefined { - return ( - pages.find((page) => page.page_type?.toLowerCase() === "overview") || - pages.find((page) => /^(?:index|home|readme)$/i.test(page.slug)) || - pages.find((page) => !page.parent_slug && (page.depth ?? 0) === 0) || - pages[0] - ); +function publicWikiTargetFromHref(href: string, knowledgeBaseId: string) { + const routePrefix = `/#${buildPublicKnowledgeWikiPath(knowledgeBaseId)}`; + if (!href.startsWith(`${routePrefix}?`)) return null; + const route = href.slice(2); + const queryIndex = route.indexOf("?"); + const slug = new URLSearchParams(route.slice(queryIndex + 1)).get("wikiSlug")?.trim(); + return slug ? { route, slug } : null; } export function WeKnoraWikiPageView({ @@ -137,7 +140,11 @@ export function WeKnoraWikiPageView({ ); const pages = publicEmbed ? publicPages : privatePages; const pagesList = useMemo(() => pages.data?.pages || [], [pages.data?.pages]); - const landingPage = useMemo(() => selectLandingPage(pagesList), [pagesList]); + const indexPage = useMemo( + () => (publicEmbed ? pagesList.find(isWikiIndexPage) : undefined), + [pagesList, publicEmbed], + ); + const landingPage = useMemo(() => selectDefaultWikiPage(pagesList), [pagesList]); const wikiSlug = requestedWikiSlug || (publicEmbed ? landingPage?.slug || "" : ""); const privatePage = useLlmWikiWikiPage( noteId || null, @@ -150,6 +157,10 @@ export function WeKnoraWikiPageView({ !!noteId && !!wikiSlug && publicEmbed, ); const page = publicEmbed ? publicPage : privatePage; + const publicWikiIndex = usePublicLlmWikiWikiIndex( + noteId || null, + !!noteId && publicEmbed && !!indexPage && wikiSlug === indexPage.slug, + ); const [downloading, setDownloading] = useState<"md" | "word" | null>(null); const [directoryQuery, setDirectoryQuery] = useState(""); const [collapsedDirectories, setCollapsedDirectories] = useState>(() => new Set()); @@ -159,7 +170,12 @@ export function WeKnoraWikiPageView({ () => filterWikiPagesWithAncestors(pagesList, directoryQuery), [directoryQuery, pagesList], ); - const wikiTree = useMemo(() => buildWikiTree(filteredPages), [filteredPages]); + const wikiTree = useMemo( + () => buildWikiTree(filteredPages, { + preferredRootPageSlug: publicEmbed ? indexPage?.slug : undefined, + }), + [filteredPages, indexPage?.slug, publicEmbed], + ); useEffect(() => { if (!publicEmbed || requestedWikiSlug || !wikiSlug) return; @@ -182,8 +198,16 @@ export function WeKnoraWikiPageView({ }, [wikiSlug, wikiTree.parentKeysBySlug]); const title = pageData?.title || wikiSlug || "Wiki 页面"; - const summary = cleanText(pageData?.summary); - const content = stripWikiFrontmatter(cleanText(pageData?.content)); + const displayTitle = publicEmbed && pageData && isWikiIndexPage(pageData) + ? "索引" + : title; + const currentPageIsIndex = !!pageData && isWikiIndexPage(pageData); + const summary = publicEmbed && currentPageIsIndex ? "" : cleanText(pageData?.summary); + const content = stripWikiFrontmatter(cleanText( + publicEmbed && currentPageIsIndex && publicWikiIndex.data?.intro + ? publicWikiIndex.data.intro + : pageData?.content, + )); const displayMarkdown = pageData ? renderableWikiMarkdown( title, @@ -196,6 +220,8 @@ export function WeKnoraWikiPageView({ return `/#${buildPublicKnowledgeWikiPath(knowledgeBaseId)}?${next.toString()}`; } : undefined, + publicEmbed ? pagesList : [], + publicEmbed ? publicWikiIndex.data?.groups || [] : [], ) : ""; const exportMarkdown = pageData @@ -239,7 +265,11 @@ export function WeKnoraWikiPageView({ const hasChildren = node.children.length > 0; const expanded = directoryQuery.trim() ? true : !collapsedDirectories.has(node.key); const active = node.page?.slug === wikiSlug; - const label = node.page ? node.label : directoryLabel(node.label); + const label = node.page + ? publicEmbed && isWikiIndexPage(node.page) + ? "索引" + : node.label + : directoryLabel(node.label); return (
{knowledgeBase.data?.name || "知识库 Wiki"} -

{title}

+

{displayTitle}

公开 Wiki @@ -467,6 +497,22 @@ export function WeKnoraWikiPageView({ isLoading={false} remarkPlugins={reasoningPlugins.remarkPlugins} rehypePlugins={reasoningPlugins.rehypePlugins} + linkClassName={(href) => + publicEmbed && publicWikiTargetFromHref(href, noteId) + ? "wiki-content-link border-b border-dashed !border-[var(--primary)] font-medium !text-primary !no-underline hover:border-solid focus-visible:rounded-sm focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary" + : undefined + } + onLinkClick={(event, href) => { + if (!publicEmbed) return; + const target = publicWikiTargetFromHref(href, noteId); + if (!target) return; + event.preventDefault(); + navigate(target.route, { + state: { + wikiPage: pagesList.find((item) => item.slug === target.slug), + }, + }); + }} /> diff --git a/frontend-web/vite.config.ts b/frontend-web/vite.config.ts index e30386d4..0094b5be 100644 --- a/frontend-web/vite.config.ts +++ b/frontend-web/vite.config.ts @@ -21,11 +21,20 @@ export default defineConfig({ }, }, }, - resolve: { - alias: [ - { - find: "@/env", - replacement: path.resolve(rootDir, "src/env.ts"), + resolve: { + alias: [ + { + // react-syntax-highlighter@16.1.1 exposes an async `sqlMore` + // loader, but its own highlight.js@10.x dependency has no + // `sql_more` module. Reuse the bundled SQL grammar so Rollup can + // resolve every generated language loader without leaving a broken + // browser-time external import. + find: /^highlight\.js\/lib\/languages\/sql_more$/, + replacement: "highlight.js/lib/languages/sql", + }, + { + find: "@/env", + replacement: path.resolve(rootDir, "src/env.ts"), }, { find: "next/navigation", diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index 0beb183a..6d44c4ea 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -1015,7 +1015,7 @@ Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` me `llmwiki.weknora.api_base_url` is the provider switch: non-empty enables WeKnora, empty preserves the legacy LLMWiki UI. WeKnora is independently deployed and owns PostgreSQL/Redis/object storage/vector storage/parsers/models; DeerFlow config contains only `api_base_url` and optional `web_base_url`. DeerFlow deliberately sends no WeKnora credential, so the WeKnora endpoint must be restricted to a trusted network and allow DeerFlow traffic itself or through its reverse proxy. There is intentionally no `default_embedding_model_id`: `WeKnoraClient` queries `/api/v1/models` and uses WeKnora's active Embedding model when creating a KB; Wiki creation additionally resolves an active KnowledgeQA model inside WeKnora for synthesis. The native BFF is `app/gateway/routers/llmwiki.py`; ownership/publication mappings live in `deerflow.persistence.llmwiki`; the browser only sees DeerFlow mapping IDs. Knowledge-base owners publish immediately (`private` -> `published`) without a review queue, and can unpublish ordinary bases. The conversation-deposit KB is system-owned, always published, shared by all DeerFlow users, excluded from My Knowledge, and immutable through user/admin detail operations; recognition requires both the reserved name and `system` owner. Existing same-name remote mappings are adopted in place as the system deposit, preserving mapping ids and document links. Its user-facing description, generated Markdown source and tags use `cmzs` branding; list/detail reads synchronize stale remote descriptions, and search results, source previews and embedded-detail JSON recursively replace legacy brand text in historical conversation-deposit payloads. Deposit requests must reference a thread accessible to the current account. Personal knowledge-base list/search views are scoped to the current DeerFlow account even for admins; frontend LLMWiki caches include the live account id, and admin write power does not make other users' private KBs appear as owned. Only `system_role === "admin"` sees the WeKnora connection-config tab. Knowledge cards navigate to a dedicated WeKnora-style detail route. Wiki-enabled bases render and default to `Wiki -> documents -> graph`; standard bases omit Wiki and render `documents -> graph`. The detail page supports sequential multi-file upload with an aggregate result for writable ordinary bases, hides it for public read-only/system-deposit bases, and uses a generic loading label while the embed session starts. It also supports URL and published manual-Markdown imports, browses real Wiki pages/content, proxies authenticated original-file previews, pages through `/chunks/{knowledge_id}`, and renders the Wiki page graph from `/knowledgebase/{kb_id}/wiki/graph`; document details use a right sheet rather than the removed KB modal. The credential-free proxy uses one WeKnora service identity, so per-user separation is enforced by DeerFlow mappings while all remote KBs physically share that identity's WeKnora workspace. WeKnora iframe detail pages first call `/api/llmwiki/knowledge-bases/{id}/weknora/session` through the normal DeerFlow API auth path, then load `/platform/knowledge-bases/{weknora_id}` with a signed `deerflow_weknora_embed` cookie for `/platform`, `/assets`, `/locales`, root assets, and API calls. Injected fetch/XHR rewriting sends all iframe `/api/v1/*`, including colliding `/api/v1/auth/*`, through `/api/llmwiki/weknora-embed/api/v1/*`; Gateway auth/CSRF bypass only applies to those signed iframe proxy requests, the proxy still revalidates the mapped KB, and cookie security honors `X-Forwarded-Proto` for HTTPS reverse proxies. `llmwiki_search` is conditionally registered in `tools.py`, is read-only, and rechecks the current DeerFlow user, agent `llmwiki_knowledge_base_ids`, and thread-context selection on every call. Source previews prefer the cleaned Wiki/LLMWiki page text when a retrieved chunk maps to a Wiki page, with raw split chunks as the fallback. Agent bindings use an inline WeKnora-style searchable/grouped multi-select in the standard agent create/detail UI; new chats use the same interaction pattern as a compact composer dropdown, and selections are written to thread metadata/context and restored on reopen. The DeerFlow frontend is native React and the WeKnora frontend is not modified or embedded. Full deployment and behavior notes: `docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`. Tests: `tests/test_llmwiki_weknora.py`. -The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. +The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. Do not add an HTML `sandbox` attribute to this trusted, permission-filtered detail iframe: a sandboxed ancestor causes Chromium's built-in Blob PDF viewer to render the browser-blocked page. The Gateway follows upstream redirects only for `/knowledge/{id}/preview` and `/knowledge-bases/{id}/files`, keeping PDF bytes and protected Wiki images inside the signed proxy instead of sending the browser to WeKnora object storage. **Embedded native knowledge-base switching:** The original WeKnora breadcrumb picker remains available inside the detail iframe. A signed 15-minute embed cookie carries only the current DeerFlow user's visible mappings (their own plus published bases); the proxied `/api/v1/knowledge-bases` response is filtered to that allowlist. Choosing another entry performs a full iframe navigation, rechecks the DeerFlow mapping and read/write permission, then reissues the cookie for the selected base. Never expose the upstream service-admin list directly or permit an arbitrary remote knowledge-base ID. @@ -1451,6 +1451,11 @@ embedding succeeds. Never delete the old generation before a remote embedding call and never fall back to WeKnora raw chunks while the local feature flag is enabled. NumPy matrices are per-mapping LRU snapshots invalidated by `index_revision`; large decode/matrix work runs through `asyncio.to_thread`. +`embedding.dimensions` is a local response-size invariant and fingerprint +input, not an outbound API field: fixed-dimension BGE/OpenAI-compatible +endpoints can reject a `dimensions` request parameter with HTTP 400. When the +value is omitted, the embedding client probes and freezes the first response's +dimension before sync lease/rebuild decisions. During a fingerprint rebuild the mapping uses the Wiki-page API until every page is successfully rebuilt. `WikiRetrievalService` partitions mixed requests per mapping, so ready libraries stay local while incomplete libraries use that @@ -1480,14 +1485,33 @@ WeKnora iframe. Third-party pages embed published Wiki content through the chrome-free route `/#/embed/knowledge/{mapping_id}` (optional exact-page `wikiSlug`). It defaults -to an overview/root Wiki page, retains directory navigation, and supports the -shared public-embed `theme` / `bg` / `text` appearance inputs. All anonymous -reader calls use `/api/public/llmwiki/knowledge-bases/{id}` and its Wiki-page -children. Those endpoints must resolve the mapping with a synthetic anonymous -user and `is_admin=False`, explicitly require `publication_status=published`, -and hard-deny the system conversation-deposit mapping. Never point this page at -the authenticated `/api/llmwiki` reads or weaken the published-only gate merely -because the mapping id is hard to guess. +to a recognized Wiki index page (title `索引`/`index`, page type `index`, or an +`index`/`home`/`readme` slug), promotes that page to the first public-directory +entry, then falls back to an overview/root page when no index exists. It retains +directory navigation and supports the shared public-embed `theme` / `bg` / +`text` appearance inputs. WeKnora dynamically renders index page-type counts, +nested category headings, page summaries, and `data-slug` links even though the +stored index Markdown only contains its introduction; the native public reader +reads the same ordered groups through the published-only +`/api/public/llmwiki/knowledge-bases/{id}/wiki/index` proxy and renders them with +`buildWikiIndexCatalogMarkdown`. The Gateway paginates every WeKnora index type +(`summary`, `entity`, `concept`, `synthesis`, `comparison`) to completion while +preserving upstream group/item order. Index Markdown +may also contain WeKnora shell-only placeholder links; +`resolveWikiIndexPageLinks` matches their visible labels to +the loaded page title, slug, or alias and rewrites them to exact embedded +`wikiSlug` routes. Preserve WeKnora's `wiki-content-link` interaction: primary +brand color, medium weight, dashed underline becoming solid on hover, focus +outline, and in-reader SPA navigation rather than a page reload. `GET +/api/public/llmwiki/knowledge-bases` is the +anonymous discovery endpoint and returns only each published ordinary mapping's +`id` and `name` (plus the collection total). All anonymous reader calls use that +public namespace and the `/knowledge-bases/{id}` Wiki-page children. Those +endpoints must resolve with a synthetic anonymous user and `is_admin=False`, +explicitly require `publication_status=published`, and hard-deny the system +conversation-deposit mapping. Never point this page at the authenticated +`/api/llmwiki` reads or weaken the published-only gate merely because the +mapping id is hard to guess. Owners manually force a full sync/vectorization through `POST /api/llmwiki/knowledge-bases/{id}/vectorize`; status is available through `GET diff --git a/offline-backend-20260512/backend/README.md b/offline-backend-20260512/backend/README.md index fe2f9ee4..fa14f92a 100644 --- a/offline-backend-20260512/backend/README.md +++ b/offline-backend-20260512/backend/README.md @@ -82,7 +82,7 @@ Per-thread isolated execution with virtual path translation: - **Skills loading**: Recursively discovers nested `SKILL.md` files under `skills/{public,custom}` and preserves nested container paths - **Conditional WeKnora LLMWiki**: a non-empty `llmwiki.weknora.api_base_url` enables the DeerFlow-native WeKnora BFF and UI; an empty address preserves the existing LLMWiki page. WeKnora remains independently deployed and owns its database, Redis, storage, parsers and models. DeerFlow keeps only the API/admin addresses, user/publication mappings, agent knowledge-base bindings and per-conversation selections; it stores and sends no WeKnora credential, so the endpoint must be restricted to a trusted network and allow DeerFlow traffic. Owners publish directly to the public LLMWiki without an approval queue for ordinary bases; the system-owned conversation-deposit base is always published, shared by all users, excluded from My Knowledge, immutable through user/admin detail operations, and recognized by both its reserved name and `system` owner. An existing same-name remote mapping is adopted in place as the system deposit so its mapping id and document links stay valid. Its user-facing knowledge-base description and generated document metadata use the `cmzs` brand; list/detail reads synchronize stale remote descriptions, while conversation-deposit search results, source previews and embedded-detail JSON rewrite legacy brand text to `cmzs`. Deposit requests must reference a thread accessible to the current account. Personal knowledge-base lists are scoped to the current DeerFlow account even for admins, while frontend LLMWiki query caches also include the active account id so switching accounts cannot reuse another account's list/detail data. Admins remain the only role that sees the WeKnora connection-config tab. Knowledge cards open a dedicated WeKnora-style detail route: Wiki-enabled bases default to Wiki, then documents and graph; standard bases hide Wiki and default to documents. Writable ordinary knowledge-base details expose a native multi-file upload action that submits files sequentially and reports aggregate success/failure; read-only public bases and the system deposit base do not. It supports file/URL/manual-Markdown imports, Wiki page browsing, document preview, real parsed chunks and Wiki graph nodes/edges. New chats use a compact WeKnora-style searchable/grouped multi-select inside the composer, while agent create/edit pages bind spaces through an inline panel. A single proxy service identity means DeerFlow users are isolated logically while their WeKnora entities share that identity's workspace. The read-only `llmwiki_search` tool reauthorizes the current DeerFlow user on every call and renders credential-free source previews; source previews prefer cleaned Wiki/LLMWiki text when a retrieved chunk maps to a Wiki page, with raw chunks as the fallback. The detail page first creates a signed iframe session through the normal DeerFlow API auth path, then points the iframe directly at the proxied WeKnora `/platform/...` route using the signed `deerflow_weknora_embed` cookie; global auth/CSRF bypass is limited to those signed iframe proxy requests, and HTTPS cookie attributes honor `X-Forwarded-Proto` behind reverse proxies. The iframe proxy keeps WeKnora's root-relative `/platform`, `/assets`, `/locales`, and `/api/v1/*` paths, but is registered last through `llmwiki.proxy_router` so it cannot intercept first-party Gateway APIs such as `POST /api/v1/auth/login/username`. See [docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md](docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md). - **Enterprise Research workbench (temporarily disabled)**: the independent `/api/enterprise-research` router, its report executor, and its ORM table registration are intentionally inactive. The implementation remains in the repository, but startup does not create or alter `enterprise_research_tasks` or `enterprise_research_report_jobs`; this keeps the active Deep Research workflow independent of the paused workbench and avoids MySQL schema errors. -- **Embedded WeKnora detail chrome**: the proxy hides WeKnora's application sidebar (`.main > .aside_box`) and expands the detail outlet to the full iframe width, while preserving the Wiki page's own index/navigation panel. +- **Embedded WeKnora detail chrome**: the proxy hides WeKnora's application sidebar (`.main > .aside_box`) and expands the detail outlet to the full iframe width, while preserving the Wiki page's own index/navigation panel. The host does not sandbox this trusted, permission-filtered detail frame because browser PDF viewers are disabled inside sandboxed ancestor frames. Preview and knowledge-base file redirects are followed by the Gateway so PDF bytes and protected Wiki images remain inside the authorized proxy instead of leaking to an unreachable object-storage URL. - **Embedded WeKnora knowledge-base switching**: the original breadcrumb picker works inside the iframe, but lists only the signed-in user's own and published knowledge bases. Every switch is reauthorized through DeerFlow and receives a fresh short-lived iframe session, so it never exposes the upstream service-admin knowledge-base list. - **Prefixed WeKnora deployment**: the frontend may be published under a separate prefix such as `/magentweb`, while embedded WeKnora detail traffic is served by the Gateway under `/deerflow`. The Gateway rewrites HTML root assets, WeKnora API URLs, redirects, and Vite dynamic dependency maps so page assets stay under `/deerflow`; the prefixed proxy routes must be registered before the unprefixed fallback. The signed embed cookie covers `/deerflow`, and auth/CSRF recognition includes all supported static directories. - **File-write safety**: `str_replace` serializes read-modify-write per `(sandbox.id, path)` so isolated sandboxes keep concurrency even when virtual paths match @@ -780,6 +780,11 @@ matched Wiki pages. It never falls back to raw documents or chunks. The local vector feature is disabled by default. Configure `llmwiki.local_wiki_index.embedding` first, then enable `enabled`, `sync_enabled`, and `internal_search_enabled` in `config.yaml`. +The optional `embedding.dimensions` value is used only to validate the returned +vector size and build the index fingerprint; it is deliberately not sent in +the embedding request because fixed-dimension BGE/OpenAI-compatible endpoints +may reject that extra field with HTTP 400. Omit it to discover the dimension +from the first successful response. The migration creates `llmwiki_wiki_pages`, `llmwiki_wiki_vectors`, and `llmwiki_wiki_sync_states`. Wiki Markdown and normalized float32 vectors are @@ -814,13 +819,26 @@ without mounting the WeKnora iframe. Published knowledge bases also have a login-free, chrome-free reader at `/#/embed/knowledge/{mapping_id}` for direct third-party iframe use. With no -`wikiSlug`, it selects an overview/root page and keeps directory navigation -inside the embed route; `?wikiSlug=...` deep-links one page. The reader supports -`theme=light|dark`, `bg`, and `text` embed appearance parameters. Its data comes +`wikiSlug`, it selects the Wiki **索引** page first (recognized by title, page +type, or `index`/`home`/`readme` slug), pins that page to the top of the public +directory, and otherwise falls back to an overview/root page; +`?wikiSlug=...` deep-links one page. The reader supports +`theme=light|dark`, `bg`, and `text` embed appearance parameters. Index-article +catalogue sections (page-type counts, nested categories, page summaries) are +read from the published-only `/wiki/index` proxy because WeKnora renders that +part dynamically rather than storing it in the index page body; group and item +ordering therefore stays identical to WeKnora. Links generated as +knowledge-base-shell placeholders are matched +against the public page title, slug, or alias and rewritten to exact embedded +`wikiSlug` links. They retain WeKnora's brand-color medium text and dashed +underline (solid on hover), and switch articles inside the current reader +without a full-page reload. Data comes only from the read-only `/api/public/llmwiki/knowledge-bases/{id}/...` namespace, which rechecks `publication_status=published` on every request and denies the system conversation-deposit knowledge base. Private or unpublished bases are -never exposed by this route. +never exposed by this route. Anonymous consumers can discover the available +public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each +entry deliberately contains only the DeerFlow mapping `id` and display `name`. Wiki management also provides **导出 Excel**. `GET /api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx` exports every processed Wiki diff --git a/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py b/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py index 799f87a7..cee7b04a 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py +++ b/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py @@ -483,6 +483,21 @@ def _safe_response_headers(headers: httpx.Headers) -> dict[str, str]: return result +def _should_follow_weknora_binary_redirect(path: str) -> bool: + """Keep protected preview/image bytes inside the DeerFlow proxy.""" + + segments = _path_segments(path) + return ( + len(segments) == 5 + and segments[:3] == ["api", "v1", "knowledge"] + and segments[4] == "preview" + ) or ( + len(segments) == 5 + and segments[:3] == ["api", "v1", "knowledge-bases"] + and segments[4] == "files" + ) + + def _forward_request_headers(request: Request) -> dict[str, str]: blocked = { "host", @@ -1571,8 +1586,13 @@ async def _proxy_weknora_request( headers["X-Tenant-ID"] = tenant_id url = f"{target_base_url.rstrip('/')}{_encode_upstream_path(upstream_path)}" + follow_binary_redirects = is_api and _should_follow_weknora_binary_redirect(upstream_path) try: - async with httpx.AsyncClient(timeout=httpx.Timeout(60.0), follow_redirects=False, trust_env=False) as client: + async with httpx.AsyncClient( + timeout=httpx.Timeout(60.0), + follow_redirects=follow_binary_redirects, + trust_env=False, + ) as client: response = await client.request( request.method, url, @@ -1586,7 +1606,11 @@ async def _proxy_weknora_request( tenant_id = _tenant_id_from_session(session) if tenant_id: headers["X-Tenant-ID"] = tenant_id - async with httpx.AsyncClient(timeout=httpx.Timeout(60.0), follow_redirects=False, trust_env=False) as client: + async with httpx.AsyncClient( + timeout=httpx.Timeout(60.0), + follow_redirects=follow_binary_redirects, + trust_env=False, + ) as client: response = await client.request( request.method, url, @@ -2417,6 +2441,27 @@ async def test_llmwiki_connection(request: Request, body: LlmWikiConnectionTest) return {"success": True, "provider": "weknora", "health": health} +@router.get("/api/public/llmwiki/knowledge-bases") +async def list_public_knowledge_bases(request: Request) -> dict[str, Any]: + """List the minimal identity of every knowledge base published as a public Wiki.""" + + rows = await _store(request).list_visible( + "__public_wiki__", + scope="public", + is_admin=False, + ) + knowledge_bases = [ + { + "id": str(row["id"]), + "name": str(row.get("name") or ""), + } + for row in rows + if row.get("publication_status") == "published" + and not _is_conversation_deposit_mapping(row) + ] + return {"knowledge_bases": knowledge_bases, "total": len(knowledge_bases)} + + @router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}") async def get_public_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any]: """Return the safe public metadata used by the chrome-free Wiki reader.""" @@ -2465,6 +2510,89 @@ async def list_public_wiki_pages( return {**result, "pages": pages} +_PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison") + + +def _public_wiki_index_item(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + slug = str(value.get("slug") or "").strip() + if not slug: + return None + return { + "slug": slug, + "title": str(value.get("title") or slug), + "summary": str(value.get("summary") or ""), + "category_path": ( + [str(item) for item in value.get("category_path", []) if str(item).strip()] + if isinstance(value.get("category_path"), list) + else [] + ), + "wiki_path": str(value.get("wiki_path") or ""), + "depth": int(value.get("depth") or 0), + } + + +async def _all_public_wiki_index(client: WeKnoraClient, remote_id: str) -> dict[str, Any]: + intro = "" + version = 0 + groups: list[dict[str, Any]] = [] + for page_type in _PUBLIC_WIKI_INDEX_TYPES: + items: list[dict[str, Any]] = [] + cursor = "" + seen_cursors: set[str] = set() + total = 0 + for _page_number in range(100): + payload = await client.get_wiki_index( + remote_id, + types=[page_type], + limit=500, + cursor=cursor, + ) + intro = intro or str(payload.get("intro") or "") + version = max(version, int(payload.get("version") or 0)) + raw_groups = payload.get("groups") if isinstance(payload.get("groups"), list) else [] + group = next( + ( + value + for value in raw_groups + if isinstance(value, dict) and value.get("type") == page_type + ), + {}, + ) + total = max(total, int(group.get("total") or 0)) + raw_items = group.get("items") if isinstance(group.get("items"), list) else [] + items.extend( + item + for value in raw_items + if (item := _public_wiki_index_item(value)) is not None + ) + next_cursor = str(group.get("next_cursor") or "") + if not next_cursor or next_cursor in seen_cursors: + break + seen_cursors.add(next_cursor) + cursor = next_cursor + else: + logger.warning( + "Stopped public Wiki index pagination at safety limit: mapping_remote_id=%s page_type=%s", + remote_id, + page_type, + ) + groups.append({"type": page_type, "total": total or len(items), "items": items}) + return {"intro": intro, "version": version, "groups": groups} + + +@router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}/wiki/index") +async def get_public_wiki_index(request: Request, mapping_id: str) -> dict[str, Any]: + """Return WeKnora's ordered dynamic Wiki index for the public reader.""" + + row = await _public_mapping(request, mapping_id) + try: + return await _all_public_wiki_index(_client_or_503(), str(row["weknora_id"])) + except WeKnoraError as exc: + _raise_upstream(exc) + + @router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}/wiki/pages/{slug:path}") async def get_public_wiki_page( request: Request, diff --git a/offline-backend-20260512/backend/config.yaml b/offline-backend-20260512/backend/config.yaml index 83017f68..a8990137 100644 --- a/offline-backend-20260512/backend/config.yaml +++ b/offline-backend-20260512/backend/config.yaml @@ -573,6 +573,8 @@ llmwiki: model: bge-m3 base_url: "" api_key: "" + # Expected BGE-M3 response size for local validation/indexing only. + # This value is not sent as an embedding API request parameter. dimensions: 1024 batch_size: 32 timeout_seconds: 30 diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py index 8e46dbce..45776f0b 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py @@ -58,7 +58,12 @@ class LocalWikiEmbeddingConfig(BaseModel): model: str = "bge-m3" base_url: str = "" api_key: str = Field(default="", repr=False) - dimensions: int | None = Field(default=1024, ge=1, le=65536) + dimensions: int | None = Field( + default=1024, + ge=1, + le=65536, + description="Expected response vector dimensions; not sent to the embedding endpoint.", + ) batch_size: int = Field(default=32, ge=1, le=512) timeout_seconds: float = Field(default=30, gt=0, le=300) max_retries: int = Field(default=3, ge=0, le=10) diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py index 620fa1bb..2bd618c1 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py @@ -492,6 +492,31 @@ class WeKnoraClient: raise WeKnoraError("WeKnora returned an invalid Wiki page") return dict(value) + async def get_wiki_index( + self, + knowledge_base_id: str, + *, + types: Sequence[str], + limit: int = 50, + cursor: str = "", + ) -> dict[str, Any]: + params: dict[str, Any] = { + "types": ",".join(value for value in types if value), + "limit": max(1, min(limit, 500)), + } + if cursor: + params["cursor"] = cursor + value = _data( + await self._request( + "GET", + f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/index", + params=params, + ) + ) + if not isinstance(value, dict): + raise WeKnoraError("WeKnora returned an invalid Wiki index") + return dict(value) + async def create_wiki_page(self, knowledge_base_id: str, payload: Mapping[str, Any]) -> dict[str, Any]: value = _data( await self._request( diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py index 65a13a2a..f4b6b8ef 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py @@ -66,9 +66,11 @@ class StrictWikiEmbeddingClient: headers = {"Content-Type": "application/json"} if self.config.api_key: headers["Authorization"] = f"Bearer {self.config.api_key}" + # ``dimensions`` is a local invariant, not a portable request option. + # OpenAI-compatible BGE endpoints commonly reject that extra field + # with HTTP 400 even though they always return a fixed-size vector. + # Keep validating the response against the configured value below. payload: dict = {"model": self.config.model, "input": texts} - if self.config.dimensions: - payload["dimensions"] = self.config.dimensions last_error: WikiEmbeddingError | None = None for attempt in range(self.config.max_retries + 1): started = monotonic() diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py b/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py index 6fa74211..667a1dfe 100644 --- a/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py +++ b/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py @@ -117,6 +117,23 @@ def test_weknora_url_rejects_credentials_and_non_http_schemes() -> None: LlmWikiConfig.model_validate({"weknora": {"api_base_url": "http://user:pass@weknora.local"}}) +def test_weknora_embed_follows_only_protected_binary_redirects() -> None: + from app.gateway.routers import llmwiki as llmwiki_router + + assert llmwiki_router._should_follow_weknora_binary_redirect( + "/api/v1/knowledge/doc-one/preview" + ) + assert llmwiki_router._should_follow_weknora_binary_redirect( + "/api/v1/knowledge-bases/kb-one/files" + ) + assert not llmwiki_router._should_follow_weknora_binary_redirect( + "/api/v1/knowledge/doc-one" + ) + assert not llmwiki_router._should_follow_weknora_binary_redirect( + "/api/v1/knowledge-bases/kb-one" + ) + + def test_agent_config_tracks_allowed_llmwiki_knowledge_bases() -> None: config = AgentConfig( name="Research agent", @@ -547,6 +564,42 @@ async def test_weknora_client_imports_url_manual_content_and_lists_wiki_pages() assert pages["pages"][0]["slug"] == "guide" +@pytest.mark.asyncio +async def test_weknora_client_reads_ordered_wiki_index_groups() -> None: + async def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v1/knowledgebase/kb-one/wiki/index" + assert request.url.params["types"] == "entity,concept" + assert request.url.params["limit"] == "50" + assert request.url.params["cursor"] == "next-page" + return httpx.Response( + 200, + json={ + "data": { + "intro": "# Wiki Index", + "version": 2, + "groups": [ + { + "type": "entity", + "total": 1, + "items": [{"slug": "entity/one", "title": "实体一"}], + } + ], + } + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client: + client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client) + index = await client.get_wiki_index( + "kb-one", + types=["entity", "concept"], + limit=50, + cursor="next-page", + ) + + assert index["groups"][0]["items"][0]["slug"] == "entity/one" + + @pytest.mark.asyncio async def test_weknora_client_creates_updates_and_deletes_wiki_pages() -> None: seen: list[tuple[str, str]] = [] diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py b/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py index d13ceaaf..f4e2f725 100644 --- a/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py +++ b/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py @@ -1,8 +1,12 @@ from __future__ import annotations +import json + +import httpx import pytest -from deerflow.config.llmwiki_config import LocalWikiIndexConfig +from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig +from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient from deerflow.integrations.weknora.local_index.sync import WikiSyncService from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache from deerflow.persistence.llmwiki_index.memory import MemoryLlmWikiIndexStore @@ -53,6 +57,55 @@ class FakeWeKnora: return dict(self.pages[slug]) +@pytest.mark.asyncio +async def test_configured_dimensions_are_validated_but_not_sent_to_bge() -> None: + requests: list[dict] = [] + + def handler(request: httpx.Request) -> httpx.Response: + payload = json.loads(request.content) + requests.append(payload) + if "dimensions" in payload: + return httpx.Response(400, json={"detail": "dimensions is not supported"}) + return httpx.Response( + 200, + json={"data": [{"index": 0, "embedding": [1.0, 0.0, 0.0]}]}, + ) + + config = LocalWikiEmbeddingConfig( + model="bge-embedding-m3", + base_url="http://embedding.test/v1", + dimensions=3, + max_retries=0, + ) + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client: + client = StrictWikiEmbeddingClient(config, http_client=http_client) + vector = await client.embed_query("Wiki page") + + assert vector == [1.0, 0.0, 0.0] + assert requests == [{"model": "bge-embedding-m3", "input": ["Wiki page"]}] + + +@pytest.mark.asyncio +async def test_omitted_dimensions_are_discovered_from_first_response() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"data": [{"index": 0, "embedding": [0.0, 1.0]}]}, + ) + + config = LocalWikiEmbeddingConfig( + model="bge-embedding-m3", + base_url="http://embedding.test/v1", + dimensions=None, + max_retries=0, + ) + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client: + client = StrictWikiEmbeddingClient(config, http_client=http_client) + await client.embed_query("dimension probe") + + assert client.dimensions == 2 + + @pytest.mark.asyncio async def test_sync_is_incremental_and_full_scan_deletes_missing_pages() -> None: config = LocalWikiIndexConfig(chunk_min_chars=10, min_similarity=0) diff --git a/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py b/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py index 5c43a851..10c63e8b 100644 --- a/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py +++ b/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py @@ -38,6 +38,22 @@ class PublicStore: return self.row +class PublicListStore: + def __init__(self, rows: list[dict[str, object]]) -> None: + self.rows = rows + self.calls: list[tuple[str, str, bool]] = [] + + async def list_visible( + self, + user_id: str, + *, + scope: str, + is_admin: bool, + ) -> list[dict[str, object]]: + self.calls.append((user_id, scope, is_admin)) + return self.rows + + def _published_row() -> dict[str, object]: return { "id": "kb-one", @@ -50,6 +66,39 @@ def _published_row() -> dict[str, object]: } +@pytest.mark.asyncio +async def test_public_knowledge_base_list_exposes_only_published_wiki_identity() -> None: + store = PublicListStore( + [ + { + **_published_row(), + "id": "1063edee-332a-43b4-bbd9-73bf0a708b94", + "owner_user_id": "secret-owner", + }, + {**_published_row(), "id": "private-kb", "publication_status": "private"}, + { + **_published_row(), + "id": "conversation-deposit", + "owner_user_id": "system", + "name": "对话沉淀", + }, + ] + ) + + payload = await llmwiki.list_public_knowledge_bases(_request(store)) + + assert store.calls == [("__public_wiki__", "public", False)] + assert payload == { + "knowledge_bases": [ + { + "id": "1063edee-332a-43b4-bbd9-73bf0a708b94", + "name": "公开知识库", + } + ], + "total": 1, + } + + @pytest.mark.asyncio async def test_public_wiki_reader_uses_anonymous_non_admin_authorization(monkeypatch) -> None: store = PublicStore(_published_row()) @@ -126,3 +175,81 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None: assert payload["pages"][0]["slug"] == "guide/start" assert payload["pages"][0]["aliases"] == [] + + +@pytest.mark.asyncio +async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None: + store = PublicStore(_published_row()) + calls: list[tuple[tuple[str, ...], str]] = [] + + class Client: + async def get_wiki_index( + self, + knowledge_base_id: str, + *, + types: list[str], + limit: int, + cursor: str, + ): + assert knowledge_base_id == "remote-one" + assert limit == 500 + page_type = types[0] + calls.append((tuple(types), cursor)) + if page_type == "concept" and not cursor: + return { + "intro": "# Wiki Index\n\n公开目录", + "version": 3, + "groups": [ + { + "type": "concept", + "total": 2, + "next_cursor": "concept-next", + "items": [ + { + "slug": "concept/first", + "title": "第一项", + "summary": "第一项摘要", + "category_path": ["流程图", "规范"], + } + ], + } + ], + } + if page_type == "concept": + return { + "intro": "# Wiki Index\n\n公开目录", + "version": 3, + "groups": [ + { + "type": "concept", + "total": 2, + "items": [ + { + "slug": "concept/second", + "title": "第二项", + "summary": "第二项摘要", + "category_path": ["流程图", "规范"], + } + ], + } + ], + } + return { + "intro": "# Wiki Index\n\n公开目录", + "version": 3, + "groups": [{"type": page_type, "total": 0, "items": []}], + } + + monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client()) + + payload = await llmwiki.get_public_wiki_index(_request(store), "kb-one") + + concept = next(group for group in payload["groups"] if group["type"] == "concept") + assert payload["intro"] == "# Wiki Index\n\n公开目录" + assert payload["version"] == 3 + assert [item["slug"] for item in concept["items"]] == [ + "concept/first", + "concept/second", + ] + assert concept["total"] == 2 + assert (("concept",), "concept-next") in calls From 0c5e2e8cb0d722a24cd2ddcd79c4aeccfb30818e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=89=E9=91=AB=E5=8D=87?= Date: Tue, 25 Aug 2026 11:24:01 +0800 Subject: [PATCH 2/4] fix: restore embedded wiki images and optimize vector sync --- .../src/core/llmwiki/wiki-page-markdown.ts | 28 ++++ .../src/strategy-components/api/llmwiki.ts | 17 +++ .../llmwiki/WeKnoraSourceDrawer.tsx | 22 ++- .../llmwiki/WeKnoraWikiPageView.tsx | 15 +- .../llmwiki/WikiProtectedImage.tsx | 75 ++++++++++ offline-backend-20260512/backend/CLAUDE.md | 15 +- offline-backend-20260512/backend/README.md | 22 ++- .../backend/app/gateway/app.py | 2 +- .../backend/app/gateway/deps.py | 7 +- .../app/gateway/llmwiki_index_scheduler.py | 6 +- .../backend/app/gateway/routers/llmwiki.py | 133 +++++++++++++++++- .../app/gateway/routers/llmwiki_index.py | 4 + offline-backend-20260512/backend/config.yaml | 40 +----- .../LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md | 60 ++------ .../harness/deerflow/config/llmwiki_config.py | 28 +++- .../deerflow/integrations/weknora/client.py | 47 ++++++- .../weknora/local_index/embedding.py | 43 +++++- .../persistence/llmwiki_index/__init__.py | 8 +- .../deerflow/persistence/llmwiki_index/sql.py | 46 +++--- .../test_llmwiki_local_index_repository.py | 17 ++- .../backend/tests/test_llmwiki_weknora.py | 22 +++ .../backend/tests/test_llmwiki_wiki_sync.py | 57 ++++++++ .../tests/test_public_llmwiki_embed.py | 31 ++++ 23 files changed, 609 insertions(+), 136 deletions(-) create mode 100644 frontend-web/src/strategy-components/components/resource-management/llmwiki/WikiProtectedImage.tsx diff --git a/frontend-web/src/core/llmwiki/wiki-page-markdown.ts b/frontend-web/src/core/llmwiki/wiki-page-markdown.ts index 4ee93185..17a2a60f 100644 --- a/frontend-web/src/core/llmwiki/wiki-page-markdown.ts +++ b/frontend-web/src/core/llmwiki/wiki-page-markdown.ts @@ -37,6 +37,34 @@ const WIKI_INDEX_TYPE_LABELS: Record = { const WIKI_INDEX_TYPE_ORDER = ["summary", "entity", "concept"]; +export const WIKI_PROTECTED_IMAGE_MARKER = "/__llmwiki-protected-image__"; + +const PROTECTED_WIKI_IMAGE_SOURCE = + /(?:storage:\/\/[0-9A-Za-z_-]+\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\/\/[^\s"'<>\])]+/gi; + +/** + * Browsers cannot load WeKnora's protected resource:// and object-storage + * schemes directly. Replace them with a safe marker before Markdown + * sanitization; the image component resolves that marker through DeerFlow. + */ +export function rewriteProtectedWikiImageSources(markdown: string): string { + return markdown.replace( + PROTECTED_WIKI_IMAGE_SOURCE, + (filePath) => `${WIKI_PROTECTED_IMAGE_MARKER}?file_path=${encodeURIComponent(filePath)}`, + ); +} + +export function parseProtectedWikiImageSource(source: string | undefined): string | null { + if (!source) return null; + try { + const url = new URL(source, window.location.origin); + if (url.pathname !== WIKI_PROTECTED_IMAGE_MARKER) return null; + return url.searchParams.get("file_path")?.trim() || null; + } catch { + return null; + } +} + export function stripWikiFrontmatter(value: string) { return value.replace(/^\s*---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, "").trim(); } diff --git a/frontend-web/src/strategy-components/api/llmwiki.ts b/frontend-web/src/strategy-components/api/llmwiki.ts index d7df1d9f..5b8f16c9 100644 --- a/frontend-web/src/strategy-components/api/llmwiki.ts +++ b/frontend-web/src/strategy-components/api/llmwiki.ts @@ -444,6 +444,23 @@ export async function getLlmWikiDocumentPreview( return response.blob(); } +export async function getLlmWikiProtectedFile( + knowledgeBaseId: string, + filePath: string, + publicAccess = false, +): Promise { + const base = publicAccess ? PUBLIC_API() : API(); + const params = new URLSearchParams({ file_path: filePath }); + const response = await apiFetch( + `${base}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/files?${params.toString()}`, + ); + if (!response.ok) { + await readJson(response, "读取 Wiki 图片"); + throw new Error("读取 Wiki 图片失败"); + } + return response.blob(); +} + export async function getLlmWikiGraph(knowledgeBaseId: string): Promise { return readJson( await apiFetch(`${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/graph?limit=200`), diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx index 272457d2..d013fa9c 100644 --- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx @@ -33,6 +33,7 @@ import { reasoningPlugins } from "@/core/streamdown"; import { exportableWikiMarkdown, renderableWikiMarkdown, + rewriteProtectedWikiImageSources, safeWikiFileName, stripWikiFrontmatter, wikiLinkMarkdown, @@ -49,6 +50,7 @@ import { type LlmWikiDocument, type LlmWikiWikiPage, } from "@/strategy-components/api/llmwiki"; +import { WikiProtectedImage } from "./WikiProtectedImage"; type SourceContextResponse = { success?: boolean; @@ -926,10 +928,18 @@ export function WeKnoraSourceDrawer({
( + + ), + }} /> @@ -964,10 +974,18 @@ export function WeKnoraSourceDrawer({
( + + ), + }} linkClassName={(href) => parseWeKnoraPreviewUrl(href)?.wikiSlug ? "wiki-content-link font-medium text-[#be2323] decoration-[#be2323]/55 decoration-dashed hover:decoration-solid" diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx index 1a6bd740..b2af1866 100644 --- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx @@ -35,6 +35,7 @@ import { markdownToDocxBlob } from "@/core/scheduled-tasks/markdown-docx"; import { exportableWikiMarkdown, renderableWikiMarkdown, + rewriteProtectedWikiImageSources, safeWikiFileName, stripWikiFrontmatter, wikiLinkMarkdown, @@ -49,6 +50,7 @@ import { usePublicLlmWikiWikiPage, usePublicLlmWikiWikiPages, } from "@/strategy-components/api/llmwiki"; +import { WikiProtectedImage } from "./WikiProtectedImage"; const LIST_PATH = "/page/workspace/knowledge"; @@ -209,7 +211,7 @@ export function WeKnoraWikiPageView({ : pageData?.content, )); const displayMarkdown = pageData - ? renderableWikiMarkdown( + ? rewriteProtectedWikiImageSources(renderableWikiMarkdown( title, content, noteId, @@ -222,7 +224,7 @@ export function WeKnoraWikiPageView({ : undefined, publicEmbed ? pagesList : [], publicEmbed ? publicWikiIndex.data?.groups || [] : [], - ) + )) : ""; const exportMarkdown = pageData ? wikiLinkMarkdown(exportableWikiMarkdown(title, summary, content), noteId) @@ -497,6 +499,15 @@ export function WeKnoraWikiPageView({ isLoading={false} remarkPlugins={reasoningPlugins.remarkPlugins} rehypePlugins={reasoningPlugins.rehypePlugins} + components={{ + img: (props) => ( + + ), + }} linkClassName={(href) => publicEmbed && publicWikiTargetFromHref(href, noteId) ? "wiki-content-link border-b border-dashed !border-[var(--primary)] font-medium !text-primary !no-underline hover:border-solid focus-visible:rounded-sm focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary" diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WikiProtectedImage.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WikiProtectedImage.tsx new file mode 100644 index 00000000..816aeecf --- /dev/null +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WikiProtectedImage.tsx @@ -0,0 +1,75 @@ +import { ImageOff, Loader2 } from "lucide-react"; +import { type ImgHTMLAttributes, useEffect, useState } from "react"; + +import { parseProtectedWikiImageSource } from "@/core/llmwiki/wiki-page-markdown"; +import { getLlmWikiProtectedFile } from "@/strategy-components/api/llmwiki"; + +type WikiProtectedImageProps = ImgHTMLAttributes & { + knowledgeBaseId: string; + publicAccess?: boolean; +}; + +export function WikiProtectedImage({ + knowledgeBaseId, + publicAccess = false, + src, + alt, + className, + ...props +}: WikiProtectedImageProps) { + const filePath = parseProtectedWikiImageSource(src); + const [objectUrl, setObjectUrl] = useState(""); + const [error, setError] = useState(""); + + useEffect(() => { + if (!filePath || !knowledgeBaseId) { + setObjectUrl(""); + setError(""); + return; + } + let cancelled = false; + let nextObjectUrl = ""; + setObjectUrl(""); + setError(""); + void getLlmWikiProtectedFile(knowledgeBaseId, filePath, publicAccess) + .then((blob) => { + if (cancelled) return; + nextObjectUrl = URL.createObjectURL(blob); + setObjectUrl(nextObjectUrl); + }) + .catch((reason: unknown) => { + if (!cancelled) { + setError(reason instanceof Error ? reason.message : "Wiki 图片加载失败"); + } + }); + return () => { + cancelled = true; + if (nextObjectUrl) URL.revokeObjectURL(nextObjectUrl); + }; + }, [filePath, knowledgeBaseId, publicAccess]); + + if (!filePath) { + return {alt}; + } + if (error) { + return ( + + + {alt || "Wiki 图片加载失败"} + + ); + } + if (!objectUrl) { + return ( + + + + ); + } + return {alt}; +} diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index 6d44c4ea..98ba14f4 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -1016,6 +1016,11 @@ Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` me `llmwiki.weknora.api_base_url` is the provider switch: non-empty enables WeKnora, empty preserves the legacy LLMWiki UI. WeKnora is independently deployed and owns PostgreSQL/Redis/object storage/vector storage/parsers/models; DeerFlow config contains only `api_base_url` and optional `web_base_url`. DeerFlow deliberately sends no WeKnora credential, so the WeKnora endpoint must be restricted to a trusted network and allow DeerFlow traffic itself or through its reverse proxy. There is intentionally no `default_embedding_model_id`: `WeKnoraClient` queries `/api/v1/models` and uses WeKnora's active Embedding model when creating a KB; Wiki creation additionally resolves an active KnowledgeQA model inside WeKnora for synthesis. The native BFF is `app/gateway/routers/llmwiki.py`; ownership/publication mappings live in `deerflow.persistence.llmwiki`; the browser only sees DeerFlow mapping IDs. Knowledge-base owners publish immediately (`private` -> `published`) without a review queue, and can unpublish ordinary bases. The conversation-deposit KB is system-owned, always published, shared by all DeerFlow users, excluded from My Knowledge, and immutable through user/admin detail operations; recognition requires both the reserved name and `system` owner. Existing same-name remote mappings are adopted in place as the system deposit, preserving mapping ids and document links. Its user-facing description, generated Markdown source and tags use `cmzs` branding; list/detail reads synchronize stale remote descriptions, and search results, source previews and embedded-detail JSON recursively replace legacy brand text in historical conversation-deposit payloads. Deposit requests must reference a thread accessible to the current account. Personal knowledge-base list/search views are scoped to the current DeerFlow account even for admins; frontend LLMWiki caches include the live account id, and admin write power does not make other users' private KBs appear as owned. Only `system_role === "admin"` sees the WeKnora connection-config tab. Knowledge cards navigate to a dedicated WeKnora-style detail route. Wiki-enabled bases render and default to `Wiki -> documents -> graph`; standard bases omit Wiki and render `documents -> graph`. The detail page supports sequential multi-file upload with an aggregate result for writable ordinary bases, hides it for public read-only/system-deposit bases, and uses a generic loading label while the embed session starts. It also supports URL and published manual-Markdown imports, browses real Wiki pages/content, proxies authenticated original-file previews, pages through `/chunks/{knowledge_id}`, and renders the Wiki page graph from `/knowledgebase/{kb_id}/wiki/graph`; document details use a right sheet rather than the removed KB modal. The credential-free proxy uses one WeKnora service identity, so per-user separation is enforced by DeerFlow mappings while all remote KBs physically share that identity's WeKnora workspace. WeKnora iframe detail pages first call `/api/llmwiki/knowledge-bases/{id}/weknora/session` through the normal DeerFlow API auth path, then load `/platform/knowledge-bases/{weknora_id}` with a signed `deerflow_weknora_embed` cookie for `/platform`, `/assets`, `/locales`, root assets, and API calls. Injected fetch/XHR rewriting sends all iframe `/api/v1/*`, including colliding `/api/v1/auth/*`, through `/api/llmwiki/weknora-embed/api/v1/*`; Gateway auth/CSRF bypass only applies to those signed iframe proxy requests, the proxy still revalidates the mapped KB, and cookie security honors `X-Forwarded-Proto` for HTTPS reverse proxies. `llmwiki_search` is conditionally registered in `tools.py`, is read-only, and rechecks the current DeerFlow user, agent `llmwiki_knowledge_base_ids`, and thread-context selection on every call. Source previews prefer the cleaned Wiki/LLMWiki page text when a retrieved chunk maps to a Wiki page, with raw split chunks as the fallback. Agent bindings use an inline WeKnora-style searchable/grouped multi-select in the standard agent create/detail UI; new chats use the same interaction pattern as a compact composer dropdown, and selections are written to thread metadata/context and restored on reopen. The DeerFlow frontend is native React and the WeKnora frontend is not modified or embedded. Full deployment and behavior notes: `docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`. Tests: `tests/test_llmwiki_weknora.py`. The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. Do not add an HTML `sandbox` attribute to this trusted, permission-filtered detail iframe: a sandboxed ancestor causes Chromium's built-in Blob PDF viewer to render the browser-blocked page. The Gateway follows upstream redirects only for `/knowledge/{id}/preview` and `/knowledge-bases/{id}/files`, keeping PDF bytes and protected Wiki images inside the signed proxy instead of sending the browser to WeKnora object storage. +Native/private and public Wiki readers must rewrite protected Markdown image +sources to their mapping-scoped DeerFlow `/files?file_path=` endpoints and load +them as Blob URLs. The embedded guard also rewrites protected fetch/XHR values +and hydrates late `img[data-protected-src]` placeholders; never expose the +service token or relax the current-KB authorization to make images work. **Embedded native knowledge-base switching:** The original WeKnora breadcrumb picker remains available inside the detail iframe. A signed 15-minute embed cookie carries only the current DeerFlow user's visible mappings (their own plus published bases); the proxied `/api/v1/knowledge-bases` response is filtered to that allowlist. Choosing another entry performs a full iframe navigation, rechecks the DeerFlow mapping and read/write permission, then reissues the cookie for the selected base. Never expose the upstream service-admin list directly or permit an arbitrary remote knowledge-base ID. @@ -1442,8 +1447,10 @@ detail. Never call `/knowledge-search` from internal search, RAG, or the `deerflow.integrations.weknora.local_index` and `deerflow.persistence.llmwiki_index`; preserve the harness → app import firewall. Gateway composition injects the store/search/sync services through -`local_index.runtime`, and the lease-protected scheduler starts only when both -`enabled` and `sync_enabled` are true. +`local_index.runtime`. `enabled` turns on local vector search and manual +vectorization; `auto_sync` is the separate opt-in for the lease-protected +scheduler. The scheduler sleeps for one full configured interval before its +first scan, so application startup must never trigger immediate vectorization. Vectors are normalized little-endian float32 BLOBs. A page mirror and all of its vectors are replaced in one transaction only after strict batched @@ -1456,6 +1463,10 @@ input, not an outbound API field: fixed-dimension BGE/OpenAI-compatible endpoints can reject a `dimensions` request parameter with HTTP 400. When the value is omitted, the embedding client probes and freezes the first response's dimension before sync lease/rebuild decisions. +Embedding errors must retain the upstream status and bounded response body in +logs, without exposing API keys. SQL vector writes are split into statements +of at most `database_batch_size` rows (default 50) inside the same atomic page +replacement transaction. During a fingerprint rebuild the mapping uses the Wiki-page API until every page is successfully rebuilt. `WikiRetrievalService` partitions mixed requests per mapping, so ready libraries stay local while incomplete libraries use that diff --git a/offline-backend-20260512/backend/README.md b/offline-backend-20260512/backend/README.md index fa14f92a..56d091ad 100644 --- a/offline-backend-20260512/backend/README.md +++ b/offline-backend-20260512/backend/README.md @@ -777,15 +777,24 @@ processed Wiki content. A knowledge base uses exact NumPy cosine similarity only after its whole Wiki has completed one coherent vectorization pass; until then internal search calls WeKnora's Wiki-page search/list API and reads the matched Wiki pages. It never falls back to raw documents or chunks. The local -vector feature is disabled by default. Configure -`llmwiki.local_wiki_index.embedding` first, then enable `enabled`, -`sync_enabled`, and `internal_search_enabled` in `config.yaml`. +vector feature is disabled by default. Configure the three +`llmwiki.local_wiki_index.embedding` values (`model`, `base_url`, and +`api_key`), then change only `enabled` to `true` in `config.yaml`. Manual +vectorization is the default. Set `auto_sync: true` only when periodic +background synchronization is wanted; the scheduler waits one full interval +after startup before its first scan, so boot does not trigger a rebuild. +`database_batch_size` caps each vector insert statement (default `50`) while +the page replacement remains atomic, preventing a large Wiki from flooding +the SQL connection in one statement. The optional `embedding.dimensions` value is used only to validate the returned vector size and build the index fingerprint; it is deliberately not sent in the embedding request because fixed-dimension BGE/OpenAI-compatible endpoints may reject that extra field with HTTP 400. Omit it to discover the dimension from the first successful response. +Embedding failures log the upstream HTTP status and a bounded response body. +The client never sends a `dimensions` request parameter. + The migration creates `llmwiki_wiki_pages`, `llmwiki_wiki_vectors`, and `llmwiki_wiki_sync_states`. Wiki Markdown and normalized float32 vectors are stored in DeerFlow's configured SQL database. Multi-worker sync is protected by @@ -839,6 +848,13 @@ system conversation-deposit knowledge base. Private or unpublished bases are never exposed by this route. Anonymous consumers can discover the available public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each entry deliberately contains only the DeerFlow mapping `id` and display `name`. +Protected `resource://` and supported object-storage images are fetched through +the current mapping instead of being exposed as upstream URLs. Authenticated +readers use `GET /api/llmwiki/knowledge-bases/{id}/files?file_path=...`; public +readers use the corresponding `/api/public/llmwiki/...` route, which requires a +published mapping. The embedded WeKnora guard applies the same mapping-scoped +proxy to document previews and Markdown images, including images inserted after +the drawer/page first renders. Wiki management also provides **导出 Excel**. `GET /api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx` exports every processed Wiki diff --git a/offline-backend-20260512/backend/app/gateway/app.py b/offline-backend-20260512/backend/app/gateway/app.py index 315d1729..a37b031f 100644 --- a/offline-backend-20260512/backend/app/gateway/app.py +++ b/offline-backend-20260512/backend/app/gateway/app.py @@ -998,7 +998,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: local_wiki_index = app.state.config.llmwiki.local_wiki_index if ( local_wiki_index.enabled - and local_wiki_index.sync_enabled + and local_wiki_index.auto_sync and getattr(app.state, "llmwiki_sync_service", None) is not None ): from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop diff --git a/offline-backend-20260512/backend/app/gateway/deps.py b/offline-backend-20260512/backend/app/gateway/deps.py index e76c54ea..1a1287e7 100644 --- a/offline-backend-20260512/backend/app/gateway/deps.py +++ b/offline-backend-20260512/backend/app/gateway/deps.py @@ -321,13 +321,16 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]: from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime from deerflow.persistence.llmwiki_index import make_llmwiki_index_store - app.state.llmwiki_index_store = make_llmwiki_index_store(sf) + local_index_config = config.llmwiki.local_wiki_index + app.state.llmwiki_index_store = make_llmwiki_index_store( + sf, + write_batch_size=local_index_config.database_batch_size, + ) app.state.llmwiki_embedding = None app.state.llmwiki_vector_cache = None app.state.llmwiki_vector_search = None app.state.llmwiki_sync_service = None app.state.llmwiki_retrieval_service = None - local_index_config = config.llmwiki.local_wiki_index from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime resolved_llmwiki = get_resolved_llmwiki_runtime(config) diff --git a/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py b/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py index 8964378e..a5363b21 100644 --- a/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py +++ b/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py @@ -48,6 +48,10 @@ async def _sync_once(app: FastAPI) -> None: async def local_wiki_index_scheduler_loop(app: FastAPI) -> None: interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds + logger.info( + "Local Wiki auto-sync scheduler started; first scan delayed by %s seconds", + interval, + ) while True: - await _sync_once(app) await asyncio.sleep(interval) + await _sync_once(app) diff --git a/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py b/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py index cee7b04a..560dadfd 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py +++ b/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py @@ -61,6 +61,10 @@ router = APIRouter(tags=["llmwiki"]) proxy_router = APIRouter(tags=["llmwiki-proxy"]) _MAX_UPLOAD_BYTES = 100 * 1024 * 1024 +_PROTECTED_WIKI_FILE_PATH = re.compile( + r"^(?:storage://[0-9A-Za-z_-]+/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs)://", + re.IGNORECASE, +) _WEKNORA_ADMIN_SESSION_CACHE: dict[str, dict[str, Any]] = {} _WEKNORA_PUBLIC_PREFIX = "/deerflow" _WEKNORA_PUBLIC_ROOT_PATHS = ( @@ -242,7 +246,7 @@ def _store(request: Request) -> LlmWikiStore: def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None: service = getattr(request.app.state, "llmwiki_sync_service", None) config = request.app.state.config.llmwiki.local_wiki_index - if service is not None and config.enabled and mapping.get("wiki_index_enabled", True): + if service is not None and config.enabled and config.auto_sync and mapping.get("wiki_index_enabled", True): from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True)) @@ -1085,6 +1089,8 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla function rewriteApiUrl(value) {{ try {{ + const protectedUrl = protectedImageApiUrl(value); + if (protectedUrl) return protectedUrl; const next = new URL(value, location.origin); if (next.origin !== location.origin) return value; const publicPrefix = cfg.publicPrefix || ""; @@ -1104,6 +1110,19 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla return value; }} }} + function protectedImageApiUrl(value) {{ + const raw = String(value || "").trim(); + if (!/^(?:storage:\\/\\/[0-9A-Za-z_-]+\\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\\/\\//i.test(raw)) {{ + return ""; + }} + const endpoint = + cfg.apiPrefix + + "/api/v1/knowledge-bases/" + + encodeURIComponent(cfg.weknoraId) + + "/files?file_path=" + + encodeURIComponent(raw); + return new URL(endpoint, location.origin).href; + }} const nativeFetch = window.fetch.bind(window); window.fetch = (input, init) => {{ if (input instanceof Request) {{ @@ -1116,6 +1135,67 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla return nativeXhrOpen.call(this, method, rewriteApiUrl(url), ...rest); }}; + // WeKnora renders protected Markdown images as transparent placeholders and + // hydrates them asynchronously. In an embedded page that hydration can run + // before its own API client has picked up our proxy prefix, leaving a 1x1 + // GIF forever. Resolve the same protected source through the scoped proxy + // as a fallback. The backend validates the scheme and current KB again. + function hydrateProtectedImage(image) {{ + const protectedSrc = String(image?.getAttribute?.("data-protected-src") || "").trim(); + const proxyUrl = protectedImageApiUrl(protectedSrc); + if (!proxyUrl) return; + if ( + image.dataset.deerflowProtectedSource === protectedSrc && + String(image.getAttribute("src") || "").startsWith("blob:") + ) return; + const attempts = Number(image.dataset.deerflowProtectedAttempts || "0"); + if (attempts >= 3 || image.dataset.deerflowProtectedLoading === "1") return; + image.dataset.deerflowProtectedLoading = "1"; + image.dataset.deerflowProtectedAttempts = String(attempts + 1); + nativeFetch(proxyUrl, {{ credentials: "same-origin", headers: {{ Accept: "image/*" }} }}) + .then((response) => {{ + if (!response.ok) throw new Error(`protected image request failed (${{response.status}})`); + return response.blob(); + }}) + .then((blob) => {{ + const objectUrl = URL.createObjectURL(blob); + image.dataset.deerflowProtectedSource = protectedSrc; + image.removeAttribute("data-img-loading"); + image.removeAttribute("data-wiki-image-error"); + image.src = objectUrl; + image.addEventListener("load", () => URL.revokeObjectURL(objectUrl), {{ once: true }}); + }}) + .catch(() => {{ + image.dataset.wikiImageError = "true"; + window.setTimeout(() => {{ + delete image.dataset.deerflowProtectedLoading; + hydrateProtectedImage(image); + }}, 1200 * (attempts + 1)); + }}) + .finally(() => {{ + delete image.dataset.deerflowProtectedLoading; + }}); + }} + function hydrateProtectedImages(scope = document) {{ + if (scope?.matches?.("img[data-protected-src]")) hydrateProtectedImage(scope); + scope?.querySelectorAll?.("img[data-protected-src]").forEach(hydrateProtectedImage); + }} + const protectedImageObserver = new MutationObserver((mutations) => {{ + mutations.forEach((mutation) => {{ + if (mutation.type === "attributes") hydrateProtectedImage(mutation.target); + mutation.addedNodes?.forEach((node) => {{ + if (node?.nodeType === Node.ELEMENT_NODE) hydrateProtectedImages(node); + }}); + }}); + }}); + protectedImageObserver.observe(document.documentElement, {{ + childList: true, + subtree: true, + attributes: true, + attributeFilter: ["data-protected-src"] + }}); + hydrateProtectedImages(); + function tone(color, alpha) {{ return `color-mix(in srgb, ${{color}} ${{Math.round(alpha * 100)}}%, transparent)`; }} @@ -1519,6 +1599,7 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla enforceLocation(); purgeGuides(); hideWeKnoraShell(); + hydrateProtectedImages(); if (tickSourceDetail() && sourceDetailMode) {{ clearInterval(maintenanceTimer); try {{ readyObserver.disconnect(); }} catch (_) {{}} @@ -1820,6 +1901,32 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]: return row +def _validate_protected_wiki_file_path(file_path: str) -> str: + value = file_path.strip() + if not value or "\x00" in value or not _PROTECTED_WIKI_FILE_PATH.match(value): + raise HTTPException(status_code=422, detail="Unsupported Wiki image path") + return value + + +async def _wiki_file_response(row: dict[str, Any], file_path: str, *, public: bool) -> Response: + try: + content, media_type = await _client_or_503().get_knowledge_base_file( + str(row["weknora_id"]), + _validate_protected_wiki_file_path(file_path), + ) + except WeKnoraError as exc: + _raise_upstream(exc) + return Response( + content=content, + media_type=media_type, + headers={ + "Cache-Control": "public, max-age=300" if public else "private, max-age=60", + "Content-Disposition": "inline", + "X-Content-Type-Options": "nosniff", + }, + ) + + async def _verify_document(client: WeKnoraClient, row: dict[str, Any], document_id: str) -> dict[str, Any]: document = await client.get_document(document_id) if str(document.get("knowledge_base_id") or "") != str(row["weknora_id"]): @@ -2510,6 +2617,18 @@ async def list_public_wiki_pages( return {**result, "pages": pages} +@router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}/files") +async def get_public_wiki_file( + request: Request, + mapping_id: str, + file_path: str = Query(..., min_length=1, max_length=8192), +) -> Response: + """Serve one protected image belonging to a published Wiki.""" + + row = await _public_mapping(request, mapping_id) + return await _wiki_file_response(row, file_path, public=True) + + _PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison") @@ -2820,6 +2939,18 @@ async def preview_document(request: Request, mapping_id: str, document_id: str) ) +@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/files") +async def get_wiki_file( + request: Request, + mapping_id: str, + file_path: str = Query(..., min_length=1, max_length=8192), +) -> Response: + """Serve one protected Wiki image after normal knowledge-base authorization.""" + + row, _, _ = await _authorized_mapping(request, mapping_id, write=False) + return await _wiki_file_response(row, file_path, public=False) + + @router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph") async def get_knowledge_base_graph( request: Request, diff --git a/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py b/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py index 313522ee..8c69ccea 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py +++ b/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from typing import Any from uuid import uuid4 @@ -14,6 +15,7 @@ from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectori from deerflow.integrations.weknora.local_index.search import WikiIndexError router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"]) +logger = logging.getLogger(__name__) class WikiVectorSearchRequest(BaseModel): @@ -181,6 +183,7 @@ async def _run_sync(app, mapping_ids: list[str], force: bool) -> None: try: await service.sync_mapping(mapping, force=force) except Exception: + logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id")) continue @@ -191,6 +194,7 @@ async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None: try: await service.sync_mapping(mapping, force=force) except Exception: + logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id")) return diff --git a/offline-backend-20260512/backend/config.yaml b/offline-backend-20260512/backend/config.yaml index a8990137..31606d1a 100644 --- a/offline-backend-20260512/backend/config.yaml +++ b/offline-backend-20260512/backend/config.yaml @@ -556,47 +556,17 @@ llmwiki: web_base_url: "http://119.254.86.251:8088" admin_email: "lqq@qq.com" admin_password: "1qaz@WSX" - # DeerFlow-local Wiki mirror/vector index. Keep disabled until the - # OpenAI-compatible embedding endpoint below is configured and reachable. + # 本地 Wiki 向量化:只需把 enabled 改为 true,并填写下面 3 个 embedding 值。 + # 默认只在页面点击“向量化”时执行;auto_sync=true 才会后台自动增量同步, + # 且服务启动时只启动定时器,不会立刻全量向量化。 local_wiki_index: enabled: false - sync_enabled: false - internal_search_enabled: false - reference_ui_enabled: false - sync_interval_seconds: 120 - chunk_max_chars: 1000 - chunk_overlap_chars: 120 - chunk_min_chars: 80 + auto_sync: false + database_batch_size: 50 embedding: - enabled: true - provider: openai model: bge-m3 base_url: "" api_key: "" - # Expected BGE-M3 response size for local validation/indexing only. - # This value is not sent as an embedding API request parameter. - dimensions: 1024 - batch_size: 32 - timeout_seconds: 30 - max_retries: 3 - engine: numpy_exact - strict_vector: true - top_k_sections: 30 - top_k_pages: 8 - max_sections_per_page: 2 - min_similarity: 0.35 - cache_max_knowledge_bases: 20 - cache_revision_check_seconds: 5 - external_api: - enabled: false - api_key: "" - previous_api_key: "" - max_query_chars: 1000 - max_knowledge_bases: 20 - max_results: 50 - default_results: 10 - include_content_max_chars: 12000 - requests_per_minute: 60 log_system: enabled: true diff --git a/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md b/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md index 0a166d91..02ed80c8 100644 --- a/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md +++ b/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md @@ -279,65 +279,25 @@ llmwiki: local_wiki_index: enabled: true - - # 同步 - sync_enabled: true - sync_interval_seconds: 120 - sync_page_size: 100 - sync_concurrency: 3 - sync_lease_seconds: 600 - sync_request_timeout_seconds: 30 - sync_max_retries: 3 - sync_retry_backoff_seconds: 5 - - # 切分 - chunk_max_chars: 1000 - chunk_overlap_chars: 120 - chunk_min_chars: 80 - title_prefix_enabled: true - - # Embedding:OpenAI-compatible /v1/embeddings + # 默认只允许手动向量化;需要定时增量同步时再开启。 + auto_sync: false + # 每条 SQL 最多插入的向量行数;整页替换仍在同一事务内完成。 + database_batch_size: 50 embedding: - enabled: true - provider: openai model: bge-m3 base_url: "$WIKI_EMBEDDING_BASE_URL" api_key: "$WIKI_EMBEDDING_API_KEY" - dimensions: 1024 - batch_size: 32 - timeout_seconds: 30 - max_retries: 3 - - # 检索 - engine: numpy_exact - strict_vector: true - top_k_sections: 30 - top_k_pages: 8 - max_sections_per_page: 2 - min_similarity: 0.35 - keyword_boost_enabled: false - keyword_boost_weight: 0.15 - cache_max_knowledge_bases: 20 - cache_revision_check_seconds: 5 - - # 外部接口 - external_api: - enabled: false - api_key: "$LLMWIKI_EXTERNAL_SEARCH_API_KEY" - max_query_chars: 1000 - max_knowledge_bases: 20 - max_results: 50 - default_results: 10 - include_content_max_chars: 12000 - requests_per_minute: 60 ``` +日常部署只需配置上面三个 Embedding 值并将 `enabled` 改为 `true`。 +`auto_sync=false` 时仅由页面“手动向量化”触发;即使设为 `true`,定时器也会先等待一个完整同步周期,不会在服务刚启动时立即重建索引。其余切分、检索、超时与重试参数继续使用代码默认值,只有确有调优需要时才覆盖。 + ### 6.1 配置校验规则 启动时应执行: 1. `enabled=true` 时,Embedding `base_url` 和 `model` 必须非空; -2. `dimensions` 可省略,但第一次成功 Embedding 后必须固定并写入索引 profile; +2. 不向上游发送 `dimensions` 参数;第一次成功 Embedding 后从响应自动发现维度,并固定到索引 profile; 3. `chunk_overlap_chars < chunk_max_chars`; 4. `top_k_pages <= top_k_sections`; 5. `external_api.enabled=true` 时必须配置非空 API Key; @@ -1885,9 +1845,7 @@ class WikiVectorEngine(ABC): ```text local_wiki_index.enabled -local_wiki_index.sync_enabled -local_wiki_index.internal_search_enabled -local_wiki_index.reference_ui_enabled +local_wiki_index.auto_sync local_wiki_index.external_api.enabled ``` diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py index 45776f0b..ae2427af 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py @@ -59,7 +59,7 @@ class LocalWikiEmbeddingConfig(BaseModel): base_url: str = "" api_key: str = Field(default="", repr=False) dimensions: int | None = Field( - default=1024, + default=None, ge=1, le=65536, description="Expected response vector dimensions; not sent to the embedding endpoint.", @@ -108,9 +108,7 @@ class LocalWikiIndexConfig(BaseModel): """DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages.""" enabled: bool = False - sync_enabled: bool = False - internal_search_enabled: bool = False - reference_ui_enabled: bool = False + auto_sync: bool = False sync_interval_seconds: int = Field(default=120, ge=10, le=86400) sync_page_size: int = Field(default=100, ge=1, le=500) sync_concurrency: int = Field(default=3, ge=1, le=32) @@ -133,9 +131,31 @@ class LocalWikiIndexConfig(BaseModel): keyword_boost_weight: float = Field(default=0.15, ge=0, le=1) cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000) cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600) + database_batch_size: int = Field(default=50, ge=1, le=500) external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig) model_config = ConfigDict(extra="ignore") + @model_validator(mode="before") + @classmethod + def _migrate_legacy_flags(cls, value): + if isinstance(value, dict) and "auto_sync" not in value and "sync_enabled" in value: + value = {**value, "auto_sync": value.get("sync_enabled")} + return value + + @property + def sync_enabled(self) -> bool: + """Backward-compatible status name; automatic work is one explicit switch.""" + + return self.auto_sync + + @property + def internal_search_enabled(self) -> bool: + return self.enabled + + @property + def reference_ui_enabled(self) -> bool: + return self.enabled + @model_validator(mode="after") def _validate_index(self): if self.chunk_overlap_chars >= self.chunk_max_chars: diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py index 2bd618c1..1589dc55 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py @@ -2,8 +2,8 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence import time +from collections.abc import Mapping, Sequence from typing import Any from urllib.parse import quote @@ -430,6 +430,51 @@ class WeKnoraClient: media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip() return response.content, media_type or "application/octet-stream" + async def get_knowledge_base_file( + self, + knowledge_base_id: str, + file_path: str, + ) -> tuple[bytes, str]: + """Return one protected Wiki attachment using DeerFlow's WeKnora identity.""" + + owns_client = self._http_client is None + client = self._http_client or httpx.AsyncClient( + timeout=httpx.Timeout(self.timeout_seconds), + follow_redirects=True, + trust_env=False, + ) + try: + for attempt in range(2): + response = await client.request( + "GET", + f"{self.base_url}/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/files", + params={"file_path": file_path}, + headers=await self._headers(force_refresh=attempt > 0, accept="*/*"), + follow_redirects=True, + ) + if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0: + break + except httpx.TimeoutException as exc: + raise WeKnoraError("WeKnora Wiki image request timed out", status_code=504) from exc + except httpx.HTTPError as exc: + raise WeKnoraError("WeKnora Wiki image is unavailable", status_code=502) from exc + finally: + if owns_client: + await client.aclose() + if response.status_code == 404: + raise WeKnoraError("WeKnora Wiki image was not found", status_code=404) + if response.status_code in {401, 403}: + if self._has_admin_credentials: + raise WeKnoraError("WeKnora rejected DeerFlow's authenticated Wiki image request", status_code=502) + raise WeKnoraError( + "WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy", + status_code=502, + ) + if response.status_code >= 400: + raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502) + media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip() + return response.content, media_type or "application/octet-stream" + async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]: value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse")) return dict(value) if isinstance(value, dict) else {} diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py index f4b6b8ef..ca654111 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py @@ -15,6 +15,21 @@ from deerflow.integrations.weknora.local_index.vector_codec import normalize_vec logger = logging.getLogger(__name__) +def _upstream_error_detail(response: httpx.Response) -> str: + """Return a bounded upstream reason without hiding the useful response body.""" + + try: + payload = response.json() + if isinstance(payload, dict): + detail = payload.get("detail") or payload.get("message") or payload.get("error") or payload + else: + detail = payload + value = str(detail) + except (ValueError, TypeError): + value = response.text + return " ".join(value.split())[:2000] or "no response body" + + class WikiEmbeddingError(RuntimeError): def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None: super().__init__(message) @@ -76,11 +91,15 @@ class StrictWikiEmbeddingClient: started = monotonic() try: response = await (await self._get_client()).post(url, headers=headers, json=payload) - if response.status_code in {401, 403}: - raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False) - if response.status_code == 429 or response.status_code >= 500: - raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True) - response.raise_for_status() + if response.status_code >= 400: + detail = _upstream_error_detail(response) + category = "authentication" if response.status_code in {401, 403} else "upstream" + retryable = response.status_code == 429 or response.status_code >= 500 + raise WikiEmbeddingError( + f"Embedding service returned HTTP {response.status_code}: {detail}", + category=category, + retryable=retryable, + ) data = response.json() items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else [] if len(items) != len(texts): @@ -98,8 +117,20 @@ class StrictWikiEmbeddingClient: except httpx.TimeoutException: last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True) except (httpx.HTTPError, ValueError, TypeError) as exc: - last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError)) + reason = " ".join(str(exc).split())[:2000] or type(exc).__name__ + last_error = WikiEmbeddingError( + f"Embedding request failed ({type(exc).__name__}): {reason}", + category="request", + retryable=isinstance(exc, httpx.HTTPError), + ) if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries): + logger.error( + "Wiki embedding batch failed model=%s batch=%d category=%s reason=%s", + self.config.model, + len(texts), + last_error.category, + last_error, + ) raise last_error await asyncio.sleep(min(30.0, 0.5 * (2**attempt))) raise last_error or WikiEmbeddingError("Embedding request failed") diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py index 71f2b522..fb8da3da 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py @@ -6,8 +6,12 @@ from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWiki from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore -def make_llmwiki_index_store(session_factory): - return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory) +def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50): + return ( + MemoryLlmWikiIndexStore() + if session_factory is None + else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size) + ) __all__ = [ diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py index c9d87b27..75f9ff68 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py @@ -6,7 +6,7 @@ from datetime import UTC, datetime, timedelta from typing import Any from uuid import uuid4 -from sqlalchemy import case, delete, func, or_, select, update +from sqlalchemy import case, delete, func, insert, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -27,8 +27,9 @@ def _dict(row) -> dict[str, Any]: class SqlLlmWikiIndexStore(LlmWikiIndexStore): - def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: + def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None: self._sf = session_factory + self.write_batch_size = max(1, min(int(write_batch_size), 500)) async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow: row = await session.get(LlmWikiWikiSyncStateRow, mapping_id) @@ -82,24 +83,29 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore): row.updated_at = now await session.flush() await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id)) - session.add_all( - [ - LlmWikiWikiVectorRow( - id=str(uuid4()), - page_id=row.id, - knowledge_base_mapping_id=mapping_id, - section_index=int(item["section_index"]), - heading=item.get("heading"), - section_content=str(item["section_content"]), - content_hash=str(item["content_hash"]), - embedding_fingerprint=fingerprint, - embedding_dimensions=int(item["embedding_dimensions"]), - vector_blob=bytes(item["vector_blob"]), - created_at=now, - ) - for item in vectors - ] - ) + vector_rows = [ + { + "id": str(uuid4()), + "page_id": row.id, + "knowledge_base_mapping_id": mapping_id, + "section_index": int(item["section_index"]), + "heading": item.get("heading"), + "section_content": str(item["section_content"]), + "content_hash": str(item["content_hash"]), + "embedding_fingerprint": fingerprint, + "embedding_dimensions": int(item["embedding_dimensions"]), + "vector_blob": bytes(item["vector_blob"]), + "created_at": now, + } + for item in vectors + ] + # Keep the page replacement atomic, but cap each SQL statement so + # large Wiki pages cannot flood the database with one huge insert. + for start in range(0, len(vector_rows), self.write_batch_size): + await session.execute( + insert(LlmWikiWikiVectorRow), + vector_rows[start : start + self.write_batch_size], + ) state = await self._ensure_state(session, mapping_id) state.index_revision = int(state.index_revision or 0) + 1 state.embedding_fingerprint = fingerprint diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py b/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py index d33a1309..ad93a117 100644 --- a/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py +++ b/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py @@ -40,20 +40,31 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t ) await session.commit() - store = SqlLlmWikiIndexStore(session_factory) + store = SqlLlmWikiIndexStore(session_factory, write_batch_size=1) assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp") assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp") page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"}) blob, dimensions = encode_vector([3.0, 4.0], dimensions=2) saved = await store.replace_page_vectors( page=page, - vectors=[{"section_index": 0, "heading": "History", "section_content": "Alice founded Acme.", "content_hash": "section", "embedding_dimensions": dimensions, "vector_blob": blob}], + vectors=[ + { + "section_index": index, + "heading": "History", + "section_content": f"Alice founded Acme. Part {index}", + "content_hash": f"section-{index}", + "embedding_dimensions": dimensions, + "vector_blob": blob, + } + for index in range(3) + ], fingerprint="fp", sync_id="sync-1", ) snapshot = await store.load_vector_snapshot("mapping-1", "fp") assert saved["slug"] == "people/alice" - assert len(snapshot["rows"]) == 1 + assert store.write_batch_size == 1 + assert len(snapshot["rows"]) == 3 assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8]) assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1 diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py b/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py index 667a1dfe..b5c3ee1f 100644 --- a/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py +++ b/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py @@ -690,6 +690,24 @@ async def test_weknora_client_proxies_original_document_preview_without_credenti assert media_type == "text/plain" +@pytest.mark.asyncio +async def test_weknora_client_fetches_protected_wiki_image() -> None: + async def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v1/knowledge-bases/kb-one/files" + assert request.url.params["file_path"] == "resource://wiki/figure.png" + return httpx.Response(200, content=b"png-bytes", headers={"Content-Type": "image/png"}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client: + client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client) + content, media_type = await client.get_knowledge_base_file( + "kb-one", + "resource://wiki/figure.png", + ) + + assert content == b"png-bytes" + assert media_type == "image/png" + + @pytest.mark.asyncio async def test_memory_store_enforces_owner_and_public_visibility() -> None: store = MemoryLlmWikiStore() @@ -1358,6 +1376,10 @@ def test_weknora_iframe_html_rewrites_public_prefix_paths() -> None: assert '"allowedWeKnoraIds":["remote-kb","other-visible-kb"]' in injected assert ".breadcrumb-link.dropdown" not in injected assert "reloadForKnowledgeBaseSwitch" in injected + assert "function protectedImageApiUrl" in injected + assert 'img[data-protected-src]' in injected + assert 'encodeURIComponent(cfg.weknoraId)' in injected + assert 'credentials: "same-origin"' in injected def test_weknora_iframe_filters_kb_list_to_signed_visible_mappings() -> None: diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py b/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py index f4e2f725..a93a573f 100644 --- a/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py +++ b/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py @@ -1,10 +1,13 @@ from __future__ import annotations +import asyncio import json +from types import SimpleNamespace import httpx import pytest +from app.gateway import llmwiki_index_scheduler from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient from deerflow.integrations.weknora.local_index.sync import WikiSyncService @@ -85,6 +88,24 @@ async def test_configured_dimensions_are_validated_but_not_sent_to_bge() -> None assert requests == [{"model": "bge-embedding-m3", "input": ["Wiki page"]}] +@pytest.mark.asyncio +async def test_embedding_error_includes_upstream_reason(caplog) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(400, json={"detail": "unsupported request parameter: dimensions"}) + + config = LocalWikiEmbeddingConfig( + model="bge-embedding-m3", + base_url="http://embedding.test/v1", + max_retries=0, + ) + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client: + client = StrictWikiEmbeddingClient(config, http_client=http_client) + with pytest.raises(RuntimeError, match="unsupported request parameter: dimensions"): + await client.embed_query("Wiki page") + + assert "unsupported request parameter: dimensions" in caplog.text + + @pytest.mark.asyncio async def test_omitted_dimensions_are_discovered_from_first_response() -> None: def handler(request: httpx.Request) -> httpx.Response: @@ -167,3 +188,39 @@ async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> Non assert store.states["kb"]["state"] == "idle" assert store.states["kb"]["embedding_fingerprint"] == "fp-new" assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"} + + +@pytest.mark.asyncio +async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None: + calls: list[str] = [] + + async def fake_sleep(seconds: float) -> None: + calls.append(f"sleep:{seconds}") + raise asyncio.CancelledError + + async def fake_sync_once(app) -> None: + calls.append("sync") + + monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep) + monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once) + app = SimpleNamespace( + state=SimpleNamespace( + config=SimpleNamespace( + llmwiki=SimpleNamespace( + local_wiki_index=SimpleNamespace(sync_interval_seconds=120), + ) + ) + ) + ) + + with pytest.raises(asyncio.CancelledError): + await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app) + + assert calls == ["sleep:120"] + + +def test_local_wiki_index_accepts_legacy_sync_flag() -> None: + config = LocalWikiIndexConfig.model_validate({"sync_enabled": True}) + + assert config.auto_sync is True + assert config.sync_enabled is True diff --git a/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py b/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py index 10c63e8b..944b94c2 100644 --- a/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py +++ b/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py @@ -177,6 +177,37 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None: assert payload["pages"][0]["aliases"] == [] +@pytest.mark.asyncio +async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None: + store = PublicStore(_published_row()) + + class Client: + async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str): + assert knowledge_base_id == "remote-one" + assert file_path == "storage://tenant/resource://wiki/figure.png" + return b"png-bytes", "image/png" + + monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client()) + + response = await llmwiki.get_public_wiki_file( + _request(store), + "kb-one", + "storage://tenant/resource://wiki/figure.png", + ) + + assert store.calls == [("kb-one", "__public_wiki__", False, False)] + assert response.body == b"png-bytes" + assert response.media_type == "image/png" + assert response.headers["cache-control"] == "public, max-age=300" + + +def test_wiki_file_proxy_rejects_non_weknora_urls() -> None: + with pytest.raises(HTTPException) as exc_info: + llmwiki._validate_protected_wiki_file_path("https://example.test/private.png") + + assert exc_info.value.status_code == 422 + + @pytest.mark.asyncio async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None: store = PublicStore(_published_row()) From 9f29ecae9e4f9fc3f43a857e17d2d5be40b2c867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=89=E9=91=AB=E5=8D=87?= Date: Tue, 25 Aug 2026 16:57:42 +0800 Subject: [PATCH 3/4] fix: harden wiki delivery and leaderboard performance --- frontend-web/src/core/llmwiki/source-links.ts | 19 ++++-- .../src/strategy-components/api/llmwiki.ts | 43 ++++++++++--- .../llmwiki/WeKnoraWikiManagementPage.tsx | 40 +++++++++++- .../llmwiki/WeKnoraWikiPageView.tsx | 11 ++-- .../llmwiki/WikiProtectedImage.tsx | 21 +++---- offline-backend-20260512/backend/CLAUDE.md | 27 +++++++- offline-backend-20260512/backend/README.md | 31 +++++++++- .../app/gateway/routers/admin_users.py | 38 ++++++++---- .../backend/app/gateway/routers/llmwiki.py | 56 ++++++++++++++++- .../app/gateway/routers/llmwiki_index.py | 36 +++++++++++ .../backend/app/gateway/weknora_embed.py | 20 ++++++ .../integrations/weknora/local_index/sync.py | 7 +++ .../deerflow/persistence/llm_metrics/model.py | 1 + .../persistence/llmwiki_index/base.py | 13 ++++ .../persistence/llmwiki_index/memory.py | 42 +++++++++++++ .../deerflow/persistence/llmwiki_index/sql.py | 62 +++++++++++++++++++ .../versions/20260821_01_fixed_questions.py | 8 +-- ...20260821_02_fixed_question_stream_speed.py | 8 +-- .../20260823_01_llmwiki_local_wiki_index.py | 4 +- ...erge_report_structure_and_llmwiki_heads.py | 34 ++++++++++ .../harness/deerflow/persistence/run/model.py | 2 + .../persistence/tool_metrics/model.py | 2 + .../scripts/sql/leaderboard_indexes_mysql.sql | 23 +++++-- .../test_admin_leaderboard_query_shape.py | 43 +++++++++++++ .../test_llmwiki_local_index_repository.py | 16 +++++ .../backend/tests/test_llmwiki_wiki_sync.py | 10 ++- .../backend/tests/test_migration_graph.py | 17 +++++ .../tests/test_public_llmwiki_embed.py | 50 +++++++++++++++ 28 files changed, 616 insertions(+), 68 deletions(-) create mode 100644 offline-backend-20260512/backend/packages/harness/deerflow/persistence/migrations/versions/20260825_01_merge_report_structure_and_llmwiki_heads.py create mode 100644 offline-backend-20260512/backend/tests/test_admin_leaderboard_query_shape.py create mode 100644 offline-backend-20260512/backend/tests/test_migration_graph.py diff --git a/frontend-web/src/core/llmwiki/source-links.ts b/frontend-web/src/core/llmwiki/source-links.ts index 60f7f677..10bc99ee 100644 --- a/frontend-web/src/core/llmwiki/source-links.ts +++ b/frontend-web/src/core/llmwiki/source-links.ts @@ -27,6 +27,16 @@ export type KnowledgeSourceLike = { slug?: string | null; }; +/** Build a HashRouter URL without dropping a reverse-proxy path such as /web/. */ +export function buildAppHashUrl(path: string): string { + const route = path.startsWith("/") ? path : `/${path}`; + if (typeof window !== "undefined") { + return `${window.location.pathname || "/"}#${route}`; + } + const base = import.meta.env.BASE_URL || "/"; + return `${base}#${route}`; +} + export function parseWeKnoraPreviewUrl(url?: string | null): WeKnoraSourceTarget | null { if (!url?.startsWith("/weknora-source-preview?")) { return null; @@ -92,7 +102,7 @@ export function buildWeKnoraSourceIframeUrl(target: WeKnoraSourceTarget): string if (target.documentId) params.set("documentId", target.documentId); if (target.wikiSlug) params.set("wikiSlug", target.wikiSlug); const query = params.toString(); - return `/#/page/workspace/knowledge/${encodeURIComponent(target.knowledgeBaseId)}${query ? `?${query}` : ""}`; + return buildAppHashUrl(`/page/workspace/knowledge/${encodeURIComponent(target.knowledgeBaseId)}${query ? `?${query}` : ""}`); } /** @@ -103,7 +113,7 @@ export function buildWeKnoraWikiPageUrl( knowledgeBaseId: string, wikiSlug: string, ): string { - return `/#${buildWeKnoraWikiPagePath(knowledgeBaseId, wikiSlug)}`; + return buildAppHashUrl(buildWeKnoraWikiPagePath(knowledgeBaseId, wikiSlug)); } export function buildWeKnoraWikiPagePath( @@ -129,7 +139,7 @@ export function buildPublicKnowledgeWikiUrl( knowledgeBaseId: string, wikiSlug?: string, ): string { - return `/#${buildPublicKnowledgeWikiPath(knowledgeBaseId, wikiSlug)}`; + return buildAppHashUrl(buildPublicKnowledgeWikiPath(knowledgeBaseId, wikiSlug)); } export function buildLegacyKnowledgeSourceUrl(source: KnowledgeSourceLike): string { @@ -160,6 +170,7 @@ export function normalizeMalformedWeKnoraHashRoute(): boolean { const { pathname, search, hash } = window.location; if (!pathname.startsWith("/page/workspace/knowledge/")) return false; if (hash && !hash.startsWith("#/")) return false; - window.location.replace(`/#${pathname}${search}`); + const base = import.meta.env.BASE_URL || "/"; + window.location.replace(`${base}#${pathname}${search}`); return true; } diff --git a/frontend-web/src/strategy-components/api/llmwiki.ts b/frontend-web/src/strategy-components/api/llmwiki.ts index 5b8f16c9..40f0fd38 100644 --- a/frontend-web/src/strategy-components/api/llmwiki.ts +++ b/frontend-web/src/strategy-components/api/llmwiki.ts @@ -179,6 +179,28 @@ export interface LlmWikiWikiIndexStatus { last_error?: string | null; last_started_at?: string | null; last_completed_at?: string | null; + progress?: { + total_pages: number; + processed_pages: number; + vectorized_pages: number; + failed_pages: number; + vector_count: number; + completion_percent: number; + }; + pages?: Array<{ + id: string; + slug: string; + title: string; + page_type: string; + status: string; + index_status: string; + vector_count: number; + vectorized: boolean; + processed_in_current_sync: boolean; + indexed_at?: string | null; + updated_at?: string | null; + last_error?: string | null; + }>; } export interface LlmWikiSourceContext { @@ -444,21 +466,22 @@ export async function getLlmWikiDocumentPreview( return response.blob(); } -export async function getLlmWikiProtectedFile( +export async function getLlmWikiProtectedFileUrl( knowledgeBaseId: string, filePath: string, publicAccess = false, -): Promise { - const base = publicAccess ? PUBLIC_API() : API(); +): Promise { const params = new URLSearchParams({ file_path: filePath }); - const response = await apiFetch( - `${base}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/files?${params.toString()}`, - ); - if (!response.ok) { - await readJson(response, "读取 Wiki 图片"); - throw new Error("读取 Wiki 图片失败"); + if (publicAccess) { + return `${PUBLIC_API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/files?${params.toString()}`; } - return response.blob(); + const access = await readJson<{ token: string; expires_in: number }>( + await apiFetch( + `${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/files/access-url?${params.toString()}`, + ), + "读取 Wiki 图片", + ); + return `${PUBLIC_API()}/files/${encodeURIComponent(access.token)}`; } export async function getLlmWikiGraph(knowledgeBaseId: string): Promise { diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage.tsx index 3eddeb72..ec06241b 100644 --- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage.tsx +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage.tsx @@ -6,6 +6,7 @@ import { toast } from "sonner"; import { Button } from "@/components/ui/button"; import { Badge } from "@/components/ui/badge"; import { Input } from "@/components/ui/input"; +import { Progress } from "@/components/ui/progress"; import { Textarea } from "@/components/ui/textarea"; import { MarkdownCodeEditor } from "@/components/wiki/MarkdownCodeEditor"; import { @@ -331,6 +332,10 @@ export function WeKnoraWikiManagementPage() { () => moveTargetOptions(pagesList), [pagesList], ); + const pageIndexBySlug = useMemo( + () => new Map((indexStatus.data?.pages ?? []).map((item) => [item.slug, item])), + [indexStatus.data?.pages], + ); const currentMoveTarget = moveTargetValue(draft.parentSlug, draft.categoryPathText, draft.folderId); useEffect(() => { @@ -387,6 +392,17 @@ export function WeKnoraWikiManagementPage() { }); }; + const renderVectorBadge = (page: WikiPageListItem) => { + const item = pageIndexBySlug.get(page.slug); + if (item?.last_error || item?.index_status === "failed" || item?.index_status === "stale") { + return 失败; + } + if (item?.vectorized) { + return 已向量化 · {item.vector_count} 段; + } + return 未向量化; + }; + const renderWikiNode = (node: WikiTreeNode, depth = 0) => { const hasChildren = node.children.length > 0; const isExpanded = query.trim() ? true : expandedDirectories.has(node.key); @@ -432,7 +448,8 @@ export function WeKnoraWikiManagementPage() { >
- {page.title || page.slug} + {page.title || page.slug} + {renderVectorBadge(page)}
{page.slug}
@@ -460,6 +477,7 @@ export function WeKnoraWikiManagementPage() {
{page.title || page.slug} + {renderVectorBadge(page)}
{page.slug}
{page.aliases?.length ? ( @@ -634,6 +652,26 @@ export function WeKnoraWikiManagementPage() {
+ + ); +} diff --git a/frontend-web/src/components/workspace/workspace-nav-chat-list.tsx b/frontend-web/src/components/workspace/workspace-nav-chat-list.tsx index a8e483aa..d3a26b04 100644 --- a/frontend-web/src/components/workspace/workspace-nav-chat-list.tsx +++ b/frontend-web/src/components/workspace/workspace-nav-chat-list.tsx @@ -135,6 +135,7 @@ export type EffectiveWorkspaceCompactNavItem = CompactMenuItem & WorkspaceCompac export function useWorkspaceCompactNavItems(): { sidebarItems: EffectiveWorkspaceCompactNavItem[]; settingsItems: EffectiveWorkspaceCompactNavItem[]; + toolboxItems: EffectiveWorkspaceCompactNavItem[]; } { const { data: overrides = [] } = useCompactMenuOverrides(); return useMemo(() => { @@ -142,6 +143,7 @@ export function useWorkspaceCompactNavItems(): { return { sidebarItems: applied.sidebarItems as EffectiveWorkspaceCompactNavItem[], settingsItems: applied.settingsItems as EffectiveWorkspaceCompactNavItem[], + toolboxItems: applied.toolboxItems as EffectiveWorkspaceCompactNavItem[], }; }, [overrides]); } diff --git a/frontend-web/src/core/page-layout/compact-menu-overrides.test.ts b/frontend-web/src/core/page-layout/compact-menu-overrides.test.ts index c67df41c..c325b252 100644 --- a/frontend-web/src/core/page-layout/compact-menu-overrides.test.ts +++ b/frontend-web/src/core/page-layout/compact-menu-overrides.test.ts @@ -18,13 +18,15 @@ test("normalizeCompactMenuType defaults to A", () => { assert.equal(isDefaultCompactMenuType("B"), false); }); -test("applyCompactMenuOverrides splits sidebar vs settings", () => { +test("applyCompactMenuOverrides splits sidebar, settings, and toolbox", () => { const items: MenuItem[] = [ { id: "a", label: "对话", icon: I, path: "/a" }, { id: "b", label: "智能体", icon: I, path: "/b" }, + { id: "c", label: "技能", icon: I, path: "/c" }, ]; const applied = applyCompactMenuOverrides(items, [ { nodeId: "b", placement: "settings", sortOrder: 0, label: "智能体设置" }, + { nodeId: "c", placement: "toolbox", sortOrder: 0, label: "技能工具" }, ]); assert.deepEqual( applied.sidebarItems.map((item) => item.id), @@ -32,4 +34,6 @@ test("applyCompactMenuOverrides splits sidebar vs settings", () => { ); assert.equal(applied.settingsItems[0]?.id, "b"); assert.equal(applied.settingsItems[0]?.label, "智能体设置"); + assert.equal(applied.toolboxItems[0]?.id, "c"); + assert.equal(applied.toolboxItems[0]?.label, "技能工具"); }); diff --git a/frontend-web/src/core/page-layout/compact-menu-overrides.ts b/frontend-web/src/core/page-layout/compact-menu-overrides.ts index 30cc4869..020373c6 100644 --- a/frontend-web/src/core/page-layout/compact-menu-overrides.ts +++ b/frontend-web/src/core/page-layout/compact-menu-overrides.ts @@ -1,7 +1,7 @@ import type { MenuItem } from "./sidebar-menu"; import { readRuntimeEnv } from "@/env"; -export type CompactMenuPlacement = "sidebar" | "settings"; +export type CompactMenuPlacement = "sidebar" | "settings" | "toolbox"; export const COMPACT_MENU_TYPES = ["A", "B"] as const; export type CompactMenuType = (typeof COMPACT_MENU_TYPES)[number]; @@ -64,7 +64,11 @@ function isNavigable(item: MenuItem): boolean { export function applyCompactMenuOverrides( items: MenuItem[], overrides: CompactMenuOverride[], -): { sidebarItems: CompactMenuItem[]; settingsItems: CompactMenuItem[] } { +): { + sidebarItems: CompactMenuItem[]; + settingsItems: CompactMenuItem[]; + toolboxItems: CompactMenuItem[]; +} { const byId = new Map(overrides.map((override) => [override.nodeId, override])); const enhanced = items.filter(isNavigable).map((item, index) => { const override = byId.get(item.id); @@ -82,6 +86,7 @@ export function applyCompactMenuOverrides( return { sidebarItems: sortItems(enhanced.filter((item) => item.compactPlacement === "sidebar")), settingsItems: sortItems(enhanced.filter((item) => item.compactPlacement === "settings")), + toolboxItems: sortItems(enhanced.filter((item) => item.compactPlacement === "toolbox")), }; } diff --git a/frontend-web/src/pages/AgentChatPage.tsx b/frontend-web/src/pages/AgentChatPage.tsx index 4487813a..f57b6b52 100644 --- a/frontend-web/src/pages/AgentChatPage.tsx +++ b/frontend-web/src/pages/AgentChatPage.tsx @@ -810,7 +810,7 @@ export default function AgentChatPage({ agentName={agentDisplayName} disabled={thread.isLoading || isDemoMode} onAskQuestion={(text) => handleSubmit({ text, files: [] })} - defaultCollapsed={taskCtxOn} + defaultCollapsed showRecentChats={!taskCtxOn} compactAppearanceActions={compactAppearanceActions} /> diff --git a/frontend-web/src/pages/MenuManagementPage.tsx b/frontend-web/src/pages/MenuManagementPage.tsx index 63907982..dc55ee87 100644 --- a/frontend-web/src/pages/MenuManagementPage.tsx +++ b/frontend-web/src/pages/MenuManagementPage.tsx @@ -261,12 +261,17 @@ function compactRowsFromTree(tree: MenuItem[], overrides: CompactMenuOverride[]) } function normalizeCompactRows(rows: CompactRow[]): CompactRow[] { + const placementOrder: Record = { + sidebar: 0, + settings: 1, + toolbox: 2, + }; return rows .slice() .sort( (a, b) => a.topIndex - b.topIndex || - Number(a.placement === "settings") - Number(b.placement === "settings") || + placementOrder[a.placement] - placementOrder[b.placement] || a.order - b.order || a.id.localeCompare(b.id), ); @@ -1091,7 +1096,7 @@ function CompactMenuManagementPanel({ baseTree }: { baseTree: MenuItem[] }) {

- 这里配置的是简洁模式工作区左侧菜单,保存后仅对当前网页使用的 + 这里配置简洁模式菜单在左侧菜单、设置和更多、右侧工具箱之间的分布,保存后仅对当前网页使用的 {COMPACT_MENU_TYPE_LABELS[runtimeType]}生效(两套部署各改各的);全量模式菜单不受影响。

@@ -1132,6 +1137,7 @@ function CompactMenuManagementPanel({ baseTree }: { baseTree: MenuItem[] }) { 左侧菜单 设置和更多 + 右侧工具箱
diff --git a/frontend-web/src/pages/PageRoutes.tsx b/frontend-web/src/pages/PageRoutes.tsx index e6374f72..3901e75a 100644 --- a/frontend-web/src/pages/PageRoutes.tsx +++ b/frontend-web/src/pages/PageRoutes.tsx @@ -3,6 +3,7 @@ import { Navigate, Route, Routes } from "react-router-dom"; import { PageSidebar } from "@/components/page-sidebar"; import { PageSidebarDark } from "@/components/page-sidebar-dark"; +import { CompactToolbox } from "@/components/workspace/compact-toolbox"; import { isChatsIframeWorkspacePath } from "@/core/page-layout/chats-iframe-route"; import { readSidebarEnabled } from "@/core/page-layout/sidebar-enabled"; import { isWujieEnv } from "@/core/embed/wujie"; @@ -102,12 +103,19 @@ function PageLayout() { !embedActive && // `?hideChrome=1` 显式去 chrome:优先级高于上面对 dashboard-agent 的强制全局 chrome。 !chromeHidden; + const compactToolboxAllowed = + !chromeByPreference && + !forceChromeForDashboardAgent && + !embedActive && + !chromeHidden && + !isChatsIframeWorkspacePath(location.pathname); // h-full(100%)而非 h-screen(100vh):配合 globals.css 的 html/body/#root height:100% // 高度链,根布局跟随容器高度。全屏下 100% 解析到视口,与 h-screen 等价;无界等嵌套 // 下容器不占满整屏时,高度才能正确收缩而非锁死整屏。 return ( + {compactToolboxAllowed && }
{pageSidebarAllowed && !headerHidden &&
}
diff --git a/frontend-web/src/strategy-components/api/compact-menu-overrides.ts b/frontend-web/src/strategy-components/api/compact-menu-overrides.ts index dfa8dd43..9b6bbc45 100644 --- a/frontend-web/src/strategy-components/api/compact-menu-overrides.ts +++ b/frontend-web/src/strategy-components/api/compact-menu-overrides.ts @@ -14,16 +14,20 @@ const BASE = () => `${getBackendBaseURL()}/api/compact-menu-overrides`; interface ServerCompactOverride { node_id: string; - placement: "sidebar" | "settings"; + placement: "sidebar" | "settings" | "toolbox"; sort_order: number; label: string | null; menu_type?: string | null; } function toOverride(s: ServerCompactOverride): CompactMenuOverride { + const placement = + s.placement === "settings" || s.placement === "toolbox" + ? s.placement + : "sidebar"; return { nodeId: s.node_id, - placement: s.placement === "settings" ? "settings" : "sidebar", + placement, sortOrder: s.sort_order ?? 0, label: s.label ?? null, }; @@ -96,8 +100,11 @@ export function useCompactMenuOverrides(menuType?: CompactMenuType) { return useQuery({ queryKey: [COMPACT_MENU_OVERRIDES_KEY, type], queryFn: () => listCompactMenuOverrides(type), - refetchOnWindowFocus: false, - staleTime: 60_000, + // 菜单管理常在另一个标签页操作。简洁模式页面重新挂载或获得焦点时必须 + // 立即读取最新配置,否则新放入右侧工具箱的项目会被旧缓存隐藏一分钟。 + refetchOnMount: "always", + refetchOnWindowFocus: "always", + staleTime: 0, }); } diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index 2d0b1f23..439b41e9 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -1024,10 +1024,14 @@ Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` me The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. Do not add an HTML `sandbox` attribute to this trusted, permission-filtered detail iframe: a sandboxed ancestor causes Chromium's built-in Blob PDF viewer to render the browser-blocked page. The Gateway follows upstream redirects only for `/knowledge/{id}/preview` and `/knowledge-bases/{id}/files`, keeping PDF bytes and protected Wiki images inside the signed proxy instead of sending the browser to WeKnora object storage. Native/private and public Wiki readers must rewrite protected Markdown image -sources to their mapping-scoped DeerFlow `/files?file_path=` endpoints and load -them as Blob URLs. The embedded guard also rewrites protected fetch/XHR values -and hydrates late `img[data-protected-src]` placeholders; never expose the -service token or relax the current-KB authorization to make images work. +sources to their mapping-scoped DeerFlow `/files?file_path=` endpoints and keep +real HTTP URLs on rendered image elements. The embedded guard also rewrites +protected fetch/XHR values and hydrates late `img[data-protected-src]` +placeholders with the signed iframe HTTP proxy URL. Observe both +`data-protected-src` and `src`, because WeKnora may replace the URL after first +paint. Never create/revoke a temporary object URL here: its lightbox reuses the +thumbnail `src` after click. Never expose the service token or relax the +current-KB authorization to make images work. **Embedded native knowledge-base switching:** The original WeKnora breadcrumb picker remains available inside the detail iframe. A signed 15-minute embed cookie carries only the current DeerFlow user's visible mappings (their own plus published bases); the proxied `/api/v1/knowledge-bases` response is filtered to that allowlist. Choosing another entry performs a full iframe navigation, rechecks the DeerFlow mapping and read/write permission, then reissues the cookie for the selected base. Never expose the upstream service-admin list directly or permit an arbitrary remote knowledge-base ID. @@ -1509,7 +1513,10 @@ authenticated readers obtain a short-lived signed capability from `/files/access-url` and load `/api/public/llmwiki/files/{token}`. The signed payload is restricted to `kind=wiki_file`, mapping id, remote id, validated protected path, and expiry, and responses provide an inline UTF-8 filename for -new-tab viewing and browser Save As. +new-tab viewing and browser Save As. The proxied WeKnora detail iframe must +likewise leave its scoped HTTP file-proxy URL on every protected image and +restore it if upstream rendering replaces `src`; its click preview reads that +same DOM value, so a fetched-and-revoked object URL is not acceptable. Third-party pages embed published Wiki content through the chrome-free route `/#/embed/knowledge/{mapping_id}` (optional exact-page `wikiSlug`). It defaults @@ -1558,11 +1565,26 @@ Internal search authorization always starts from DeerFlow mapping ids and is rechecked by the Gateway. External endpoints use the isolated `/api/external/llmwiki/wiki` namespace and `X-API-Key`; require published + external-enabled + index-enabled on every request and response, and hard-deny -the `system` / `对话沉淀` mapping. Never return remote WeKnora ids, provenance -refs, credentials, or vector bytes externally. Regression tests: +the `system` / `对话沉淀` mapping. Those API-key endpoints never return remote +WeKnora ids, provenance refs, credentials, or vector bytes. + +`POST /api/knowledge/vector-search` is a deliberately separate anonymous API. +It has no token/API-key requirement or rate limit, applies route-scoped +`Access-Control-Allow-Origin: *`, searches every published local Wiki index +regardless of `external_search_enabled`, and still hard-denies the `system` / +`对话沉淀` mapping before and after search. Its contract is section-level (one +result per matched vector, so Wiki-page fields and URLs may repeat), default +`top_k=10`, with optional `include_page_content`. Unlike the protected legacy +external namespace, this endpoint intentionally returns DeerFlow/WeKnora ids +and raw source/chunk provenance, but never credentials or vector bytes. Wiki +links use `llmwiki.local_wiki_index.frontend_base_url` (development default +`http://127.0.0.1:5174`) and point to the native +`/#/page/workspace/knowledge/{mapping_id}/wiki/view?wikiSlug=...` route. +Regression tests: `tests/test_llmwiki_local_index_repository.py`, `tests/test_llmwiki_wiki_sync.py`, `tests/test_llmwiki_vector_search.py`, -`tests/test_external_llmwiki_search.py`, and +`tests/test_external_llmwiki_search.py`, +`tests/test_public_knowledge_vector_search.py`, and `tests/test_llmwiki_wiki_retrieval.py`. The sandbox whole-report rewrite may carry an optional `reportOutline` Markdown diff --git a/offline-backend-20260512/backend/README.md b/offline-backend-20260512/backend/README.md index a83468ed..ba042b73 100644 --- a/offline-backend-20260512/backend/README.md +++ b/offline-backend-20260512/backend/README.md @@ -837,7 +837,10 @@ URLs. Public images use the published file proxy directly; private images first obtain a 24-hour signed capability and then load through `GET /api/public/llmwiki/files/{token}` with a real HTTP URL and inline filename, allowing zoom, new-tab viewing, and browser Save As without exposing -the user's bearer token. +the user's bearer token. The embedded WeKnora detail page follows the same +rule: late `img[data-protected-src]` placeholders keep their scoped HTTP proxy +URL on the DOM node, so WeKnora's click-to-enlarge viewer never receives an +already-revoked `blob:` URL. ## Leaderboard query and schema startup safety @@ -894,6 +897,21 @@ Authenticated operations are available below `/api/llmwiki/wiki-index` and `/api/external/llmwiki/wiki` require `X-API-Key`, are rate-limited, and only expose mappings that are both internally published and explicitly marked `external_search_enabled`; the system `对话沉淀` mapping is always denied. + +A separate browser-friendly endpoint, `POST /api/knowledge/vector-search`, is +fully anonymous: it requires neither a DeerFlow token nor `X-API-Key`, has no +rate limit, and responds with `Access-Control-Allow-Origin: *`. It searches the +local vectors of every published knowledge base regardless of +`external_search_enabled`, while still excluding the system `对话沉淀` base. +The minimal request is `{"query":"如何部署"}`; optional fields are `top_k` +(default `10`, maximum `100`) and `include_page_content` (default `false`). Each +matched vector section is returned as its own ranked result, including the +matched text, score, DeerFlow and WeKnora knowledge/Wiki ids, page metadata, +raw source/chunk references, best-effort document/chunk ids, and a directly +usable `wiki_url`. Set `llmwiki.local_wiki_index.frontend_base_url` to the +deployed frontend origin/base path; its development default is +`http://127.0.0.1:5174`. + See `docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md` for configuration, security boundaries, rollout, and API contracts. diff --git a/offline-backend-20260512/backend/app/gateway/app.py b/offline-backend-20260512/backend/app/gateway/app.py index a37b031f..8153a20f 100644 --- a/offline-backend-20260512/backend/app/gateway/app.py +++ b/offline-backend-20260512/backend/app/gateway/app.py @@ -24,8 +24,9 @@ from app.gateway.auth.disabled_mode import is_auth_disabled # noqa: E402 from app.gateway.auth_middleware import AuthMiddleware from app.gateway.config import get_gateway_config from app.gateway.csrf_middleware import CSRFMiddleware -from app.gateway.deps import langgraph_runtime -from app.gateway.routers import ( +from app.gateway.deps import langgraph_runtime +from app.gateway.public_knowledge_cors import PublicKnowledgeCORSMiddleware +from app.gateway.routers import ( ai_writing, admin_active_runs, admin_users, @@ -66,8 +67,9 @@ from app.gateway.routers import ( positions, position_roles, public_config, - public_embed, - public_leaderboard, + public_embed, + public_knowledge_vector_search, + public_leaderboard, public_scheduled_tasks, recommend, flow_extract, @@ -1473,11 +1475,16 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an "Set GATEWAY_CORS_ORIGINS or GATEWAY_CORS_ALLOW_ALL for deployment.", ", ".join(dev_origins), ) - app.add_middleware( - CORSMiddleware, - allow_origins=dev_origins, - **cors_kwargs, - ) + app.add_middleware( + CORSMiddleware, + allow_origins=dev_origins, + **cors_kwargs, + ) + + # This one anonymous API is intentionally callable by browser code from + # every origin. Keep the override route-scoped so authenticated APIs retain + # the deployment's normal CORS allow-list. + app.add_middleware(PublicKnowledgeCORSMiddleware) # Include routers # Models API is mounted at /api/models @@ -1626,8 +1633,11 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an app.include_router(public_config.router) app.include_router(html_page_favorites.router) - # Product knowledge base API is mounted at /api/knowledge - app.include_router(knowledge.router) + # Product knowledge base API is mounted at /api/knowledge + app.include_router(knowledge.router) + # Anonymous, section-level vector search over every published Wiki base + # except the system conversation-deposit base. + app.include_router(public_knowledge_vector_search.router) # External knowledge ingest (publish md artifacts) at /api/knowledge-ingest app.include_router(knowledge_ingest.router) diff --git a/offline-backend-20260512/backend/app/gateway/auth_middleware.py b/offline-backend-20260512/backend/app/gateway/auth_middleware.py index 604824de..eabd9bf5 100644 --- a/offline-backend-20260512/backend/app/gateway/auth_middleware.py +++ b/offline-backend-20260512/backend/app/gateway/auth_middleware.py @@ -68,9 +68,11 @@ _PUBLIC_EXACT_PATHS: frozenset[str] = frozenset( "/api/v1/auth/login/token", "/api/v1/auth/register", "/api/v1/auth/logout", - "/api/v1/auth/setup-status", - "/api/v1/auth/initialize", - # 并行多智能体面板专用的「绕过常规登录」取 token 端点(内网部署限制了标准登录时用)。 + "/api/v1/auth/setup-status", + "/api/v1/auth/initialize", + # Anonymous vector search over published local Wiki indexes. + "/api/knowledge/vector-search", + # 并行多智能体面板专用的「绕过常规登录」取 token 端点(内网部署限制了标准登录时用)。 "/api/parallel-agents/auth/token", } ) diff --git a/offline-backend-20260512/backend/app/gateway/csrf_middleware.py b/offline-backend-20260512/backend/app/gateway/csrf_middleware.py index 433f9dfb..8aec22a3 100644 --- a/offline-backend-20260512/backend/app/gateway/csrf_middleware.py +++ b/offline-backend-20260512/backend/app/gateway/csrf_middleware.py @@ -48,8 +48,10 @@ def should_check_csrf(request: Request) -> bool: path = app_relative_path(request) stripped = path.rstrip("/") # Exempt /api/v1/auth/me endpoint - if stripped == "/api/v1/auth/me": - return False + if stripped == "/api/v1/auth/me": + return False + if stripped == "/api/knowledge/vector-search": + return False # Anonymous public endpoints (mirror auth_middleware._PUBLIC_PATH_PREFIXES). # These intentionally have no cookie session, so the Double-Submit Cookie # pattern is both impossible (no token to compare) and unnecessary (no diff --git a/offline-backend-20260512/backend/app/gateway/public_knowledge_cors.py b/offline-backend-20260512/backend/app/gateway/public_knowledge_cors.py new file mode 100644 index 00000000..7b5e6c84 --- /dev/null +++ b/offline-backend-20260512/backend/app/gateway/public_knowledge_cors.py @@ -0,0 +1,49 @@ +"""Route-scoped permissive CORS for the anonymous knowledge vector API.""" + +from __future__ import annotations + +from collections.abc import Callable + +from fastapi import Request, Response +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.types import ASGIApp + +from app.gateway.proxy_path import app_relative_path + +PUBLIC_KNOWLEDGE_VECTOR_SEARCH_PATH = "/api/knowledge/vector-search" + + +def is_public_knowledge_vector_search(request: Request) -> bool: + return app_relative_path(request).rstrip("/") == PUBLIC_KNOWLEDGE_VECTOR_SEARCH_PATH + + +def _apply_public_cors(response: Response) -> Response: + response.headers["Access-Control-Allow-Origin"] = "*" + response.headers["Access-Control-Allow-Methods"] = "POST, OPTIONS" + response.headers["Access-Control-Allow-Headers"] = "*" + response.headers["Access-Control-Allow-Private-Network"] = "true" + response.headers["Access-Control-Max-Age"] = "600" + if "Access-Control-Allow-Credentials" in response.headers: + del response.headers["Access-Control-Allow-Credentials"] + return response + + +class PublicKnowledgeCORSMiddleware(BaseHTTPMiddleware): + """Allow every browser origin for one credential-free public route only.""" + + def __init__(self, app: ASGIApp) -> None: + super().__init__(app) + + async def dispatch(self, request: Request, call_next: Callable) -> Response: + if not is_public_knowledge_vector_search(request): + return await call_next(request) + if request.method.upper() == "OPTIONS": + return _apply_public_cors(Response(status_code=204)) + return _apply_public_cors(await call_next(request)) + + +__all__ = [ + "PUBLIC_KNOWLEDGE_VECTOR_SEARCH_PATH", + "PublicKnowledgeCORSMiddleware", + "is_public_knowledge_vector_search", +] diff --git a/offline-backend-20260512/backend/app/gateway/routers/compact_menu_overrides.py b/offline-backend-20260512/backend/app/gateway/routers/compact_menu_overrides.py index aa182d44..f73ee10c 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/compact_menu_overrides.py +++ b/offline-backend-20260512/backend/app/gateway/routers/compact_menu_overrides.py @@ -17,8 +17,8 @@ from pydantic import BaseModel, Field from app.gateway.deps import get_current_user, get_optional_user_from_request from deerflow.persistence.compact_menu_overrides import ( - CompactMenuOverrideStore, DEFAULT_COMPACT_MENU_TYPE, + CompactMenuOverrideStore, normalize_compact_menu_type, ) @@ -27,7 +27,7 @@ router = APIRouter(prefix="/api/compact-menu-overrides", tags=["compact-menu-ove class CompactMenuOverride(BaseModel): node_id: str = Field(..., min_length=1, max_length=128) - placement: Literal["sidebar", "settings"] = "sidebar" + placement: Literal["sidebar", "settings", "toolbox"] = "sidebar" sort_order: int = 0 label: str | None = Field(default=None, max_length=255) diff --git a/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py b/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py index dc719b71..488c26b5 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py +++ b/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py @@ -1151,35 +1151,15 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla if (!proxyUrl) return; if ( image.dataset.deerflowProtectedSource === protectedSrc && - String(image.getAttribute("src") || "").startsWith("blob:") + String(image.getAttribute("src") || "") === proxyUrl ) return; - const attempts = Number(image.dataset.deerflowProtectedAttempts || "0"); - if (attempts >= 3 || image.dataset.deerflowProtectedLoading === "1") return; - image.dataset.deerflowProtectedLoading = "1"; - image.dataset.deerflowProtectedAttempts = String(attempts + 1); - nativeFetch(proxyUrl, {{ credentials: "same-origin", headers: {{ Accept: "image/*" }} }}) - .then((response) => {{ - if (!response.ok) throw new Error(`protected image request failed (${{response.status}})`); - return response.blob(); - }}) - .then((blob) => {{ - const objectUrl = URL.createObjectURL(blob); - image.dataset.deerflowProtectedSource = protectedSrc; - image.removeAttribute("data-img-loading"); - image.removeAttribute("data-wiki-image-error"); - image.src = objectUrl; - image.addEventListener("load", () => URL.revokeObjectURL(objectUrl), {{ once: true }}); - }}) - .catch(() => {{ - image.dataset.wikiImageError = "true"; - window.setTimeout(() => {{ - delete image.dataset.deerflowProtectedLoading; - hydrateProtectedImage(image); - }}, 1200 * (attempts + 1)); - }}) - .finally(() => {{ - delete image.dataset.deerflowProtectedLoading; - }}); + // Keep the real HTTP proxy URL on the DOM node. WeKnora's preview/lightbox + // reuses the thumbnail's src when the user clicks it; a short-lived blob URL + // may render once and then become invalid before the preview opens. + image.dataset.deerflowProtectedSource = protectedSrc; + image.removeAttribute("data-img-loading"); + image.removeAttribute("data-wiki-image-error"); + image.src = proxyUrl; }} function hydrateProtectedImages(scope = document) {{ if (scope?.matches?.("img[data-protected-src]")) hydrateProtectedImage(scope); @@ -1197,7 +1177,7 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla childList: true, subtree: true, attributes: true, - attributeFilter: ["data-protected-src"] + attributeFilter: ["data-protected-src", "src"] }}); hydrateProtectedImages(); diff --git a/offline-backend-20260512/backend/app/gateway/routers/public_knowledge_vector_search.py b/offline-backend-20260512/backend/app/gateway/routers/public_knowledge_vector_search.py new file mode 100644 index 00000000..db254f3f --- /dev/null +++ b/offline-backend-20260512/backend/app/gateway/routers/public_knowledge_vector_search.py @@ -0,0 +1,152 @@ +"""Anonymous vector search over published, locally indexed WeKnora Wiki data.""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import quote, urlencode + +from fastapi import APIRouter, HTTPException, Request +from pydantic import BaseModel, Field + +from deerflow.integrations.weknora.local_index.search import WikiIndexError + +router = APIRouter(tags=["public-knowledge-vector-search"]) + + +class PublicKnowledgeVectorSearchRequest(BaseModel): + query: str = Field(..., min_length=1, max_length=10_000) + top_k: int = Field(default=10, ge=1, le=100) + include_page_content: bool = False + + +def _is_conversation_deposit(mapping: dict[str, Any]) -> bool: + return str(mapping.get("owner_user_id") or "") == "system" and str(mapping.get("name") or "").strip() == "对话沉淀" + + +async def _published_mappings(request: Request) -> list[dict[str, Any]]: + store = getattr(request.app.state, "llmwiki_store", None) + if store is None: + raise HTTPException(status_code=503, detail={"code": "LLMWIKI_STORE_UNAVAILABLE"}) + rows = await store.list_visible("system", scope="public", is_admin=False) + return [row for row in rows if row.get("publication_status") == "published" and not _is_conversation_deposit(row)] + + +def _unique_strings(values: list[str]) -> list[str]: + return list(dict.fromkeys(value for value in values if value)) + + +def _reference_metadata(item: dict[str, Any]) -> dict[str, Any]: + source_refs = item.get("source_refs") if isinstance(item.get("source_refs"), list) else [] + chunk_refs = item.get("chunk_refs") if isinstance(item.get("chunk_refs"), list) else [] + document_ids: list[str] = [] + document_names: list[str] = [] + chunk_ids: list[str] = [] + source_ref_ids: list[str] = [] + + for ref in source_refs: + if isinstance(ref, dict): + document_id = str(ref.get("document_id") or ref.get("knowledge_id") or ref.get("documentId") or ref.get("knowledgeId") or "").strip() + document_name = str( + ref.get("document_name") or ref.get("knowledge_title") or ref.get("documentName") or ref.get("knowledgeTitle") or ref.get("file_name") or ref.get("filename") or ref.get("title") or ref.get("name") or "" + ).strip() + chunk_id = str(ref.get("chunk_id") or ref.get("chunkId") or "").strip() + if document_id: + document_ids.append(document_id) + if document_name: + document_names.append(document_name) + if chunk_id: + chunk_ids.append(chunk_id) + elif isinstance(ref, (str, int)) and str(ref).strip(): + source_ref_ids.append(str(ref).strip()) + + for ref in chunk_refs: + if isinstance(ref, dict): + chunk_id = str(ref.get("chunk_id") or ref.get("chunkId") or ref.get("id") or "").strip() + else: + chunk_id = str(ref or "").strip() + if chunk_id: + chunk_ids.append(chunk_id) + + document_ids = _unique_strings(document_ids) + document_names = _unique_strings(document_names) + chunk_ids = _unique_strings(chunk_ids) + source_ref_ids = _unique_strings(source_ref_ids) + return { + "document_id": document_ids[0] if document_ids else None, + "document_name": document_names[0] if document_names else None, + "document_ids": document_ids, + "document_names": document_names, + "chunk_id": chunk_ids[0] if chunk_ids else None, + "chunk_ids": chunk_ids, + "source_ref_ids": source_ref_ids, + } + + +def _wiki_link(frontend_base_url: str, mapping_id: str, wiki_slug: str) -> tuple[str, str]: + query = urlencode({"wikiSlug": wiki_slug}) + route = f"/#/page/workspace/knowledge/{quote(mapping_id, safe='')}/wiki/view?{query}" + return route, f"{frontend_base_url.rstrip('/')}{route}" + + +@router.post("/api/knowledge/vector-search") +async def public_knowledge_vector_search( + request: Request, + body: PublicKnowledgeVectorSearchRequest, +) -> dict[str, Any]: + mappings = await _published_mappings(request) + search = getattr(request.app.state, "llmwiki_vector_search", None) + if search is None: + raise HTTPException(status_code=503, detail={"code": "WIKI_INDEX_UNAVAILABLE"}) + try: + payload = await search.search_sections( + body.query, + mappings, + top_k_sections=body.top_k, + external=True, + ) + except WikiIndexError as exc: + raise HTTPException( + status_code=exc.status_code, + detail={"code": exc.code, "message": str(exc)}, + ) from None + + # Publication can change while an embedding/search call is in flight. A + # second read ensures an unpublished or newly reserved result is never + # emitted by this anonymous endpoint. + still_published = {str(row["id"]) for row in await _published_mappings(request)} + frontend_base_url = request.app.state.config.llmwiki.local_wiki_index.frontend_base_url + results: list[dict[str, Any]] = [] + for raw_item in payload.get("results") or []: + if str(raw_item.get("knowledge_base_id") or "") not in still_published: + continue + item = dict(raw_item) + wiki_route, wiki_url = _wiki_link( + frontend_base_url, + str(item["knowledge_base_id"]), + str(item["wiki_slug"]), + ) + item.update(_reference_metadata(item)) + item["wiki_route"] = wiki_route + item["wiki_url"] = wiki_url + if not body.include_page_content: + item.pop("page_content", None) + results.append(item) + + return { + "query": body.query, + "top_k": body.top_k, + "mode": "vector", + "results": results, + "count": len(results), + "knowledge_base_count": payload.get("knowledge_base_count", len(mappings)), + "index_fingerprint": payload.get("index_fingerprint"), + "partial": bool(payload.get("partial", False)), + "warnings": payload.get("warnings") or [], + } + + +__all__ = [ + "PublicKnowledgeVectorSearchRequest", + "public_knowledge_vector_search", + "router", +] diff --git a/offline-backend-20260512/backend/config.yaml b/offline-backend-20260512/backend/config.yaml index 31606d1a..aa389625 100644 --- a/offline-backend-20260512/backend/config.yaml +++ b/offline-backend-20260512/backend/config.yaml @@ -561,6 +561,8 @@ llmwiki: # 且服务启动时只启动定时器,不会立刻全量向量化。 local_wiki_index: enabled: false + # 免登录 POST /api/knowledge/vector-search 返回的 Wiki 一键跳转链接前缀。 + frontend_base_url: "http://127.0.0.1:5174" auto_sync: false database_batch_size: 50 embedding: diff --git a/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md b/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md index 02ed80c8..38840a5c 100644 --- a/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md +++ b/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md @@ -1132,6 +1132,35 @@ Content-Type: application/json } ``` +#### 13.4.1 免鉴权浏览器向量搜索 + +本部署另行提供一个与上述 API-Key 命名空间隔离的新接口: + +```http +POST /api/knowledge/vector-search +Content-Type: application/json +``` + +最小请求只需检索词: + +```json +{"query": "任正非早期创业经历"} +``` + +可选参数只有 `top_k`(默认 `10`,最大 `100`)和 +`include_page_content`(默认 `false`)。接口不需要登录 Token 或 +`X-API-Key`,不做应用内限流,并针对这一条路由返回 +`Access-Control-Allow-Origin: *`。检索范围是全部 `published` 知识库, +不检查 `external_search_enabled`,但在检索前后都排除系统“对话沉淀”库。 + +响应按向量分段展开:同一 Wiki 页命中多个分段时会返回多条结果。每条结果 +包含命中文本和分数、DeerFlow/WeKnora 知识库与 Wiki 页 ID、页面元数据、 +原始 `source_refs`/`chunk_refs`、可提取到的文档/分块 ID,以及可直接打开的 +`wiki_url`。当 `include_page_content=true` 时额外返回完整 `page_content`。 +`wiki_url` 的前缀配置为 +`llmwiki.local_wiki_index.frontend_base_url`,开发默认值是 +`http://127.0.0.1:5174`。 + ### 13.5 外部页面详情 ```http @@ -1156,6 +1185,9 @@ X-API-Key: 如果为了少改 AuthMiddleware 而把接口挂到 `/api/public/`,路由自身也必须强制校验 API Key;但长期推荐独立 `/api/external/` 语义。 +上面的原则仍适用于通用服务间接口;13.4.1 是经过明确产品确认的例外,并通过 +精确路径白名单和路由级 CORS 隔离,不能据此放宽其他 API。 + ### 13.7 状态码约定 | 状态码 | 场景 | diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py index ae2427af..291a06d5 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py @@ -108,6 +108,10 @@ class LocalWikiIndexConfig(BaseModel): """DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages.""" enabled: bool = False + frontend_base_url: str = Field( + default="http://127.0.0.1:5174", + description="DeerFlow frontend base URL used to build directly usable Wiki links in public vector-search results.", + ) auto_sync: bool = False sync_interval_seconds: int = Field(default=120, ge=10, le=86400) sync_page_size: int = Field(default=100, ge=1, le=500) @@ -135,6 +139,14 @@ class LocalWikiIndexConfig(BaseModel): external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig) model_config = ConfigDict(extra="ignore") + @field_validator("frontend_base_url") + @classmethod + def _validate_frontend_base_url(cls, value: str) -> str: + normalized = normalize_service_url(value) + if not normalized: + raise ValueError("local Wiki index frontend_base_url is required") + return normalized + @model_validator(mode="before") @classmethod def _migrate_legacy_flags(cls, value): diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/search.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/search.py index 5e893b12..17261736 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/search.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/search.py @@ -46,29 +46,12 @@ class WikiVectorSearchService: raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400) if not mappings: return {"query": normalized_query, "mode": "vector", "results": [], "knowledge_base_count": 0, "index_fingerprint": self.embedding.fingerprint[:12], "partial": False, "warnings": []} - try: - query_vector = await self._embed_query(normalized_query) - except (WikiEmbeddingError, ValueError) as exc: - raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None - fingerprint = self.embedding.fingerprint - tasks = [self._search_mapping(normalized_query, query_vector, mapping, fingerprint, include_drafts=include_drafts, external=external) for mapping in mappings] - batches = await asyncio.gather(*tasks, return_exceptions=True) - candidates: list[dict[str, Any]] = [] - warnings: list[str] = [] - for mapping, batch in zip(mappings, batches, strict=True): - if isinstance(batch, Exception): - warnings.append(f"{mapping.get('id')}: {batch}") - else: - candidates.extend(batch) - if warnings and not candidates and len(warnings) == len(mappings): - if any("FINGERPRINT" in item for item in warnings): - code = "WIKI_INDEX_FINGERPRINT_MISMATCH" - elif any("rebuild" in item.lower() for item in warnings): - code = "WIKI_INDEX_REBUILDING" - else: - code = "WIKI_INDEX_NOT_READY" - raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409) - candidates.sort(key=lambda item: item["score"], reverse=True) + fingerprint, candidates, warnings = await self._collect_candidates( + normalized_query, + mappings, + include_drafts=include_drafts, + external=external, + ) grouped: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) for candidate in candidates: page = candidate["page"] @@ -111,6 +94,141 @@ class WikiVectorSearchService: limit = min(top_k_pages or self.config.top_k_pages, self.config.top_k_pages if top_k_pages is None else 50) return {"query": normalized_query, "mode": "vector", "results": results[:limit], "knowledge_base_count": len(mappings), "index_fingerprint": fingerprint[:12], "partial": bool(warnings), "warnings": warnings} + async def search_sections( + self, + query: str, + mappings: list[dict[str, Any]], + *, + top_k_sections: int = 10, + external: bool = False, + ) -> dict[str, Any]: + """Return one result per matching Wiki vector section. + + The regular ``search`` method intentionally groups several sections + under one Wiki page for citations. Public consumers of the standalone + vector API need the opposite contract: every matched vector is a + separate ranked item, while the complete page and knowledge-base + metadata remains attached to that item. + """ + + normalized_query = query.strip() + if not normalized_query: + raise WikiIndexError("WIKI_QUERY_EMPTY", "Search query is empty", status_code=400) + limit = max(1, min(int(top_k_sections), 100)) + if not mappings: + return { + "query": normalized_query, + "mode": "vector", + "results": [], + "knowledge_base_count": 0, + "index_fingerprint": self.embedding.fingerprint[:12], + "partial": False, + "warnings": [], + } + fingerprint, candidates, warnings = await self._collect_candidates( + normalized_query, + mappings, + include_drafts=False, + external=external, + section_limit=limit, + ) + results: list[dict[str, Any]] = [] + for candidate in candidates[:limit]: + page = candidate["page"] + mapping = candidate["mapping"] + results.append( + { + "kind": "wiki_section", + "id": candidate.get("id") or f"{page['id']}:{candidate['section_index']}", + "section_id": candidate.get("id") or f"{page['id']}:{candidate['section_index']}", + "section_index": int(candidate["section_index"]), + "heading": candidate.get("heading"), + "content": str(candidate.get("section_content") or ""), + "section_content_hash": candidate.get("content_hash") or "", + "score": float(candidate["score"]), + "knowledge_base_id": mapping["id"], + "weknora_knowledge_base_id": mapping.get("weknora_id") or "", + "knowledge_base_name": mapping.get("name") or "", + "knowledge_base_description": mapping.get("description") or "", + "knowledge_base_type": mapping.get("kb_type") or "document", + "publication_status": mapping.get("publication_status") or "", + "published_at": mapping.get("published_at"), + "wiki_page_id": page["id"], + "weknora_wiki_page_id": page.get("remote_page_id"), + "wiki_slug": page["slug"], + "title": page.get("title") or page["slug"], + "summary": page.get("summary") or "", + "page_content": page.get("content_md") or "", + "page_type": page.get("page_type") or "article", + "page_status": page.get("status") or "published", + "page_version": int(page.get("page_version") or 0), + "aliases": page.get("aliases") or [], + "category_path": page.get("category_path") or [], + "parent_slug": page.get("parent_slug"), + "wiki_path": page.get("wiki_path"), + "source_refs": page.get("source_refs") or [], + "chunk_refs": page.get("chunk_refs") or [], + "page_content_hash": page.get("content_hash") or "", + "remote_created_at": page.get("remote_created_at"), + "remote_updated_at": page.get("remote_updated_at"), + "indexed_at": page.get("indexed_at"), + } + ) + return { + "query": normalized_query, + "mode": "vector", + "results": results, + "knowledge_base_count": len(mappings), + "index_fingerprint": fingerprint[:12], + "partial": bool(warnings), + "warnings": warnings, + } + + async def _collect_candidates( + self, + query: str, + mappings: list[dict[str, Any]], + *, + include_drafts: bool, + external: bool, + section_limit: int | None = None, + ) -> tuple[str, list[dict[str, Any]], list[str]]: + try: + query_vector = await self._embed_query(query) + except (WikiEmbeddingError, ValueError) as exc: + raise WikiIndexError("WIKI_EMBEDDING_UNAVAILABLE", str(exc), status_code=503) from None + fingerprint = self.embedding.fingerprint + tasks = [ + self._search_mapping( + query, + query_vector, + mapping, + fingerprint, + include_drafts=include_drafts, + external=external, + section_limit=section_limit, + ) + for mapping in mappings + ] + batches = await asyncio.gather(*tasks, return_exceptions=True) + candidates: list[dict[str, Any]] = [] + warnings: list[str] = [] + for mapping, batch in zip(mappings, batches, strict=True): + if isinstance(batch, Exception): + warnings.append(f"{mapping.get('id')}: {batch}") + else: + candidates.extend(batch) + if warnings and not candidates and len(warnings) == len(mappings): + if any("FINGERPRINT" in item for item in warnings): + code = "WIKI_INDEX_FINGERPRINT_MISMATCH" + elif any("rebuild" in item.lower() for item in warnings): + code = "WIKI_INDEX_REBUILDING" + else: + code = "WIKI_INDEX_NOT_READY" + raise WikiIndexError(code, "Local Wiki index is not ready", status_code=409) + candidates.sort(key=lambda item: item["score"], reverse=True) + return fingerprint, candidates, warnings + async def _embed_query(self, query: str) -> np.ndarray: digest = hashlib.sha256(query.encode("utf-8")).hexdigest() key = f"{self.embedding.fingerprint}:{digest}" @@ -126,7 +244,17 @@ class WikiVectorSearchService: self._query_vectors.popitem(last=False) return vector - async def _search_mapping(self, query: str, vector: np.ndarray, mapping: dict[str, Any], fingerprint: str, *, include_drafts: bool, external: bool) -> list[dict[str, Any]]: + async def _search_mapping( + self, + query: str, + vector: np.ndarray, + mapping: dict[str, Any], + fingerprint: str, + *, + include_drafts: bool, + external: bool, + section_limit: int | None = None, + ) -> list[dict[str, Any]]: mapping_id = str(mapping["id"]) try: snapshot = await self.cache.get(mapping_id, fingerprint) @@ -148,7 +276,13 @@ class WikiVectorSearchService: ) if eligible.size == 0: return [] - indices, scores = await asyncio.to_thread(self._top_indices, snapshot, vector, eligible) + indices, scores = await asyncio.to_thread( + self._top_indices, + snapshot, + vector, + eligible, + section_limit, + ) terms = {term.lower() for term in re.findall(r"[\w\u4e00-\u9fff]{2,}", query)} candidates = [] for index, score in zip(indices, scores, strict=True): @@ -166,9 +300,15 @@ class WikiVectorSearchService: candidates.append({**row, "page": page, "mapping": mapping, "score": final_score}) return candidates - def _top_indices(self, snapshot: WikiVectorSnapshot, vector: np.ndarray, eligible: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + def _top_indices( + self, + snapshot: WikiVectorSnapshot, + vector: np.ndarray, + eligible: np.ndarray, + section_limit: int | None = None, + ) -> tuple[np.ndarray, np.ndarray]: scores = snapshot.matrix[eligible] @ vector - count = min(self.config.top_k_sections, scores.size) + count = min(section_limit or self.config.top_k_sections, scores.size) if count <= 0: return np.asarray([], dtype=int), np.asarray([], dtype=np.float32) positions = np.argpartition(scores, -count)[-count:] diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/compact_menu_overrides/memory.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/compact_menu_overrides/memory.py index 3b4898ca..20f26b76 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/compact_menu_overrides/memory.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/compact_menu_overrides/memory.py @@ -19,7 +19,7 @@ def _normalize(data: dict[str, Any], menu_type: str, updated_by: str | None) -> return { "node_id": storage_node_id(menu_type, node_id), "menu_type": menu_type, - "placement": placement if placement in {"sidebar", "settings"} else "sidebar", + "placement": placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar", "sort_order": int(data.get("sort_order", 0)), "label": (data.get("label") or None), "updated_by": updated_by, diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/compact_menu_overrides/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/compact_menu_overrides/sql.py index 8acc289f..77909132 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/compact_menu_overrides/sql.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/compact_menu_overrides/sql.py @@ -35,7 +35,7 @@ def _row_from(data: dict[str, Any], menu_type: str, updated_by: str | None) -> C return CompactMenuOverrideRow( node_id=storage_node_id(menu_type, node_id), menu_type=menu_type, - placement=placement if placement in {"sidebar", "settings"} else "sidebar", + placement=placement if placement in {"sidebar", "settings", "toolbox"} else "sidebar", sort_order=int(data.get("sort_order", 0)), label=(data.get("label") or None), updated_by=updated_by, diff --git a/offline-backend-20260512/backend/tests/test_compact_menu_overrides.py b/offline-backend-20260512/backend/tests/test_compact_menu_overrides.py index a9153e32..f3dae0e8 100644 --- a/offline-backend-20260512/backend/tests/test_compact_menu_overrides.py +++ b/offline-backend-20260512/backend/tests/test_compact_menu_overrides.py @@ -12,8 +12,8 @@ from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from deerflow.persistence.base import Base from deerflow.persistence.compact_menu_overrides import ( - MemoryCompactMenuOverrideStore, CompactMenuOverrideRepository, + MemoryCompactMenuOverrideStore, normalize_compact_menu_type, ) from deerflow.persistence.compact_menu_overrides.keys import public_node_id, storage_node_id @@ -53,15 +53,17 @@ async def test_replace_all_roundtrips(store: CompactMenuOverrideRepository): [ {"node_id": "workspace-chats", "placement": "settings", "sort_order": 1, "label": "会话"}, {"node_id": "workspace-agents", "placement": "sidebar", "sort_order": 0, "label": None}, + {"node_id": "workspace-skills", "placement": "toolbox", "sort_order": 0, "label": "技能工具"}, ], updated_by="admin-1", ) - assert len(out) == 2 + assert len(out) == 3 rows = {r["node_id"]: r for r in await store.list_overrides()} assert rows["workspace-chats"]["label"] == "会话" assert rows["workspace-chats"]["placement"] == "settings" assert rows["workspace-chats"]["menu_type"] == "A" assert rows["workspace-agents"]["updated_by"] == "admin-1" + assert rows["workspace-skills"]["placement"] == "toolbox" @pytest.mark.asyncio diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py b/offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py index b27cbbf5..a249c886 100644 --- a/offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py +++ b/offline-backend-20260512/backend/tests/test_llmwiki_vector_search.py @@ -56,6 +56,13 @@ async def test_exact_search_groups_sections_by_wiki_page_and_filters_drafts() -> with_drafts = await service.search("alice", [mapping], include_drafts=True) assert {item["wiki_slug"] for item in with_drafts["results"]} == {"alice", "secret"} + section_result = await service.search_sections("alice", [mapping], top_k_sections=2, external=True) + assert [item["content"] for item in section_result["results"]] == ["Alpha", "Beta"] + assert all(item["wiki_slug"] == "alice" for item in section_result["results"]) + assert all(item["kind"] == "wiki_section" for item in section_result["results"]) + assert section_result["results"][0]["wiki_page_id"] + assert "page_content" in section_result["results"][0] + narrow_service = WikiVectorSearchService( LocalWikiIndexConfig(min_similarity=0, top_k_sections=1, top_k_pages=1), FakeEmbedding(), diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py b/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py index b5c3ee1f..cfcf45af 100644 --- a/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py +++ b/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py @@ -1379,7 +1379,9 @@ def test_weknora_iframe_html_rewrites_public_prefix_paths() -> None: assert "function protectedImageApiUrl" in injected assert 'img[data-protected-src]' in injected assert 'encodeURIComponent(cfg.weknoraId)' in injected - assert 'credentials: "same-origin"' in injected + assert "image.src = proxyUrl;" in injected + assert "URL.createObjectURL(blob)" not in injected + assert 'attributeFilter: ["data-protected-src", "src"]' in injected def test_weknora_iframe_filters_kb_list_to_signed_visible_mappings() -> None: