fix: restore embedded wiki images and optimize vector sync
This commit is contained in:
1 parent
30943159bf
commit
0c5e2e8cb0
23 files changed
+609
-136
No files matched your search
@@ -37,6 +37,34 @@ const WIKI_INDEX_TYPE_LABELS: Record<string, string> = {
|
|||||||
|
|
||||||
const WIKI_INDEX_TYPE_ORDER = ["summary", "entity", "concept"];
|
const WIKI_INDEX_TYPE_ORDER = ["summary", "entity", "concept"];
|
||||||
|
|
||||||
|
export const WIKI_PROTECTED_IMAGE_MARKER = "/__llmwiki-protected-image__";
|
||||||
|
|
||||||
|
const PROTECTED_WIKI_IMAGE_SOURCE =
|
||||||
|
/(?:storage:\/\/[0-9A-Za-z_-]+\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\/\/[^\s"'<>\])]+/gi;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Browsers cannot load WeKnora's protected resource:// and object-storage
|
||||||
|
* schemes directly. Replace them with a safe marker before Markdown
|
||||||
|
* sanitization; the image component resolves that marker through DeerFlow.
|
||||||
|
*/
|
||||||
|
export function rewriteProtectedWikiImageSources(markdown: string): string {
|
||||||
|
return markdown.replace(
|
||||||
|
PROTECTED_WIKI_IMAGE_SOURCE,
|
||||||
|
(filePath) => `${WIKI_PROTECTED_IMAGE_MARKER}?file_path=${encodeURIComponent(filePath)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function parseProtectedWikiImageSource(source: string | undefined): string | null {
|
||||||
|
if (!source) return null;
|
||||||
|
try {
|
||||||
|
const url = new URL(source, window.location.origin);
|
||||||
|
if (url.pathname !== WIKI_PROTECTED_IMAGE_MARKER) return null;
|
||||||
|
return url.searchParams.get("file_path")?.trim() || null;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export function stripWikiFrontmatter(value: string) {
|
export function stripWikiFrontmatter(value: string) {
|
||||||
return value.replace(/^\s*---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, "").trim();
|
return value.replace(/^\s*---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, "").trim();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -444,6 +444,23 @@ export async function getLlmWikiDocumentPreview(
|
|||||||
return response.blob();
|
return response.blob();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function getLlmWikiProtectedFile(
|
||||||
|
knowledgeBaseId: string,
|
||||||
|
filePath: string,
|
||||||
|
publicAccess = false,
|
||||||
|
): Promise<Blob> {
|
||||||
|
const base = publicAccess ? PUBLIC_API() : API();
|
||||||
|
const params = new URLSearchParams({ file_path: filePath });
|
||||||
|
const response = await apiFetch(
|
||||||
|
`${base}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/files?${params.toString()}`,
|
||||||
|
);
|
||||||
|
if (!response.ok) {
|
||||||
|
await readJson<void>(response, "读取 Wiki 图片");
|
||||||
|
throw new Error("读取 Wiki 图片失败");
|
||||||
|
}
|
||||||
|
return response.blob();
|
||||||
|
}
|
||||||
|
|
||||||
export async function getLlmWikiGraph(knowledgeBaseId: string): Promise<LlmWikiGraph> {
|
export async function getLlmWikiGraph(knowledgeBaseId: string): Promise<LlmWikiGraph> {
|
||||||
return readJson(
|
return readJson(
|
||||||
await apiFetch(`${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/graph?limit=200`),
|
await apiFetch(`${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/graph?limit=200`),
|
||||||
|
|||||||
+20
-2
@@ -33,6 +33,7 @@ import { reasoningPlugins } from "@/core/streamdown";
|
|||||||
import {
|
import {
|
||||||
exportableWikiMarkdown,
|
exportableWikiMarkdown,
|
||||||
renderableWikiMarkdown,
|
renderableWikiMarkdown,
|
||||||
|
rewriteProtectedWikiImageSources,
|
||||||
safeWikiFileName,
|
safeWikiFileName,
|
||||||
stripWikiFrontmatter,
|
stripWikiFrontmatter,
|
||||||
wikiLinkMarkdown,
|
wikiLinkMarkdown,
|
||||||
@@ -49,6 +50,7 @@ import {
|
|||||||
type LlmWikiDocument,
|
type LlmWikiDocument,
|
||||||
type LlmWikiWikiPage,
|
type LlmWikiWikiPage,
|
||||||
} from "@/strategy-components/api/llmwiki";
|
} from "@/strategy-components/api/llmwiki";
|
||||||
|
import { WikiProtectedImage } from "./WikiProtectedImage";
|
||||||
|
|
||||||
type SourceContextResponse = {
|
type SourceContextResponse = {
|
||||||
success?: boolean;
|
success?: boolean;
|
||||||
@@ -926,10 +928,18 @@ export function WeKnoraSourceDrawer({
|
|||||||
</div>
|
</div>
|
||||||
<MarkdownContent
|
<MarkdownContent
|
||||||
className="max-w-none"
|
className="max-w-none"
|
||||||
content={preview.text}
|
content={rewriteProtectedWikiImageSources(preview.text)}
|
||||||
isLoading={false}
|
isLoading={false}
|
||||||
remarkPlugins={reasoningPlugins.remarkPlugins}
|
remarkPlugins={reasoningPlugins.remarkPlugins}
|
||||||
rehypePlugins={reasoningPlugins.rehypePlugins}
|
rehypePlugins={reasoningPlugins.rehypePlugins}
|
||||||
|
components={{
|
||||||
|
img: (props) => (
|
||||||
|
<WikiProtectedImage
|
||||||
|
{...props}
|
||||||
|
knowledgeBaseId={target?.knowledgeBaseId || ""}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
}}
|
||||||
/>
|
/>
|
||||||
</article>
|
</article>
|
||||||
</section>
|
</section>
|
||||||
@@ -964,10 +974,18 @@ export function WeKnoraSourceDrawer({
|
|||||||
<div className="rounded-lg border bg-background p-5">
|
<div className="rounded-lg border bg-background p-5">
|
||||||
<MarkdownContent
|
<MarkdownContent
|
||||||
className="max-w-none"
|
className="max-w-none"
|
||||||
content={wikiDisplayContent}
|
content={rewriteProtectedWikiImageSources(wikiDisplayContent)}
|
||||||
isLoading={false}
|
isLoading={false}
|
||||||
remarkPlugins={reasoningPlugins.remarkPlugins}
|
remarkPlugins={reasoningPlugins.remarkPlugins}
|
||||||
rehypePlugins={reasoningPlugins.rehypePlugins}
|
rehypePlugins={reasoningPlugins.rehypePlugins}
|
||||||
|
components={{
|
||||||
|
img: (props) => (
|
||||||
|
<WikiProtectedImage
|
||||||
|
{...props}
|
||||||
|
knowledgeBaseId={target?.knowledgeBaseId || ""}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
}}
|
||||||
linkClassName={(href) =>
|
linkClassName={(href) =>
|
||||||
parseWeKnoraPreviewUrl(href)?.wikiSlug
|
parseWeKnoraPreviewUrl(href)?.wikiSlug
|
||||||
? "wiki-content-link font-medium text-[#be2323] decoration-[#be2323]/55 decoration-dashed hover:decoration-solid"
|
? "wiki-content-link font-medium text-[#be2323] decoration-[#be2323]/55 decoration-dashed hover:decoration-solid"
|
||||||
|
|||||||
+13
-2
@@ -35,6 +35,7 @@ import { markdownToDocxBlob } from "@/core/scheduled-tasks/markdown-docx";
|
|||||||
import {
|
import {
|
||||||
exportableWikiMarkdown,
|
exportableWikiMarkdown,
|
||||||
renderableWikiMarkdown,
|
renderableWikiMarkdown,
|
||||||
|
rewriteProtectedWikiImageSources,
|
||||||
safeWikiFileName,
|
safeWikiFileName,
|
||||||
stripWikiFrontmatter,
|
stripWikiFrontmatter,
|
||||||
wikiLinkMarkdown,
|
wikiLinkMarkdown,
|
||||||
@@ -49,6 +50,7 @@ import {
|
|||||||
usePublicLlmWikiWikiPage,
|
usePublicLlmWikiWikiPage,
|
||||||
usePublicLlmWikiWikiPages,
|
usePublicLlmWikiWikiPages,
|
||||||
} from "@/strategy-components/api/llmwiki";
|
} from "@/strategy-components/api/llmwiki";
|
||||||
|
import { WikiProtectedImage } from "./WikiProtectedImage";
|
||||||
|
|
||||||
const LIST_PATH = "/page/workspace/knowledge";
|
const LIST_PATH = "/page/workspace/knowledge";
|
||||||
|
|
||||||
@@ -209,7 +211,7 @@ export function WeKnoraWikiPageView({
|
|||||||
: pageData?.content,
|
: pageData?.content,
|
||||||
));
|
));
|
||||||
const displayMarkdown = pageData
|
const displayMarkdown = pageData
|
||||||
? renderableWikiMarkdown(
|
? rewriteProtectedWikiImageSources(renderableWikiMarkdown(
|
||||||
title,
|
title,
|
||||||
content,
|
content,
|
||||||
noteId,
|
noteId,
|
||||||
@@ -222,7 +224,7 @@ export function WeKnoraWikiPageView({
|
|||||||
: undefined,
|
: undefined,
|
||||||
publicEmbed ? pagesList : [],
|
publicEmbed ? pagesList : [],
|
||||||
publicEmbed ? publicWikiIndex.data?.groups || [] : [],
|
publicEmbed ? publicWikiIndex.data?.groups || [] : [],
|
||||||
)
|
))
|
||||||
: "";
|
: "";
|
||||||
const exportMarkdown = pageData
|
const exportMarkdown = pageData
|
||||||
? wikiLinkMarkdown(exportableWikiMarkdown(title, summary, content), noteId)
|
? wikiLinkMarkdown(exportableWikiMarkdown(title, summary, content), noteId)
|
||||||
@@ -497,6 +499,15 @@ export function WeKnoraWikiPageView({
|
|||||||
isLoading={false}
|
isLoading={false}
|
||||||
remarkPlugins={reasoningPlugins.remarkPlugins}
|
remarkPlugins={reasoningPlugins.remarkPlugins}
|
||||||
rehypePlugins={reasoningPlugins.rehypePlugins}
|
rehypePlugins={reasoningPlugins.rehypePlugins}
|
||||||
|
components={{
|
||||||
|
img: (props) => (
|
||||||
|
<WikiProtectedImage
|
||||||
|
{...props}
|
||||||
|
knowledgeBaseId={noteId}
|
||||||
|
publicAccess={publicEmbed}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
}}
|
||||||
linkClassName={(href) =>
|
linkClassName={(href) =>
|
||||||
publicEmbed && publicWikiTargetFromHref(href, noteId)
|
publicEmbed && publicWikiTargetFromHref(href, noteId)
|
||||||
? "wiki-content-link border-b border-dashed !border-[var(--primary)] font-medium !text-primary !no-underline hover:border-solid focus-visible:rounded-sm focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary"
|
? "wiki-content-link border-b border-dashed !border-[var(--primary)] font-medium !text-primary !no-underline hover:border-solid focus-visible:rounded-sm focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary"
|
||||||
|
|||||||
+75
@@ -0,0 +1,75 @@
|
|||||||
|
import { ImageOff, Loader2 } from "lucide-react";
|
||||||
|
import { type ImgHTMLAttributes, useEffect, useState } from "react";
|
||||||
|
|
||||||
|
import { parseProtectedWikiImageSource } from "@/core/llmwiki/wiki-page-markdown";
|
||||||
|
import { getLlmWikiProtectedFile } from "@/strategy-components/api/llmwiki";
|
||||||
|
|
||||||
|
type WikiProtectedImageProps = ImgHTMLAttributes<HTMLImageElement> & {
|
||||||
|
knowledgeBaseId: string;
|
||||||
|
publicAccess?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function WikiProtectedImage({
|
||||||
|
knowledgeBaseId,
|
||||||
|
publicAccess = false,
|
||||||
|
src,
|
||||||
|
alt,
|
||||||
|
className,
|
||||||
|
...props
|
||||||
|
}: WikiProtectedImageProps) {
|
||||||
|
const filePath = parseProtectedWikiImageSource(src);
|
||||||
|
const [objectUrl, setObjectUrl] = useState("");
|
||||||
|
const [error, setError] = useState("");
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!filePath || !knowledgeBaseId) {
|
||||||
|
setObjectUrl("");
|
||||||
|
setError("");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let cancelled = false;
|
||||||
|
let nextObjectUrl = "";
|
||||||
|
setObjectUrl("");
|
||||||
|
setError("");
|
||||||
|
void getLlmWikiProtectedFile(knowledgeBaseId, filePath, publicAccess)
|
||||||
|
.then((blob) => {
|
||||||
|
if (cancelled) return;
|
||||||
|
nextObjectUrl = URL.createObjectURL(blob);
|
||||||
|
setObjectUrl(nextObjectUrl);
|
||||||
|
})
|
||||||
|
.catch((reason: unknown) => {
|
||||||
|
if (!cancelled) {
|
||||||
|
setError(reason instanceof Error ? reason.message : "Wiki 图片加载失败");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
if (nextObjectUrl) URL.revokeObjectURL(nextObjectUrl);
|
||||||
|
};
|
||||||
|
}, [filePath, knowledgeBaseId, publicAccess]);
|
||||||
|
|
||||||
|
if (!filePath) {
|
||||||
|
return <img {...props} src={src} alt={alt} className={className} />;
|
||||||
|
}
|
||||||
|
if (error) {
|
||||||
|
return (
|
||||||
|
<span
|
||||||
|
className="my-3 flex min-h-24 items-center justify-center gap-2 rounded-lg border border-dashed bg-muted/40 px-4 text-sm text-muted-foreground"
|
||||||
|
data-wiki-image-error={error}
|
||||||
|
role="img"
|
||||||
|
aria-label={alt || "Wiki 图片加载失败"}
|
||||||
|
>
|
||||||
|
<ImageOff className="size-5" />
|
||||||
|
{alt || "Wiki 图片加载失败"}
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!objectUrl) {
|
||||||
|
return (
|
||||||
|
<span className="my-3 flex min-h-24 items-center justify-center rounded-lg bg-muted/30 text-muted-foreground">
|
||||||
|
<Loader2 className="size-5 animate-spin" aria-label="正在加载 Wiki 图片" />
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return <img {...props} src={objectUrl} alt={alt} className={className} />;
|
||||||
|
}
|
||||||
@@ -1016,6 +1016,11 @@ Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` me
|
|||||||
`llmwiki.weknora.api_base_url` is the provider switch: non-empty enables WeKnora, empty preserves the legacy LLMWiki UI. WeKnora is independently deployed and owns PostgreSQL/Redis/object storage/vector storage/parsers/models; DeerFlow config contains only `api_base_url` and optional `web_base_url`. DeerFlow deliberately sends no WeKnora credential, so the WeKnora endpoint must be restricted to a trusted network and allow DeerFlow traffic itself or through its reverse proxy. There is intentionally no `default_embedding_model_id`: `WeKnoraClient` queries `/api/v1/models` and uses WeKnora's active Embedding model when creating a KB; Wiki creation additionally resolves an active KnowledgeQA model inside WeKnora for synthesis. The native BFF is `app/gateway/routers/llmwiki.py`; ownership/publication mappings live in `deerflow.persistence.llmwiki`; the browser only sees DeerFlow mapping IDs. Knowledge-base owners publish immediately (`private` -> `published`) without a review queue, and can unpublish ordinary bases. The conversation-deposit KB is system-owned, always published, shared by all DeerFlow users, excluded from My Knowledge, and immutable through user/admin detail operations; recognition requires both the reserved name and `system` owner. Existing same-name remote mappings are adopted in place as the system deposit, preserving mapping ids and document links. Its user-facing description, generated Markdown source and tags use `cmzs` branding; list/detail reads synchronize stale remote descriptions, and search results, source previews and embedded-detail JSON recursively replace legacy brand text in historical conversation-deposit payloads. Deposit requests must reference a thread accessible to the current account. Personal knowledge-base list/search views are scoped to the current DeerFlow account even for admins; frontend LLMWiki caches include the live account id, and admin write power does not make other users' private KBs appear as owned. Only `system_role === "admin"` sees the WeKnora connection-config tab. Knowledge cards navigate to a dedicated WeKnora-style detail route. Wiki-enabled bases render and default to `Wiki -> documents -> graph`; standard bases omit Wiki and render `documents -> graph`. The detail page supports sequential multi-file upload with an aggregate result for writable ordinary bases, hides it for public read-only/system-deposit bases, and uses a generic loading label while the embed session starts. It also supports URL and published manual-Markdown imports, browses real Wiki pages/content, proxies authenticated original-file previews, pages through `/chunks/{knowledge_id}`, and renders the Wiki page graph from `/knowledgebase/{kb_id}/wiki/graph`; document details use a right sheet rather than the removed KB modal. The credential-free proxy uses one WeKnora service identity, so per-user separation is enforced by DeerFlow mappings while all remote KBs physically share that identity's WeKnora workspace. WeKnora iframe detail pages first call `/api/llmwiki/knowledge-bases/{id}/weknora/session` through the normal DeerFlow API auth path, then load `/platform/knowledge-bases/{weknora_id}` with a signed `deerflow_weknora_embed` cookie for `/platform`, `/assets`, `/locales`, root assets, and API calls. Injected fetch/XHR rewriting sends all iframe `/api/v1/*`, including colliding `/api/v1/auth/*`, through `/api/llmwiki/weknora-embed/api/v1/*`; Gateway auth/CSRF bypass only applies to those signed iframe proxy requests, the proxy still revalidates the mapped KB, and cookie security honors `X-Forwarded-Proto` for HTTPS reverse proxies. `llmwiki_search` is conditionally registered in `tools.py`, is read-only, and rechecks the current DeerFlow user, agent `llmwiki_knowledge_base_ids`, and thread-context selection on every call. Source previews prefer the cleaned Wiki/LLMWiki page text when a retrieved chunk maps to a Wiki page, with raw split chunks as the fallback. Agent bindings use an inline WeKnora-style searchable/grouped multi-select in the standard agent create/detail UI; new chats use the same interaction pattern as a compact composer dropdown, and selections are written to thread metadata/context and restored on reopen. The DeerFlow frontend is native React and the WeKnora frontend is not modified or embedded. Full deployment and behavior notes: `docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`. Tests: `tests/test_llmwiki_weknora.py`.
|
`llmwiki.weknora.api_base_url` is the provider switch: non-empty enables WeKnora, empty preserves the legacy LLMWiki UI. WeKnora is independently deployed and owns PostgreSQL/Redis/object storage/vector storage/parsers/models; DeerFlow config contains only `api_base_url` and optional `web_base_url`. DeerFlow deliberately sends no WeKnora credential, so the WeKnora endpoint must be restricted to a trusted network and allow DeerFlow traffic itself or through its reverse proxy. There is intentionally no `default_embedding_model_id`: `WeKnoraClient` queries `/api/v1/models` and uses WeKnora's active Embedding model when creating a KB; Wiki creation additionally resolves an active KnowledgeQA model inside WeKnora for synthesis. The native BFF is `app/gateway/routers/llmwiki.py`; ownership/publication mappings live in `deerflow.persistence.llmwiki`; the browser only sees DeerFlow mapping IDs. Knowledge-base owners publish immediately (`private` -> `published`) without a review queue, and can unpublish ordinary bases. The conversation-deposit KB is system-owned, always published, shared by all DeerFlow users, excluded from My Knowledge, and immutable through user/admin detail operations; recognition requires both the reserved name and `system` owner. Existing same-name remote mappings are adopted in place as the system deposit, preserving mapping ids and document links. Its user-facing description, generated Markdown source and tags use `cmzs` branding; list/detail reads synchronize stale remote descriptions, and search results, source previews and embedded-detail JSON recursively replace legacy brand text in historical conversation-deposit payloads. Deposit requests must reference a thread accessible to the current account. Personal knowledge-base list/search views are scoped to the current DeerFlow account even for admins; frontend LLMWiki caches include the live account id, and admin write power does not make other users' private KBs appear as owned. Only `system_role === "admin"` sees the WeKnora connection-config tab. Knowledge cards navigate to a dedicated WeKnora-style detail route. Wiki-enabled bases render and default to `Wiki -> documents -> graph`; standard bases omit Wiki and render `documents -> graph`. The detail page supports sequential multi-file upload with an aggregate result for writable ordinary bases, hides it for public read-only/system-deposit bases, and uses a generic loading label while the embed session starts. It also supports URL and published manual-Markdown imports, browses real Wiki pages/content, proxies authenticated original-file previews, pages through `/chunks/{knowledge_id}`, and renders the Wiki page graph from `/knowledgebase/{kb_id}/wiki/graph`; document details use a right sheet rather than the removed KB modal. The credential-free proxy uses one WeKnora service identity, so per-user separation is enforced by DeerFlow mappings while all remote KBs physically share that identity's WeKnora workspace. WeKnora iframe detail pages first call `/api/llmwiki/knowledge-bases/{id}/weknora/session` through the normal DeerFlow API auth path, then load `/platform/knowledge-bases/{weknora_id}` with a signed `deerflow_weknora_embed` cookie for `/platform`, `/assets`, `/locales`, root assets, and API calls. Injected fetch/XHR rewriting sends all iframe `/api/v1/*`, including colliding `/api/v1/auth/*`, through `/api/llmwiki/weknora-embed/api/v1/*`; Gateway auth/CSRF bypass only applies to those signed iframe proxy requests, the proxy still revalidates the mapped KB, and cookie security honors `X-Forwarded-Proto` for HTTPS reverse proxies. `llmwiki_search` is conditionally registered in `tools.py`, is read-only, and rechecks the current DeerFlow user, agent `llmwiki_knowledge_base_ids`, and thread-context selection on every call. Source previews prefer the cleaned Wiki/LLMWiki page text when a retrieved chunk maps to a Wiki page, with raw split chunks as the fallback. Agent bindings use an inline WeKnora-style searchable/grouped multi-select in the standard agent create/detail UI; new chats use the same interaction pattern as a compact composer dropdown, and selections are written to thread metadata/context and restored on reopen. The DeerFlow frontend is native React and the WeKnora frontend is not modified or embedded. Full deployment and behavior notes: `docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`. Tests: `tests/test_llmwiki_weknora.py`.
|
||||||
|
|
||||||
The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. Do not add an HTML `sandbox` attribute to this trusted, permission-filtered detail iframe: a sandboxed ancestor causes Chromium's built-in Blob PDF viewer to render the browser-blocked page. The Gateway follows upstream redirects only for `/knowledge/{id}/preview` and `/knowledge-bases/{id}/files`, keeping PDF bytes and protected Wiki images inside the signed proxy instead of sending the browser to WeKnora object storage.
|
The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. Do not add an HTML `sandbox` attribute to this trusted, permission-filtered detail iframe: a sandboxed ancestor causes Chromium's built-in Blob PDF viewer to render the browser-blocked page. The Gateway follows upstream redirects only for `/knowledge/{id}/preview` and `/knowledge-bases/{id}/files`, keeping PDF bytes and protected Wiki images inside the signed proxy instead of sending the browser to WeKnora object storage.
|
||||||
|
Native/private and public Wiki readers must rewrite protected Markdown image
|
||||||
|
sources to their mapping-scoped DeerFlow `/files?file_path=` endpoints and load
|
||||||
|
them as Blob URLs. The embedded guard also rewrites protected fetch/XHR values
|
||||||
|
and hydrates late `img[data-protected-src]` placeholders; never expose the
|
||||||
|
service token or relax the current-KB authorization to make images work.
|
||||||
|
|
||||||
**Embedded native knowledge-base switching:** The original WeKnora breadcrumb picker remains available inside the detail iframe. A signed 15-minute embed cookie carries only the current DeerFlow user's visible mappings (their own plus published bases); the proxied `/api/v1/knowledge-bases` response is filtered to that allowlist. Choosing another entry performs a full iframe navigation, rechecks the DeerFlow mapping and read/write permission, then reissues the cookie for the selected base. Never expose the upstream service-admin list directly or permit an arbitrary remote knowledge-base ID.
|
**Embedded native knowledge-base switching:** The original WeKnora breadcrumb picker remains available inside the detail iframe. A signed 15-minute embed cookie carries only the current DeerFlow user's visible mappings (their own plus published bases); the proxied `/api/v1/knowledge-bases` response is filtered to that allowlist. Choosing another entry performs a full iframe navigation, rechecks the DeerFlow mapping and read/write permission, then reissues the cookie for the selected base. Never expose the upstream service-admin list directly or permit an arbitrary remote knowledge-base ID.
|
||||||
|
|
||||||
@@ -1442,8 +1447,10 @@ detail. Never call `/knowledge-search` from internal search, RAG, or the
|
|||||||
`deerflow.integrations.weknora.local_index` and
|
`deerflow.integrations.weknora.local_index` and
|
||||||
`deerflow.persistence.llmwiki_index`; preserve the harness → app import
|
`deerflow.persistence.llmwiki_index`; preserve the harness → app import
|
||||||
firewall. Gateway composition injects the store/search/sync services through
|
firewall. Gateway composition injects the store/search/sync services through
|
||||||
`local_index.runtime`, and the lease-protected scheduler starts only when both
|
`local_index.runtime`. `enabled` turns on local vector search and manual
|
||||||
`enabled` and `sync_enabled` are true.
|
vectorization; `auto_sync` is the separate opt-in for the lease-protected
|
||||||
|
scheduler. The scheduler sleeps for one full configured interval before its
|
||||||
|
first scan, so application startup must never trigger immediate vectorization.
|
||||||
|
|
||||||
Vectors are normalized little-endian float32 BLOBs. A page mirror and all of
|
Vectors are normalized little-endian float32 BLOBs. A page mirror and all of
|
||||||
its vectors are replaced in one transaction only after strict batched
|
its vectors are replaced in one transaction only after strict batched
|
||||||
@@ -1456,6 +1463,10 @@ input, not an outbound API field: fixed-dimension BGE/OpenAI-compatible
|
|||||||
endpoints can reject a `dimensions` request parameter with HTTP 400. When the
|
endpoints can reject a `dimensions` request parameter with HTTP 400. When the
|
||||||
value is omitted, the embedding client probes and freezes the first response's
|
value is omitted, the embedding client probes and freezes the first response's
|
||||||
dimension before sync lease/rebuild decisions.
|
dimension before sync lease/rebuild decisions.
|
||||||
|
Embedding errors must retain the upstream status and bounded response body in
|
||||||
|
logs, without exposing API keys. SQL vector writes are split into statements
|
||||||
|
of at most `database_batch_size` rows (default 50) inside the same atomic page
|
||||||
|
replacement transaction.
|
||||||
During a fingerprint rebuild the mapping uses the Wiki-page API until every
|
During a fingerprint rebuild the mapping uses the Wiki-page API until every
|
||||||
page is successfully rebuilt. `WikiRetrievalService` partitions mixed requests
|
page is successfully rebuilt. `WikiRetrievalService` partitions mixed requests
|
||||||
per mapping, so ready libraries stay local while incomplete libraries use that
|
per mapping, so ready libraries stay local while incomplete libraries use that
|
||||||
|
|||||||
@@ -777,15 +777,24 @@ processed Wiki content. A knowledge base uses exact NumPy cosine similarity
|
|||||||
only after its whole Wiki has completed one coherent vectorization pass; until
|
only after its whole Wiki has completed one coherent vectorization pass; until
|
||||||
then internal search calls WeKnora's Wiki-page search/list API and reads the
|
then internal search calls WeKnora's Wiki-page search/list API and reads the
|
||||||
matched Wiki pages. It never falls back to raw documents or chunks. The local
|
matched Wiki pages. It never falls back to raw documents or chunks. The local
|
||||||
vector feature is disabled by default. Configure
|
vector feature is disabled by default. Configure the three
|
||||||
`llmwiki.local_wiki_index.embedding` first, then enable `enabled`,
|
`llmwiki.local_wiki_index.embedding` values (`model`, `base_url`, and
|
||||||
`sync_enabled`, and `internal_search_enabled` in `config.yaml`.
|
`api_key`), then change only `enabled` to `true` in `config.yaml`. Manual
|
||||||
|
vectorization is the default. Set `auto_sync: true` only when periodic
|
||||||
|
background synchronization is wanted; the scheduler waits one full interval
|
||||||
|
after startup before its first scan, so boot does not trigger a rebuild.
|
||||||
|
`database_batch_size` caps each vector insert statement (default `50`) while
|
||||||
|
the page replacement remains atomic, preventing a large Wiki from flooding
|
||||||
|
the SQL connection in one statement.
|
||||||
The optional `embedding.dimensions` value is used only to validate the returned
|
The optional `embedding.dimensions` value is used only to validate the returned
|
||||||
vector size and build the index fingerprint; it is deliberately not sent in
|
vector size and build the index fingerprint; it is deliberately not sent in
|
||||||
the embedding request because fixed-dimension BGE/OpenAI-compatible endpoints
|
the embedding request because fixed-dimension BGE/OpenAI-compatible endpoints
|
||||||
may reject that extra field with HTTP 400. Omit it to discover the dimension
|
may reject that extra field with HTTP 400. Omit it to discover the dimension
|
||||||
from the first successful response.
|
from the first successful response.
|
||||||
|
|
||||||
|
Embedding failures log the upstream HTTP status and a bounded response body.
|
||||||
|
The client never sends a `dimensions` request parameter.
|
||||||
|
|
||||||
The migration creates `llmwiki_wiki_pages`, `llmwiki_wiki_vectors`, and
|
The migration creates `llmwiki_wiki_pages`, `llmwiki_wiki_vectors`, and
|
||||||
`llmwiki_wiki_sync_states`. Wiki Markdown and normalized float32 vectors are
|
`llmwiki_wiki_sync_states`. Wiki Markdown and normalized float32 vectors are
|
||||||
stored in DeerFlow's configured SQL database. Multi-worker sync is protected by
|
stored in DeerFlow's configured SQL database. Multi-worker sync is protected by
|
||||||
@@ -839,6 +848,13 @@ system conversation-deposit knowledge base. Private or unpublished bases are
|
|||||||
never exposed by this route. Anonymous consumers can discover the available
|
never exposed by this route. Anonymous consumers can discover the available
|
||||||
public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each
|
public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each
|
||||||
entry deliberately contains only the DeerFlow mapping `id` and display `name`.
|
entry deliberately contains only the DeerFlow mapping `id` and display `name`.
|
||||||
|
Protected `resource://` and supported object-storage images are fetched through
|
||||||
|
the current mapping instead of being exposed as upstream URLs. Authenticated
|
||||||
|
readers use `GET /api/llmwiki/knowledge-bases/{id}/files?file_path=...`; public
|
||||||
|
readers use the corresponding `/api/public/llmwiki/...` route, which requires a
|
||||||
|
published mapping. The embedded WeKnora guard applies the same mapping-scoped
|
||||||
|
proxy to document previews and Markdown images, including images inserted after
|
||||||
|
the drawer/page first renders.
|
||||||
|
|
||||||
Wiki management also provides **导出 Excel**. `GET
|
Wiki management also provides **导出 Excel**. `GET
|
||||||
/api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx` exports every processed Wiki
|
/api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx` exports every processed Wiki
|
||||||
|
|||||||
@@ -998,7 +998,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
|
|||||||
local_wiki_index = app.state.config.llmwiki.local_wiki_index
|
local_wiki_index = app.state.config.llmwiki.local_wiki_index
|
||||||
if (
|
if (
|
||||||
local_wiki_index.enabled
|
local_wiki_index.enabled
|
||||||
and local_wiki_index.sync_enabled
|
and local_wiki_index.auto_sync
|
||||||
and getattr(app.state, "llmwiki_sync_service", None) is not None
|
and getattr(app.state, "llmwiki_sync_service", None) is not None
|
||||||
):
|
):
|
||||||
from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop
|
from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop
|
||||||
|
|||||||
@@ -321,13 +321,16 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
|||||||
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
|
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
|
||||||
from deerflow.persistence.llmwiki_index import make_llmwiki_index_store
|
from deerflow.persistence.llmwiki_index import make_llmwiki_index_store
|
||||||
|
|
||||||
app.state.llmwiki_index_store = make_llmwiki_index_store(sf)
|
local_index_config = config.llmwiki.local_wiki_index
|
||||||
|
app.state.llmwiki_index_store = make_llmwiki_index_store(
|
||||||
|
sf,
|
||||||
|
write_batch_size=local_index_config.database_batch_size,
|
||||||
|
)
|
||||||
app.state.llmwiki_embedding = None
|
app.state.llmwiki_embedding = None
|
||||||
app.state.llmwiki_vector_cache = None
|
app.state.llmwiki_vector_cache = None
|
||||||
app.state.llmwiki_vector_search = None
|
app.state.llmwiki_vector_search = None
|
||||||
app.state.llmwiki_sync_service = None
|
app.state.llmwiki_sync_service = None
|
||||||
app.state.llmwiki_retrieval_service = None
|
app.state.llmwiki_retrieval_service = None
|
||||||
local_index_config = config.llmwiki.local_wiki_index
|
|
||||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||||
|
|
||||||
resolved_llmwiki = get_resolved_llmwiki_runtime(config)
|
resolved_llmwiki = get_resolved_llmwiki_runtime(config)
|
||||||
|
|||||||
@@ -48,6 +48,10 @@ async def _sync_once(app: FastAPI) -> None:
|
|||||||
|
|
||||||
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
|
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
|
||||||
interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds
|
interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds
|
||||||
|
logger.info(
|
||||||
|
"Local Wiki auto-sync scheduler started; first scan delayed by %s seconds",
|
||||||
|
interval,
|
||||||
|
)
|
||||||
while True:
|
while True:
|
||||||
await _sync_once(app)
|
|
||||||
await asyncio.sleep(interval)
|
await asyncio.sleep(interval)
|
||||||
|
await _sync_once(app)
|
||||||
@@ -61,6 +61,10 @@ router = APIRouter(tags=["llmwiki"])
|
|||||||
proxy_router = APIRouter(tags=["llmwiki-proxy"])
|
proxy_router = APIRouter(tags=["llmwiki-proxy"])
|
||||||
|
|
||||||
_MAX_UPLOAD_BYTES = 100 * 1024 * 1024
|
_MAX_UPLOAD_BYTES = 100 * 1024 * 1024
|
||||||
|
_PROTECTED_WIKI_FILE_PATH = re.compile(
|
||||||
|
r"^(?:storage://[0-9A-Za-z_-]+/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs)://",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
_WEKNORA_ADMIN_SESSION_CACHE: dict[str, dict[str, Any]] = {}
|
_WEKNORA_ADMIN_SESSION_CACHE: dict[str, dict[str, Any]] = {}
|
||||||
_WEKNORA_PUBLIC_PREFIX = "/deerflow"
|
_WEKNORA_PUBLIC_PREFIX = "/deerflow"
|
||||||
_WEKNORA_PUBLIC_ROOT_PATHS = (
|
_WEKNORA_PUBLIC_ROOT_PATHS = (
|
||||||
@@ -242,7 +246,7 @@ def _store(request: Request) -> LlmWikiStore:
|
|||||||
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
|
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
|
||||||
service = getattr(request.app.state, "llmwiki_sync_service", None)
|
service = getattr(request.app.state, "llmwiki_sync_service", None)
|
||||||
config = request.app.state.config.llmwiki.local_wiki_index
|
config = request.app.state.config.llmwiki.local_wiki_index
|
||||||
if service is not None and config.enabled and mapping.get("wiki_index_enabled", True):
|
if service is not None and config.enabled and config.auto_sync and mapping.get("wiki_index_enabled", True):
|
||||||
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
|
||||||
|
|
||||||
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
|
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
|
||||||
@@ -1085,6 +1089,8 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
|
|||||||
|
|
||||||
function rewriteApiUrl(value) {{
|
function rewriteApiUrl(value) {{
|
||||||
try {{
|
try {{
|
||||||
|
const protectedUrl = protectedImageApiUrl(value);
|
||||||
|
if (protectedUrl) return protectedUrl;
|
||||||
const next = new URL(value, location.origin);
|
const next = new URL(value, location.origin);
|
||||||
if (next.origin !== location.origin) return value;
|
if (next.origin !== location.origin) return value;
|
||||||
const publicPrefix = cfg.publicPrefix || "";
|
const publicPrefix = cfg.publicPrefix || "";
|
||||||
@@ -1104,6 +1110,19 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
|
|||||||
return value;
|
return value;
|
||||||
}}
|
}}
|
||||||
}}
|
}}
|
||||||
|
function protectedImageApiUrl(value) {{
|
||||||
|
const raw = String(value || "").trim();
|
||||||
|
if (!/^(?:storage:\\/\\/[0-9A-Za-z_-]+\\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\\/\\//i.test(raw)) {{
|
||||||
|
return "";
|
||||||
|
}}
|
||||||
|
const endpoint =
|
||||||
|
cfg.apiPrefix +
|
||||||
|
"/api/v1/knowledge-bases/" +
|
||||||
|
encodeURIComponent(cfg.weknoraId) +
|
||||||
|
"/files?file_path=" +
|
||||||
|
encodeURIComponent(raw);
|
||||||
|
return new URL(endpoint, location.origin).href;
|
||||||
|
}}
|
||||||
const nativeFetch = window.fetch.bind(window);
|
const nativeFetch = window.fetch.bind(window);
|
||||||
window.fetch = (input, init) => {{
|
window.fetch = (input, init) => {{
|
||||||
if (input instanceof Request) {{
|
if (input instanceof Request) {{
|
||||||
@@ -1116,6 +1135,67 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
|
|||||||
return nativeXhrOpen.call(this, method, rewriteApiUrl(url), ...rest);
|
return nativeXhrOpen.call(this, method, rewriteApiUrl(url), ...rest);
|
||||||
}};
|
}};
|
||||||
|
|
||||||
|
// WeKnora renders protected Markdown images as transparent placeholders and
|
||||||
|
// hydrates them asynchronously. In an embedded page that hydration can run
|
||||||
|
// before its own API client has picked up our proxy prefix, leaving a 1x1
|
||||||
|
// GIF forever. Resolve the same protected source through the scoped proxy
|
||||||
|
// as a fallback. The backend validates the scheme and current KB again.
|
||||||
|
function hydrateProtectedImage(image) {{
|
||||||
|
const protectedSrc = String(image?.getAttribute?.("data-protected-src") || "").trim();
|
||||||
|
const proxyUrl = protectedImageApiUrl(protectedSrc);
|
||||||
|
if (!proxyUrl) return;
|
||||||
|
if (
|
||||||
|
image.dataset.deerflowProtectedSource === protectedSrc &&
|
||||||
|
String(image.getAttribute("src") || "").startsWith("blob:")
|
||||||
|
) return;
|
||||||
|
const attempts = Number(image.dataset.deerflowProtectedAttempts || "0");
|
||||||
|
if (attempts >= 3 || image.dataset.deerflowProtectedLoading === "1") return;
|
||||||
|
image.dataset.deerflowProtectedLoading = "1";
|
||||||
|
image.dataset.deerflowProtectedAttempts = String(attempts + 1);
|
||||||
|
nativeFetch(proxyUrl, {{ credentials: "same-origin", headers: {{ Accept: "image/*" }} }})
|
||||||
|
.then((response) => {{
|
||||||
|
if (!response.ok) throw new Error(`protected image request failed (${{response.status}})`);
|
||||||
|
return response.blob();
|
||||||
|
}})
|
||||||
|
.then((blob) => {{
|
||||||
|
const objectUrl = URL.createObjectURL(blob);
|
||||||
|
image.dataset.deerflowProtectedSource = protectedSrc;
|
||||||
|
image.removeAttribute("data-img-loading");
|
||||||
|
image.removeAttribute("data-wiki-image-error");
|
||||||
|
image.src = objectUrl;
|
||||||
|
image.addEventListener("load", () => URL.revokeObjectURL(objectUrl), {{ once: true }});
|
||||||
|
}})
|
||||||
|
.catch(() => {{
|
||||||
|
image.dataset.wikiImageError = "true";
|
||||||
|
window.setTimeout(() => {{
|
||||||
|
delete image.dataset.deerflowProtectedLoading;
|
||||||
|
hydrateProtectedImage(image);
|
||||||
|
}}, 1200 * (attempts + 1));
|
||||||
|
}})
|
||||||
|
.finally(() => {{
|
||||||
|
delete image.dataset.deerflowProtectedLoading;
|
||||||
|
}});
|
||||||
|
}}
|
||||||
|
function hydrateProtectedImages(scope = document) {{
|
||||||
|
if (scope?.matches?.("img[data-protected-src]")) hydrateProtectedImage(scope);
|
||||||
|
scope?.querySelectorAll?.("img[data-protected-src]").forEach(hydrateProtectedImage);
|
||||||
|
}}
|
||||||
|
const protectedImageObserver = new MutationObserver((mutations) => {{
|
||||||
|
mutations.forEach((mutation) => {{
|
||||||
|
if (mutation.type === "attributes") hydrateProtectedImage(mutation.target);
|
||||||
|
mutation.addedNodes?.forEach((node) => {{
|
||||||
|
if (node?.nodeType === Node.ELEMENT_NODE) hydrateProtectedImages(node);
|
||||||
|
}});
|
||||||
|
}});
|
||||||
|
}});
|
||||||
|
protectedImageObserver.observe(document.documentElement, {{
|
||||||
|
childList: true,
|
||||||
|
subtree: true,
|
||||||
|
attributes: true,
|
||||||
|
attributeFilter: ["data-protected-src"]
|
||||||
|
}});
|
||||||
|
hydrateProtectedImages();
|
||||||
|
|
||||||
function tone(color, alpha) {{
|
function tone(color, alpha) {{
|
||||||
return `color-mix(in srgb, ${{color}} ${{Math.round(alpha * 100)}}%, transparent)`;
|
return `color-mix(in srgb, ${{color}} ${{Math.round(alpha * 100)}}%, transparent)`;
|
||||||
}}
|
}}
|
||||||
@@ -1519,6 +1599,7 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
|
|||||||
enforceLocation();
|
enforceLocation();
|
||||||
purgeGuides();
|
purgeGuides();
|
||||||
hideWeKnoraShell();
|
hideWeKnoraShell();
|
||||||
|
hydrateProtectedImages();
|
||||||
if (tickSourceDetail() && sourceDetailMode) {{
|
if (tickSourceDetail() && sourceDetailMode) {{
|
||||||
clearInterval(maintenanceTimer);
|
clearInterval(maintenanceTimer);
|
||||||
try {{ readyObserver.disconnect(); }} catch (_) {{}}
|
try {{ readyObserver.disconnect(); }} catch (_) {{}}
|
||||||
@@ -1820,6 +1901,32 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
|
|||||||
return row
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_protected_wiki_file_path(file_path: str) -> str:
|
||||||
|
value = file_path.strip()
|
||||||
|
if not value or "\x00" in value or not _PROTECTED_WIKI_FILE_PATH.match(value):
|
||||||
|
raise HTTPException(status_code=422, detail="Unsupported Wiki image path")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
async def _wiki_file_response(row: dict[str, Any], file_path: str, *, public: bool) -> Response:
|
||||||
|
try:
|
||||||
|
content, media_type = await _client_or_503().get_knowledge_base_file(
|
||||||
|
str(row["weknora_id"]),
|
||||||
|
_validate_protected_wiki_file_path(file_path),
|
||||||
|
)
|
||||||
|
except WeKnoraError as exc:
|
||||||
|
_raise_upstream(exc)
|
||||||
|
return Response(
|
||||||
|
content=content,
|
||||||
|
media_type=media_type,
|
||||||
|
headers={
|
||||||
|
"Cache-Control": "public, max-age=300" if public else "private, max-age=60",
|
||||||
|
"Content-Disposition": "inline",
|
||||||
|
"X-Content-Type-Options": "nosniff",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def _verify_document(client: WeKnoraClient, row: dict[str, Any], document_id: str) -> dict[str, Any]:
|
async def _verify_document(client: WeKnoraClient, row: dict[str, Any], document_id: str) -> dict[str, Any]:
|
||||||
document = await client.get_document(document_id)
|
document = await client.get_document(document_id)
|
||||||
if str(document.get("knowledge_base_id") or "") != str(row["weknora_id"]):
|
if str(document.get("knowledge_base_id") or "") != str(row["weknora_id"]):
|
||||||
@@ -2510,6 +2617,18 @@ async def list_public_wiki_pages(
|
|||||||
return {**result, "pages": pages}
|
return {**result, "pages": pages}
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}/files")
|
||||||
|
async def get_public_wiki_file(
|
||||||
|
request: Request,
|
||||||
|
mapping_id: str,
|
||||||
|
file_path: str = Query(..., min_length=1, max_length=8192),
|
||||||
|
) -> Response:
|
||||||
|
"""Serve one protected image belonging to a published Wiki."""
|
||||||
|
|
||||||
|
row = await _public_mapping(request, mapping_id)
|
||||||
|
return await _wiki_file_response(row, file_path, public=True)
|
||||||
|
|
||||||
|
|
||||||
_PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison")
|
_PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison")
|
||||||
|
|
||||||
|
|
||||||
@@ -2820,6 +2939,18 @@ async def preview_document(request: Request, mapping_id: str, document_id: str)
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/files")
|
||||||
|
async def get_wiki_file(
|
||||||
|
request: Request,
|
||||||
|
mapping_id: str,
|
||||||
|
file_path: str = Query(..., min_length=1, max_length=8192),
|
||||||
|
) -> Response:
|
||||||
|
"""Serve one protected Wiki image after normal knowledge-base authorization."""
|
||||||
|
|
||||||
|
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
|
||||||
|
return await _wiki_file_response(row, file_path, public=False)
|
||||||
|
|
||||||
|
|
||||||
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph")
|
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph")
|
||||||
async def get_knowledge_base_graph(
|
async def get_knowledge_base_graph(
|
||||||
request: Request,
|
request: Request,
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
@@ -14,6 +15,7 @@ from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectori
|
|||||||
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
from deerflow.integrations.weknora.local_index.search import WikiIndexError
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"])
|
router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"])
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class WikiVectorSearchRequest(BaseModel):
|
class WikiVectorSearchRequest(BaseModel):
|
||||||
@@ -181,6 +183,7 @@ async def _run_sync(app, mapping_ids: list[str], force: bool) -> None:
|
|||||||
try:
|
try:
|
||||||
await service.sync_mapping(mapping, force=force)
|
await service.sync_mapping(mapping, force=force)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id"))
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|
||||||
@@ -191,6 +194,7 @@ async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
|
|||||||
try:
|
try:
|
||||||
await service.sync_mapping(mapping, force=force)
|
await service.sync_mapping(mapping, force=force)
|
||||||
except Exception:
|
except Exception:
|
||||||
|
logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id"))
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -556,47 +556,17 @@ llmwiki:
|
|||||||
web_base_url: "http://119.254.86.251:8088"
|
web_base_url: "http://119.254.86.251:8088"
|
||||||
admin_email: "lqq@qq.com"
|
admin_email: "lqq@qq.com"
|
||||||
admin_password: "1qaz@WSX"
|
admin_password: "1qaz@WSX"
|
||||||
# DeerFlow-local Wiki mirror/vector index. Keep disabled until the
|
# 本地 Wiki 向量化:只需把 enabled 改为 true,并填写下面 3 个 embedding 值。
|
||||||
# OpenAI-compatible embedding endpoint below is configured and reachable.
|
# 默认只在页面点击“向量化”时执行;auto_sync=true 才会后台自动增量同步,
|
||||||
|
# 且服务启动时只启动定时器,不会立刻全量向量化。
|
||||||
local_wiki_index:
|
local_wiki_index:
|
||||||
enabled: false
|
enabled: false
|
||||||
sync_enabled: false
|
auto_sync: false
|
||||||
internal_search_enabled: false
|
database_batch_size: 50
|
||||||
reference_ui_enabled: false
|
|
||||||
sync_interval_seconds: 120
|
|
||||||
chunk_max_chars: 1000
|
|
||||||
chunk_overlap_chars: 120
|
|
||||||
chunk_min_chars: 80
|
|
||||||
embedding:
|
embedding:
|
||||||
enabled: true
|
|
||||||
provider: openai
|
|
||||||
model: bge-m3
|
model: bge-m3
|
||||||
base_url: ""
|
base_url: ""
|
||||||
api_key: ""
|
api_key: ""
|
||||||
# Expected BGE-M3 response size for local validation/indexing only.
|
|
||||||
# This value is not sent as an embedding API request parameter.
|
|
||||||
dimensions: 1024
|
|
||||||
batch_size: 32
|
|
||||||
timeout_seconds: 30
|
|
||||||
max_retries: 3
|
|
||||||
engine: numpy_exact
|
|
||||||
strict_vector: true
|
|
||||||
top_k_sections: 30
|
|
||||||
top_k_pages: 8
|
|
||||||
max_sections_per_page: 2
|
|
||||||
min_similarity: 0.35
|
|
||||||
cache_max_knowledge_bases: 20
|
|
||||||
cache_revision_check_seconds: 5
|
|
||||||
external_api:
|
|
||||||
enabled: false
|
|
||||||
api_key: ""
|
|
||||||
previous_api_key: ""
|
|
||||||
max_query_chars: 1000
|
|
||||||
max_knowledge_bases: 20
|
|
||||||
max_results: 50
|
|
||||||
default_results: 10
|
|
||||||
include_content_max_chars: 12000
|
|
||||||
requests_per_minute: 60
|
|
||||||
|
|
||||||
log_system:
|
log_system:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|||||||
@@ -279,65 +279,25 @@ llmwiki:
|
|||||||
|
|
||||||
local_wiki_index:
|
local_wiki_index:
|
||||||
enabled: true
|
enabled: true
|
||||||
|
# 默认只允许手动向量化;需要定时增量同步时再开启。
|
||||||
# 同步
|
auto_sync: false
|
||||||
sync_enabled: true
|
# 每条 SQL 最多插入的向量行数;整页替换仍在同一事务内完成。
|
||||||
sync_interval_seconds: 120
|
database_batch_size: 50
|
||||||
sync_page_size: 100
|
|
||||||
sync_concurrency: 3
|
|
||||||
sync_lease_seconds: 600
|
|
||||||
sync_request_timeout_seconds: 30
|
|
||||||
sync_max_retries: 3
|
|
||||||
sync_retry_backoff_seconds: 5
|
|
||||||
|
|
||||||
# 切分
|
|
||||||
chunk_max_chars: 1000
|
|
||||||
chunk_overlap_chars: 120
|
|
||||||
chunk_min_chars: 80
|
|
||||||
title_prefix_enabled: true
|
|
||||||
|
|
||||||
# Embedding:OpenAI-compatible /v1/embeddings
|
|
||||||
embedding:
|
embedding:
|
||||||
enabled: true
|
|
||||||
provider: openai
|
|
||||||
model: bge-m3
|
model: bge-m3
|
||||||
base_url: "$WIKI_EMBEDDING_BASE_URL"
|
base_url: "$WIKI_EMBEDDING_BASE_URL"
|
||||||
api_key: "$WIKI_EMBEDDING_API_KEY"
|
api_key: "$WIKI_EMBEDDING_API_KEY"
|
||||||
dimensions: 1024
|
|
||||||
batch_size: 32
|
|
||||||
timeout_seconds: 30
|
|
||||||
max_retries: 3
|
|
||||||
|
|
||||||
# 检索
|
|
||||||
engine: numpy_exact
|
|
||||||
strict_vector: true
|
|
||||||
top_k_sections: 30
|
|
||||||
top_k_pages: 8
|
|
||||||
max_sections_per_page: 2
|
|
||||||
min_similarity: 0.35
|
|
||||||
keyword_boost_enabled: false
|
|
||||||
keyword_boost_weight: 0.15
|
|
||||||
cache_max_knowledge_bases: 20
|
|
||||||
cache_revision_check_seconds: 5
|
|
||||||
|
|
||||||
# 外部接口
|
|
||||||
external_api:
|
|
||||||
enabled: false
|
|
||||||
api_key: "$LLMWIKI_EXTERNAL_SEARCH_API_KEY"
|
|
||||||
max_query_chars: 1000
|
|
||||||
max_knowledge_bases: 20
|
|
||||||
max_results: 50
|
|
||||||
default_results: 10
|
|
||||||
include_content_max_chars: 12000
|
|
||||||
requests_per_minute: 60
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
日常部署只需配置上面三个 Embedding 值并将 `enabled` 改为 `true`。
|
||||||
|
`auto_sync=false` 时仅由页面“手动向量化”触发;即使设为 `true`,定时器也会先等待一个完整同步周期,不会在服务刚启动时立即重建索引。其余切分、检索、超时与重试参数继续使用代码默认值,只有确有调优需要时才覆盖。
|
||||||
|
|
||||||
### 6.1 配置校验规则
|
### 6.1 配置校验规则
|
||||||
|
|
||||||
启动时应执行:
|
启动时应执行:
|
||||||
|
|
||||||
1. `enabled=true` 时,Embedding `base_url` 和 `model` 必须非空;
|
1. `enabled=true` 时,Embedding `base_url` 和 `model` 必须非空;
|
||||||
2. `dimensions` 可省略,但第一次成功 Embedding 后必须固定并写入索引 profile;
|
2. 不向上游发送 `dimensions` 参数;第一次成功 Embedding 后从响应自动发现维度,并固定到索引 profile;
|
||||||
3. `chunk_overlap_chars < chunk_max_chars`;
|
3. `chunk_overlap_chars < chunk_max_chars`;
|
||||||
4. `top_k_pages <= top_k_sections`;
|
4. `top_k_pages <= top_k_sections`;
|
||||||
5. `external_api.enabled=true` 时必须配置非空 API Key;
|
5. `external_api.enabled=true` 时必须配置非空 API Key;
|
||||||
@@ -1885,9 +1845,7 @@ class WikiVectorEngine(ABC):
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
local_wiki_index.enabled
|
local_wiki_index.enabled
|
||||||
local_wiki_index.sync_enabled
|
local_wiki_index.auto_sync
|
||||||
local_wiki_index.internal_search_enabled
|
|
||||||
local_wiki_index.reference_ui_enabled
|
|
||||||
local_wiki_index.external_api.enabled
|
local_wiki_index.external_api.enabled
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
+24
-4
@@ -59,7 +59,7 @@ class LocalWikiEmbeddingConfig(BaseModel):
|
|||||||
base_url: str = ""
|
base_url: str = ""
|
||||||
api_key: str = Field(default="", repr=False)
|
api_key: str = Field(default="", repr=False)
|
||||||
dimensions: int | None = Field(
|
dimensions: int | None = Field(
|
||||||
default=1024,
|
default=None,
|
||||||
ge=1,
|
ge=1,
|
||||||
le=65536,
|
le=65536,
|
||||||
description="Expected response vector dimensions; not sent to the embedding endpoint.",
|
description="Expected response vector dimensions; not sent to the embedding endpoint.",
|
||||||
@@ -108,9 +108,7 @@ class LocalWikiIndexConfig(BaseModel):
|
|||||||
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
|
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
|
||||||
|
|
||||||
enabled: bool = False
|
enabled: bool = False
|
||||||
sync_enabled: bool = False
|
auto_sync: bool = False
|
||||||
internal_search_enabled: bool = False
|
|
||||||
reference_ui_enabled: bool = False
|
|
||||||
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
|
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
|
||||||
sync_page_size: int = Field(default=100, ge=1, le=500)
|
sync_page_size: int = Field(default=100, ge=1, le=500)
|
||||||
sync_concurrency: int = Field(default=3, ge=1, le=32)
|
sync_concurrency: int = Field(default=3, ge=1, le=32)
|
||||||
@@ -133,9 +131,31 @@ class LocalWikiIndexConfig(BaseModel):
|
|||||||
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
|
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
|
||||||
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
|
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
|
||||||
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
|
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
|
||||||
|
database_batch_size: int = Field(default=50, ge=1, le=500)
|
||||||
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
|
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
|
||||||
model_config = ConfigDict(extra="ignore")
|
model_config = ConfigDict(extra="ignore")
|
||||||
|
|
||||||
|
@model_validator(mode="before")
|
||||||
|
@classmethod
|
||||||
|
def _migrate_legacy_flags(cls, value):
|
||||||
|
if isinstance(value, dict) and "auto_sync" not in value and "sync_enabled" in value:
|
||||||
|
value = {**value, "auto_sync": value.get("sync_enabled")}
|
||||||
|
return value
|
||||||
|
|
||||||
|
@property
|
||||||
|
def sync_enabled(self) -> bool:
|
||||||
|
"""Backward-compatible status name; automatic work is one explicit switch."""
|
||||||
|
|
||||||
|
return self.auto_sync
|
||||||
|
|
||||||
|
@property
|
||||||
|
def internal_search_enabled(self) -> bool:
|
||||||
|
return self.enabled
|
||||||
|
|
||||||
|
@property
|
||||||
|
def reference_ui_enabled(self) -> bool:
|
||||||
|
return self.enabled
|
||||||
|
|
||||||
@model_validator(mode="after")
|
@model_validator(mode="after")
|
||||||
def _validate_index(self):
|
def _validate_index(self):
|
||||||
if self.chunk_overlap_chars >= self.chunk_max_chars:
|
if self.chunk_overlap_chars >= self.chunk_max_chars:
|
||||||
|
|||||||
+46
-1
@@ -2,8 +2,8 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections.abc import Mapping, Sequence
|
|
||||||
import time
|
import time
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import quote
|
from urllib.parse import quote
|
||||||
|
|
||||||
@@ -430,6 +430,51 @@ class WeKnoraClient:
|
|||||||
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
|
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
|
||||||
return response.content, media_type or "application/octet-stream"
|
return response.content, media_type or "application/octet-stream"
|
||||||
|
|
||||||
|
async def get_knowledge_base_file(
|
||||||
|
self,
|
||||||
|
knowledge_base_id: str,
|
||||||
|
file_path: str,
|
||||||
|
) -> tuple[bytes, str]:
|
||||||
|
"""Return one protected Wiki attachment using DeerFlow's WeKnora identity."""
|
||||||
|
|
||||||
|
owns_client = self._http_client is None
|
||||||
|
client = self._http_client or httpx.AsyncClient(
|
||||||
|
timeout=httpx.Timeout(self.timeout_seconds),
|
||||||
|
follow_redirects=True,
|
||||||
|
trust_env=False,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
for attempt in range(2):
|
||||||
|
response = await client.request(
|
||||||
|
"GET",
|
||||||
|
f"{self.base_url}/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/files",
|
||||||
|
params={"file_path": file_path},
|
||||||
|
headers=await self._headers(force_refresh=attempt > 0, accept="*/*"),
|
||||||
|
follow_redirects=True,
|
||||||
|
)
|
||||||
|
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
|
||||||
|
break
|
||||||
|
except httpx.TimeoutException as exc:
|
||||||
|
raise WeKnoraError("WeKnora Wiki image request timed out", status_code=504) from exc
|
||||||
|
except httpx.HTTPError as exc:
|
||||||
|
raise WeKnoraError("WeKnora Wiki image is unavailable", status_code=502) from exc
|
||||||
|
finally:
|
||||||
|
if owns_client:
|
||||||
|
await client.aclose()
|
||||||
|
if response.status_code == 404:
|
||||||
|
raise WeKnoraError("WeKnora Wiki image was not found", status_code=404)
|
||||||
|
if response.status_code in {401, 403}:
|
||||||
|
if self._has_admin_credentials:
|
||||||
|
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated Wiki image request", status_code=502)
|
||||||
|
raise WeKnoraError(
|
||||||
|
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
|
||||||
|
status_code=502,
|
||||||
|
)
|
||||||
|
if response.status_code >= 400:
|
||||||
|
raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502)
|
||||||
|
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
|
||||||
|
return response.content, media_type or "application/octet-stream"
|
||||||
|
|
||||||
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
|
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
|
||||||
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
|
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
|
||||||
return dict(value) if isinstance(value, dict) else {}
|
return dict(value) if isinstance(value, dict) else {}
|
||||||
|
|||||||
+37
-6
@@ -15,6 +15,21 @@ from deerflow.integrations.weknora.local_index.vector_codec import normalize_vec
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _upstream_error_detail(response: httpx.Response) -> str:
|
||||||
|
"""Return a bounded upstream reason without hiding the useful response body."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
payload = response.json()
|
||||||
|
if isinstance(payload, dict):
|
||||||
|
detail = payload.get("detail") or payload.get("message") or payload.get("error") or payload
|
||||||
|
else:
|
||||||
|
detail = payload
|
||||||
|
value = str(detail)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
value = response.text
|
||||||
|
return " ".join(value.split())[:2000] or "no response body"
|
||||||
|
|
||||||
|
|
||||||
class WikiEmbeddingError(RuntimeError):
|
class WikiEmbeddingError(RuntimeError):
|
||||||
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
|
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
|
||||||
super().__init__(message)
|
super().__init__(message)
|
||||||
@@ -76,11 +91,15 @@ class StrictWikiEmbeddingClient:
|
|||||||
started = monotonic()
|
started = monotonic()
|
||||||
try:
|
try:
|
||||||
response = await (await self._get_client()).post(url, headers=headers, json=payload)
|
response = await (await self._get_client()).post(url, headers=headers, json=payload)
|
||||||
if response.status_code in {401, 403}:
|
if response.status_code >= 400:
|
||||||
raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False)
|
detail = _upstream_error_detail(response)
|
||||||
if response.status_code == 429 or response.status_code >= 500:
|
category = "authentication" if response.status_code in {401, 403} else "upstream"
|
||||||
raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True)
|
retryable = response.status_code == 429 or response.status_code >= 500
|
||||||
response.raise_for_status()
|
raise WikiEmbeddingError(
|
||||||
|
f"Embedding service returned HTTP {response.status_code}: {detail}",
|
||||||
|
category=category,
|
||||||
|
retryable=retryable,
|
||||||
|
)
|
||||||
data = response.json()
|
data = response.json()
|
||||||
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
|
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
|
||||||
if len(items) != len(texts):
|
if len(items) != len(texts):
|
||||||
@@ -98,8 +117,20 @@ class StrictWikiEmbeddingClient:
|
|||||||
except httpx.TimeoutException:
|
except httpx.TimeoutException:
|
||||||
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
|
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
|
||||||
except (httpx.HTTPError, ValueError, TypeError) as exc:
|
except (httpx.HTTPError, ValueError, TypeError) as exc:
|
||||||
last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError))
|
reason = " ".join(str(exc).split())[:2000] or type(exc).__name__
|
||||||
|
last_error = WikiEmbeddingError(
|
||||||
|
f"Embedding request failed ({type(exc).__name__}): {reason}",
|
||||||
|
category="request",
|
||||||
|
retryable=isinstance(exc, httpx.HTTPError),
|
||||||
|
)
|
||||||
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
|
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
|
||||||
|
logger.error(
|
||||||
|
"Wiki embedding batch failed model=%s batch=%d category=%s reason=%s",
|
||||||
|
self.config.model,
|
||||||
|
len(texts),
|
||||||
|
last_error.category,
|
||||||
|
last_error,
|
||||||
|
)
|
||||||
raise last_error
|
raise last_error
|
||||||
await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
|
await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
|
||||||
raise last_error or WikiEmbeddingError("Embedding request failed")
|
raise last_error or WikiEmbeddingError("Embedding request failed")
|
||||||
|
|||||||
+6
-2
@@ -6,8 +6,12 @@ from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWiki
|
|||||||
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
|
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
|
||||||
|
|
||||||
|
|
||||||
def make_llmwiki_index_store(session_factory):
|
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
|
||||||
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory)
|
return (
|
||||||
|
MemoryLlmWikiIndexStore()
|
||||||
|
if session_factory is None
|
||||||
|
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
|
|||||||
+26
-20
@@ -6,7 +6,7 @@ from datetime import UTC, datetime, timedelta
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlalchemy import case, delete, func, or_, select, update
|
from sqlalchemy import case, delete, func, insert, or_, select, update
|
||||||
from sqlalchemy.exc import IntegrityError
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||||
|
|
||||||
@@ -27,8 +27,9 @@ def _dict(row) -> dict[str, Any]:
|
|||||||
|
|
||||||
|
|
||||||
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
||||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None:
|
||||||
self._sf = session_factory
|
self._sf = session_factory
|
||||||
|
self.write_batch_size = max(1, min(int(write_batch_size), 500))
|
||||||
|
|
||||||
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
|
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
|
||||||
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
|
||||||
@@ -82,24 +83,29 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
|
|||||||
row.updated_at = now
|
row.updated_at = now
|
||||||
await session.flush()
|
await session.flush()
|
||||||
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
|
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
|
||||||
session.add_all(
|
vector_rows = [
|
||||||
[
|
{
|
||||||
LlmWikiWikiVectorRow(
|
"id": str(uuid4()),
|
||||||
id=str(uuid4()),
|
"page_id": row.id,
|
||||||
page_id=row.id,
|
"knowledge_base_mapping_id": mapping_id,
|
||||||
knowledge_base_mapping_id=mapping_id,
|
"section_index": int(item["section_index"]),
|
||||||
section_index=int(item["section_index"]),
|
"heading": item.get("heading"),
|
||||||
heading=item.get("heading"),
|
"section_content": str(item["section_content"]),
|
||||||
section_content=str(item["section_content"]),
|
"content_hash": str(item["content_hash"]),
|
||||||
content_hash=str(item["content_hash"]),
|
"embedding_fingerprint": fingerprint,
|
||||||
embedding_fingerprint=fingerprint,
|
"embedding_dimensions": int(item["embedding_dimensions"]),
|
||||||
embedding_dimensions=int(item["embedding_dimensions"]),
|
"vector_blob": bytes(item["vector_blob"]),
|
||||||
vector_blob=bytes(item["vector_blob"]),
|
"created_at": now,
|
||||||
created_at=now,
|
}
|
||||||
)
|
for item in vectors
|
||||||
for item in vectors
|
]
|
||||||
]
|
# Keep the page replacement atomic, but cap each SQL statement so
|
||||||
)
|
# large Wiki pages cannot flood the database with one huge insert.
|
||||||
|
for start in range(0, len(vector_rows), self.write_batch_size):
|
||||||
|
await session.execute(
|
||||||
|
insert(LlmWikiWikiVectorRow),
|
||||||
|
vector_rows[start : start + self.write_batch_size],
|
||||||
|
)
|
||||||
state = await self._ensure_state(session, mapping_id)
|
state = await self._ensure_state(session, mapping_id)
|
||||||
state.index_revision = int(state.index_revision or 0) + 1
|
state.index_revision = int(state.index_revision or 0) + 1
|
||||||
state.embedding_fingerprint = fingerprint
|
state.embedding_fingerprint = fingerprint
|
||||||
|
|||||||
@@ -40,20 +40,31 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
|
|||||||
)
|
)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
|
|
||||||
store = SqlLlmWikiIndexStore(session_factory)
|
store = SqlLlmWikiIndexStore(session_factory, write_batch_size=1)
|
||||||
assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp")
|
assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp")
|
||||||
assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp")
|
assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp")
|
||||||
page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"})
|
page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"})
|
||||||
blob, dimensions = encode_vector([3.0, 4.0], dimensions=2)
|
blob, dimensions = encode_vector([3.0, 4.0], dimensions=2)
|
||||||
saved = await store.replace_page_vectors(
|
saved = await store.replace_page_vectors(
|
||||||
page=page,
|
page=page,
|
||||||
vectors=[{"section_index": 0, "heading": "History", "section_content": "Alice founded Acme.", "content_hash": "section", "embedding_dimensions": dimensions, "vector_blob": blob}],
|
vectors=[
|
||||||
|
{
|
||||||
|
"section_index": index,
|
||||||
|
"heading": "History",
|
||||||
|
"section_content": f"Alice founded Acme. Part {index}",
|
||||||
|
"content_hash": f"section-{index}",
|
||||||
|
"embedding_dimensions": dimensions,
|
||||||
|
"vector_blob": blob,
|
||||||
|
}
|
||||||
|
for index in range(3)
|
||||||
|
],
|
||||||
fingerprint="fp",
|
fingerprint="fp",
|
||||||
sync_id="sync-1",
|
sync_id="sync-1",
|
||||||
)
|
)
|
||||||
snapshot = await store.load_vector_snapshot("mapping-1", "fp")
|
snapshot = await store.load_vector_snapshot("mapping-1", "fp")
|
||||||
assert saved["slug"] == "people/alice"
|
assert saved["slug"] == "people/alice"
|
||||||
assert len(snapshot["rows"]) == 1
|
assert store.write_batch_size == 1
|
||||||
|
assert len(snapshot["rows"]) == 3
|
||||||
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
|
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
|
||||||
|
|
||||||
assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
|
assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
|
||||||
|
|||||||
@@ -690,6 +690,24 @@ async def test_weknora_client_proxies_original_document_preview_without_credenti
|
|||||||
assert media_type == "text/plain"
|
assert media_type == "text/plain"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_weknora_client_fetches_protected_wiki_image() -> None:
|
||||||
|
async def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
assert request.url.path == "/api/v1/knowledge-bases/kb-one/files"
|
||||||
|
assert request.url.params["file_path"] == "resource://wiki/figure.png"
|
||||||
|
return httpx.Response(200, content=b"png-bytes", headers={"Content-Type": "image/png"})
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
|
||||||
|
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
|
||||||
|
content, media_type = await client.get_knowledge_base_file(
|
||||||
|
"kb-one",
|
||||||
|
"resource://wiki/figure.png",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert content == b"png-bytes"
|
||||||
|
assert media_type == "image/png"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_memory_store_enforces_owner_and_public_visibility() -> None:
|
async def test_memory_store_enforces_owner_and_public_visibility() -> None:
|
||||||
store = MemoryLlmWikiStore()
|
store = MemoryLlmWikiStore()
|
||||||
@@ -1358,6 +1376,10 @@ def test_weknora_iframe_html_rewrites_public_prefix_paths() -> None:
|
|||||||
assert '"allowedWeKnoraIds":["remote-kb","other-visible-kb"]' in injected
|
assert '"allowedWeKnoraIds":["remote-kb","other-visible-kb"]' in injected
|
||||||
assert ".breadcrumb-link.dropdown" not in injected
|
assert ".breadcrumb-link.dropdown" not in injected
|
||||||
assert "reloadForKnowledgeBaseSwitch" in injected
|
assert "reloadForKnowledgeBaseSwitch" in injected
|
||||||
|
assert "function protectedImageApiUrl" in injected
|
||||||
|
assert 'img[data-protected-src]' in injected
|
||||||
|
assert 'encodeURIComponent(cfg.weknoraId)' in injected
|
||||||
|
assert 'credentials: "same-origin"' in injected
|
||||||
|
|
||||||
|
|
||||||
def test_weknora_iframe_filters_kb_list_to_signed_visible_mappings() -> None:
|
def test_weknora_iframe_filters_kb_list_to_signed_visible_mappings() -> None:
|
||||||
|
|||||||
@@ -1,10 +1,13 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from app.gateway import llmwiki_index_scheduler
|
||||||
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig
|
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig
|
||||||
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
|
||||||
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
|
||||||
@@ -85,6 +88,24 @@ async def test_configured_dimensions_are_validated_but_not_sent_to_bge() -> None
|
|||||||
assert requests == [{"model": "bge-embedding-m3", "input": ["Wiki page"]}]
|
assert requests == [{"model": "bge-embedding-m3", "input": ["Wiki page"]}]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_embedding_error_includes_upstream_reason(caplog) -> None:
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(400, json={"detail": "unsupported request parameter: dimensions"})
|
||||||
|
|
||||||
|
config = LocalWikiEmbeddingConfig(
|
||||||
|
model="bge-embedding-m3",
|
||||||
|
base_url="http://embedding.test/v1",
|
||||||
|
max_retries=0,
|
||||||
|
)
|
||||||
|
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
|
||||||
|
client = StrictWikiEmbeddingClient(config, http_client=http_client)
|
||||||
|
with pytest.raises(RuntimeError, match="unsupported request parameter: dimensions"):
|
||||||
|
await client.embed_query("Wiki page")
|
||||||
|
|
||||||
|
assert "unsupported request parameter: dimensions" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_omitted_dimensions_are_discovered_from_first_response() -> None:
|
async def test_omitted_dimensions_are_discovered_from_first_response() -> None:
|
||||||
def handler(request: httpx.Request) -> httpx.Response:
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
@@ -167,3 +188,39 @@ async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> Non
|
|||||||
assert store.states["kb"]["state"] == "idle"
|
assert store.states["kb"]["state"] == "idle"
|
||||||
assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
|
assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
|
||||||
assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}
|
assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
|
||||||
|
calls: list[str] = []
|
||||||
|
|
||||||
|
async def fake_sleep(seconds: float) -> None:
|
||||||
|
calls.append(f"sleep:{seconds}")
|
||||||
|
raise asyncio.CancelledError
|
||||||
|
|
||||||
|
async def fake_sync_once(app) -> None:
|
||||||
|
calls.append("sync")
|
||||||
|
|
||||||
|
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
|
||||||
|
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
|
||||||
|
app = SimpleNamespace(
|
||||||
|
state=SimpleNamespace(
|
||||||
|
config=SimpleNamespace(
|
||||||
|
llmwiki=SimpleNamespace(
|
||||||
|
local_wiki_index=SimpleNamespace(sync_interval_seconds=120),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(asyncio.CancelledError):
|
||||||
|
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
|
||||||
|
|
||||||
|
assert calls == ["sleep:120"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_local_wiki_index_accepts_legacy_sync_flag() -> None:
|
||||||
|
config = LocalWikiIndexConfig.model_validate({"sync_enabled": True})
|
||||||
|
|
||||||
|
assert config.auto_sync is True
|
||||||
|
assert config.sync_enabled is True
|
||||||
@@ -177,6 +177,37 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None:
|
|||||||
assert payload["pages"][0]["aliases"] == []
|
assert payload["pages"][0]["aliases"] == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None:
|
||||||
|
store = PublicStore(_published_row())
|
||||||
|
|
||||||
|
class Client:
|
||||||
|
async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str):
|
||||||
|
assert knowledge_base_id == "remote-one"
|
||||||
|
assert file_path == "storage://tenant/resource://wiki/figure.png"
|
||||||
|
return b"png-bytes", "image/png"
|
||||||
|
|
||||||
|
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
|
||||||
|
|
||||||
|
response = await llmwiki.get_public_wiki_file(
|
||||||
|
_request(store),
|
||||||
|
"kb-one",
|
||||||
|
"storage://tenant/resource://wiki/figure.png",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert store.calls == [("kb-one", "__public_wiki__", False, False)]
|
||||||
|
assert response.body == b"png-bytes"
|
||||||
|
assert response.media_type == "image/png"
|
||||||
|
assert response.headers["cache-control"] == "public, max-age=300"
|
||||||
|
|
||||||
|
|
||||||
|
def test_wiki_file_proxy_rejects_non_weknora_urls() -> None:
|
||||||
|
with pytest.raises(HTTPException) as exc_info:
|
||||||
|
llmwiki._validate_protected_wiki_file_path("https://example.test/private.png")
|
||||||
|
|
||||||
|
assert exc_info.value.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None:
|
async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None:
|
||||||
store = PublicStore(_published_row())
|
store = PublicStore(_published_row())
|
||||||
|
|||||||
Reference in new issue
Block a user