fix: restore embedded wiki images and optimize vector sync

This commit is contained in:
吉鑫升 committed 2026-08-25 11:24:01 +08:00
1 parent 30943159bf
commit 0c5e2e8cb0
23 files changed
+609 -136

No files matched your search

@@ -37,6 +37,34 @@ const WIKI_INDEX_TYPE_LABELS: Record<string, string> = {
const WIKI_INDEX_TYPE_ORDER = ["summary", "entity", "concept"]; const WIKI_INDEX_TYPE_ORDER = ["summary", "entity", "concept"];
export const WIKI_PROTECTED_IMAGE_MARKER = "/__llmwiki-protected-image__";
const PROTECTED_WIKI_IMAGE_SOURCE =
/(?:storage:\/\/[0-9A-Za-z_-]+\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\/\/[^\s"'<>\])]+/gi;
/**
* Browsers cannot load WeKnora's protected resource:// and object-storage
* schemes directly. Replace them with a safe marker before Markdown
* sanitization; the image component resolves that marker through DeerFlow.
*/
export function rewriteProtectedWikiImageSources(markdown: string): string {
return markdown.replace(
PROTECTED_WIKI_IMAGE_SOURCE,
(filePath) => `${WIKI_PROTECTED_IMAGE_MARKER}?file_path=${encodeURIComponent(filePath)}`,
);
}
export function parseProtectedWikiImageSource(source: string | undefined): string | null {
if (!source) return null;
try {
const url = new URL(source, window.location.origin);
if (url.pathname !== WIKI_PROTECTED_IMAGE_MARKER) return null;
return url.searchParams.get("file_path")?.trim() || null;
} catch {
return null;
}
}
export function stripWikiFrontmatter(value: string) { export function stripWikiFrontmatter(value: string) {
return value.replace(/^\s*---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, "").trim(); return value.replace(/^\s*---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, "").trim();
} }
@@ -444,6 +444,23 @@ export async function getLlmWikiDocumentPreview(
return response.blob(); return response.blob();
} }
export async function getLlmWikiProtectedFile(
knowledgeBaseId: string,
filePath: string,
publicAccess = false,
): Promise<Blob> {
const base = publicAccess ? PUBLIC_API() : API();
const params = new URLSearchParams({ file_path: filePath });
const response = await apiFetch(
`${base}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/files?${params.toString()}`,
);
if (!response.ok) {
await readJson<void>(response, "读取 Wiki 图片");
throw new Error("读取 Wiki 图片失败");
}
return response.blob();
}
export async function getLlmWikiGraph(knowledgeBaseId: string): Promise<LlmWikiGraph> { export async function getLlmWikiGraph(knowledgeBaseId: string): Promise<LlmWikiGraph> {
return readJson( return readJson(
await apiFetch(`${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/graph?limit=200`), await apiFetch(`${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/graph?limit=200`),
@@ -33,6 +33,7 @@ import { reasoningPlugins } from "@/core/streamdown";
import { import {
exportableWikiMarkdown, exportableWikiMarkdown,
renderableWikiMarkdown, renderableWikiMarkdown,
rewriteProtectedWikiImageSources,
safeWikiFileName, safeWikiFileName,
stripWikiFrontmatter, stripWikiFrontmatter,
wikiLinkMarkdown, wikiLinkMarkdown,
@@ -49,6 +50,7 @@ import {
type LlmWikiDocument, type LlmWikiDocument,
type LlmWikiWikiPage, type LlmWikiWikiPage,
} from "@/strategy-components/api/llmwiki"; } from "@/strategy-components/api/llmwiki";
import { WikiProtectedImage } from "./WikiProtectedImage";
type SourceContextResponse = { type SourceContextResponse = {
success?: boolean; success?: boolean;
@@ -926,10 +928,18 @@ export function WeKnoraSourceDrawer({
</div> </div>
<MarkdownContent <MarkdownContent
className="max-w-none" className="max-w-none"
content={preview.text} content={rewriteProtectedWikiImageSources(preview.text)}
isLoading={false} isLoading={false}
remarkPlugins={reasoningPlugins.remarkPlugins} remarkPlugins={reasoningPlugins.remarkPlugins}
rehypePlugins={reasoningPlugins.rehypePlugins} rehypePlugins={reasoningPlugins.rehypePlugins}
components={{
img: (props) => (
<WikiProtectedImage
{...props}
knowledgeBaseId={target?.knowledgeBaseId || ""}
/>
),
}}
/> />
</article> </article>
</section> </section>
@@ -964,10 +974,18 @@ export function WeKnoraSourceDrawer({
<div className="rounded-lg border bg-background p-5"> <div className="rounded-lg border bg-background p-5">
<MarkdownContent <MarkdownContent
className="max-w-none" className="max-w-none"
content={wikiDisplayContent} content={rewriteProtectedWikiImageSources(wikiDisplayContent)}
isLoading={false} isLoading={false}
remarkPlugins={reasoningPlugins.remarkPlugins} remarkPlugins={reasoningPlugins.remarkPlugins}
rehypePlugins={reasoningPlugins.rehypePlugins} rehypePlugins={reasoningPlugins.rehypePlugins}
components={{
img: (props) => (
<WikiProtectedImage
{...props}
knowledgeBaseId={target?.knowledgeBaseId || ""}
/>
),
}}
linkClassName={(href) => linkClassName={(href) =>
parseWeKnoraPreviewUrl(href)?.wikiSlug parseWeKnoraPreviewUrl(href)?.wikiSlug
? "wiki-content-link font-medium text-[#be2323] decoration-[#be2323]/55 decoration-dashed hover:decoration-solid" ? "wiki-content-link font-medium text-[#be2323] decoration-[#be2323]/55 decoration-dashed hover:decoration-solid"
@@ -35,6 +35,7 @@ import { markdownToDocxBlob } from "@/core/scheduled-tasks/markdown-docx";
import { import {
exportableWikiMarkdown, exportableWikiMarkdown,
renderableWikiMarkdown, renderableWikiMarkdown,
rewriteProtectedWikiImageSources,
safeWikiFileName, safeWikiFileName,
stripWikiFrontmatter, stripWikiFrontmatter,
wikiLinkMarkdown, wikiLinkMarkdown,
@@ -49,6 +50,7 @@ import {
usePublicLlmWikiWikiPage, usePublicLlmWikiWikiPage,
usePublicLlmWikiWikiPages, usePublicLlmWikiWikiPages,
} from "@/strategy-components/api/llmwiki"; } from "@/strategy-components/api/llmwiki";
import { WikiProtectedImage } from "./WikiProtectedImage";
const LIST_PATH = "/page/workspace/knowledge"; const LIST_PATH = "/page/workspace/knowledge";
@@ -209,7 +211,7 @@ export function WeKnoraWikiPageView({
: pageData?.content, : pageData?.content,
)); ));
const displayMarkdown = pageData const displayMarkdown = pageData
? renderableWikiMarkdown( ? rewriteProtectedWikiImageSources(renderableWikiMarkdown(
title, title,
content, content,
noteId, noteId,
@@ -222,7 +224,7 @@ export function WeKnoraWikiPageView({
: undefined, : undefined,
publicEmbed ? pagesList : [], publicEmbed ? pagesList : [],
publicEmbed ? publicWikiIndex.data?.groups || [] : [], publicEmbed ? publicWikiIndex.data?.groups || [] : [],
) ))
: ""; : "";
const exportMarkdown = pageData const exportMarkdown = pageData
? wikiLinkMarkdown(exportableWikiMarkdown(title, summary, content), noteId) ? wikiLinkMarkdown(exportableWikiMarkdown(title, summary, content), noteId)
@@ -497,6 +499,15 @@ export function WeKnoraWikiPageView({
isLoading={false} isLoading={false}
remarkPlugins={reasoningPlugins.remarkPlugins} remarkPlugins={reasoningPlugins.remarkPlugins}
rehypePlugins={reasoningPlugins.rehypePlugins} rehypePlugins={reasoningPlugins.rehypePlugins}
components={{
img: (props) => (
<WikiProtectedImage
{...props}
knowledgeBaseId={noteId}
publicAccess={publicEmbed}
/>
),
}}
linkClassName={(href) => linkClassName={(href) =>
publicEmbed && publicWikiTargetFromHref(href, noteId) publicEmbed && publicWikiTargetFromHref(href, noteId)
? "wiki-content-link border-b border-dashed !border-[var(--primary)] font-medium !text-primary !no-underline hover:border-solid focus-visible:rounded-sm focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary" ? "wiki-content-link border-b border-dashed !border-[var(--primary)] font-medium !text-primary !no-underline hover:border-solid focus-visible:rounded-sm focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary"
@@ -0,0 +1,75 @@
import { ImageOff, Loader2 } from "lucide-react";
import { type ImgHTMLAttributes, useEffect, useState } from "react";
import { parseProtectedWikiImageSource } from "@/core/llmwiki/wiki-page-markdown";
import { getLlmWikiProtectedFile } from "@/strategy-components/api/llmwiki";
type WikiProtectedImageProps = ImgHTMLAttributes<HTMLImageElement> & {
knowledgeBaseId: string;
publicAccess?: boolean;
};
export function WikiProtectedImage({
knowledgeBaseId,
publicAccess = false,
src,
alt,
className,
...props
}: WikiProtectedImageProps) {
const filePath = parseProtectedWikiImageSource(src);
const [objectUrl, setObjectUrl] = useState("");
const [error, setError] = useState("");
useEffect(() => {
if (!filePath || !knowledgeBaseId) {
setObjectUrl("");
setError("");
return;
}
let cancelled = false;
let nextObjectUrl = "";
setObjectUrl("");
setError("");
void getLlmWikiProtectedFile(knowledgeBaseId, filePath, publicAccess)
.then((blob) => {
if (cancelled) return;
nextObjectUrl = URL.createObjectURL(blob);
setObjectUrl(nextObjectUrl);
})
.catch((reason: unknown) => {
if (!cancelled) {
setError(reason instanceof Error ? reason.message : "Wiki 图片加载失败");
}
});
return () => {
cancelled = true;
if (nextObjectUrl) URL.revokeObjectURL(nextObjectUrl);
};
}, [filePath, knowledgeBaseId, publicAccess]);
if (!filePath) {
return <img {...props} src={src} alt={alt} className={className} />;
}
if (error) {
return (
<span
className="my-3 flex min-h-24 items-center justify-center gap-2 rounded-lg border border-dashed bg-muted/40 px-4 text-sm text-muted-foreground"
data-wiki-image-error={error}
role="img"
aria-label={alt || "Wiki 图片加载失败"}
>
<ImageOff className="size-5" />
{alt || "Wiki 图片加载失败"}
</span>
);
}
if (!objectUrl) {
return (
<span className="my-3 flex min-h-24 items-center justify-center rounded-lg bg-muted/30 text-muted-foreground">
<Loader2 className="size-5 animate-spin" aria-label="正在加载 Wiki 图片" />
</span>
);
}
return <img {...props} src={objectUrl} alt={alt} className={className} />;
}
+13 -2
View File
@@ -1016,6 +1016,11 @@ Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` me
`llmwiki.weknora.api_base_url` is the provider switch: non-empty enables WeKnora, empty preserves the legacy LLMWiki UI. WeKnora is independently deployed and owns PostgreSQL/Redis/object storage/vector storage/parsers/models; DeerFlow config contains only `api_base_url` and optional `web_base_url`. DeerFlow deliberately sends no WeKnora credential, so the WeKnora endpoint must be restricted to a trusted network and allow DeerFlow traffic itself or through its reverse proxy. There is intentionally no `default_embedding_model_id`: `WeKnoraClient` queries `/api/v1/models` and uses WeKnora's active Embedding model when creating a KB; Wiki creation additionally resolves an active KnowledgeQA model inside WeKnora for synthesis. The native BFF is `app/gateway/routers/llmwiki.py`; ownership/publication mappings live in `deerflow.persistence.llmwiki`; the browser only sees DeerFlow mapping IDs. Knowledge-base owners publish immediately (`private` -> `published`) without a review queue, and can unpublish ordinary bases. The conversation-deposit KB is system-owned, always published, shared by all DeerFlow users, excluded from My Knowledge, and immutable through user/admin detail operations; recognition requires both the reserved name and `system` owner. Existing same-name remote mappings are adopted in place as the system deposit, preserving mapping ids and document links. Its user-facing description, generated Markdown source and tags use `cmzs` branding; list/detail reads synchronize stale remote descriptions, and search results, source previews and embedded-detail JSON recursively replace legacy brand text in historical conversation-deposit payloads. Deposit requests must reference a thread accessible to the current account. Personal knowledge-base list/search views are scoped to the current DeerFlow account even for admins; frontend LLMWiki caches include the live account id, and admin write power does not make other users' private KBs appear as owned. Only `system_role === "admin"` sees the WeKnora connection-config tab. Knowledge cards navigate to a dedicated WeKnora-style detail route. Wiki-enabled bases render and default to `Wiki -> documents -> graph`; standard bases omit Wiki and render `documents -> graph`. The detail page supports sequential multi-file upload with an aggregate result for writable ordinary bases, hides it for public read-only/system-deposit bases, and uses a generic loading label while the embed session starts. It also supports URL and published manual-Markdown imports, browses real Wiki pages/content, proxies authenticated original-file previews, pages through `/chunks/{knowledge_id}`, and renders the Wiki page graph from `/knowledgebase/{kb_id}/wiki/graph`; document details use a right sheet rather than the removed KB modal. The credential-free proxy uses one WeKnora service identity, so per-user separation is enforced by DeerFlow mappings while all remote KBs physically share that identity's WeKnora workspace. WeKnora iframe detail pages first call `/api/llmwiki/knowledge-bases/{id}/weknora/session` through the normal DeerFlow API auth path, then load `/platform/knowledge-bases/{weknora_id}` with a signed `deerflow_weknora_embed` cookie for `/platform`, `/assets`, `/locales`, root assets, and API calls. Injected fetch/XHR rewriting sends all iframe `/api/v1/*`, including colliding `/api/v1/auth/*`, through `/api/llmwiki/weknora-embed/api/v1/*`; Gateway auth/CSRF bypass only applies to those signed iframe proxy requests, the proxy still revalidates the mapped KB, and cookie security honors `X-Forwarded-Proto` for HTTPS reverse proxies. `llmwiki_search` is conditionally registered in `tools.py`, is read-only, and rechecks the current DeerFlow user, agent `llmwiki_knowledge_base_ids`, and thread-context selection on every call. Source previews prefer the cleaned Wiki/LLMWiki page text when a retrieved chunk maps to a Wiki page, with raw split chunks as the fallback. Agent bindings use an inline WeKnora-style searchable/grouped multi-select in the standard agent create/detail UI; new chats use the same interaction pattern as a compact composer dropdown, and selections are written to thread metadata/context and restored on reopen. The DeerFlow frontend is native React and the WeKnora frontend is not modified or embedded. Full deployment and behavior notes: `docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`. Tests: `tests/test_llmwiki_weknora.py`. `llmwiki.weknora.api_base_url` is the provider switch: non-empty enables WeKnora, empty preserves the legacy LLMWiki UI. WeKnora is independently deployed and owns PostgreSQL/Redis/object storage/vector storage/parsers/models; DeerFlow config contains only `api_base_url` and optional `web_base_url`. DeerFlow deliberately sends no WeKnora credential, so the WeKnora endpoint must be restricted to a trusted network and allow DeerFlow traffic itself or through its reverse proxy. There is intentionally no `default_embedding_model_id`: `WeKnoraClient` queries `/api/v1/models` and uses WeKnora's active Embedding model when creating a KB; Wiki creation additionally resolves an active KnowledgeQA model inside WeKnora for synthesis. The native BFF is `app/gateway/routers/llmwiki.py`; ownership/publication mappings live in `deerflow.persistence.llmwiki`; the browser only sees DeerFlow mapping IDs. Knowledge-base owners publish immediately (`private` -> `published`) without a review queue, and can unpublish ordinary bases. The conversation-deposit KB is system-owned, always published, shared by all DeerFlow users, excluded from My Knowledge, and immutable through user/admin detail operations; recognition requires both the reserved name and `system` owner. Existing same-name remote mappings are adopted in place as the system deposit, preserving mapping ids and document links. Its user-facing description, generated Markdown source and tags use `cmzs` branding; list/detail reads synchronize stale remote descriptions, and search results, source previews and embedded-detail JSON recursively replace legacy brand text in historical conversation-deposit payloads. Deposit requests must reference a thread accessible to the current account. Personal knowledge-base list/search views are scoped to the current DeerFlow account even for admins; frontend LLMWiki caches include the live account id, and admin write power does not make other users' private KBs appear as owned. Only `system_role === "admin"` sees the WeKnora connection-config tab. Knowledge cards navigate to a dedicated WeKnora-style detail route. Wiki-enabled bases render and default to `Wiki -> documents -> graph`; standard bases omit Wiki and render `documents -> graph`. The detail page supports sequential multi-file upload with an aggregate result for writable ordinary bases, hides it for public read-only/system-deposit bases, and uses a generic loading label while the embed session starts. It also supports URL and published manual-Markdown imports, browses real Wiki pages/content, proxies authenticated original-file previews, pages through `/chunks/{knowledge_id}`, and renders the Wiki page graph from `/knowledgebase/{kb_id}/wiki/graph`; document details use a right sheet rather than the removed KB modal. The credential-free proxy uses one WeKnora service identity, so per-user separation is enforced by DeerFlow mappings while all remote KBs physically share that identity's WeKnora workspace. WeKnora iframe detail pages first call `/api/llmwiki/knowledge-bases/{id}/weknora/session` through the normal DeerFlow API auth path, then load `/platform/knowledge-bases/{weknora_id}` with a signed `deerflow_weknora_embed` cookie for `/platform`, `/assets`, `/locales`, root assets, and API calls. Injected fetch/XHR rewriting sends all iframe `/api/v1/*`, including colliding `/api/v1/auth/*`, through `/api/llmwiki/weknora-embed/api/v1/*`; Gateway auth/CSRF bypass only applies to those signed iframe proxy requests, the proxy still revalidates the mapped KB, and cookie security honors `X-Forwarded-Proto` for HTTPS reverse proxies. `llmwiki_search` is conditionally registered in `tools.py`, is read-only, and rechecks the current DeerFlow user, agent `llmwiki_knowledge_base_ids`, and thread-context selection on every call. Source previews prefer the cleaned Wiki/LLMWiki page text when a retrieved chunk maps to a Wiki page, with raw split chunks as the fallback. Agent bindings use an inline WeKnora-style searchable/grouped multi-select in the standard agent create/detail UI; new chats use the same interaction pattern as a compact composer dropdown, and selections are written to thread metadata/context and restored on reopen. The DeerFlow frontend is native React and the WeKnora frontend is not modified or embedded. Full deployment and behavior notes: `docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`. Tests: `tests/test_llmwiki_weknora.py`.
The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. Do not add an HTML `sandbox` attribute to this trusted, permission-filtered detail iframe: a sandboxed ancestor causes Chromium's built-in Blob PDF viewer to render the browser-blocked page. The Gateway follows upstream redirects only for `/knowledge/{id}/preview` and `/knowledge-bases/{id}/files`, keeping PDF bytes and protected Wiki images inside the signed proxy instead of sending the browser to WeKnora object storage. The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. Do not add an HTML `sandbox` attribute to this trusted, permission-filtered detail iframe: a sandboxed ancestor causes Chromium's built-in Blob PDF viewer to render the browser-blocked page. The Gateway follows upstream redirects only for `/knowledge/{id}/preview` and `/knowledge-bases/{id}/files`, keeping PDF bytes and protected Wiki images inside the signed proxy instead of sending the browser to WeKnora object storage.
Native/private and public Wiki readers must rewrite protected Markdown image
sources to their mapping-scoped DeerFlow `/files?file_path=` endpoints and load
them as Blob URLs. The embedded guard also rewrites protected fetch/XHR values
and hydrates late `img[data-protected-src]` placeholders; never expose the
service token or relax the current-KB authorization to make images work.
**Embedded native knowledge-base switching:** The original WeKnora breadcrumb picker remains available inside the detail iframe. A signed 15-minute embed cookie carries only the current DeerFlow user's visible mappings (their own plus published bases); the proxied `/api/v1/knowledge-bases` response is filtered to that allowlist. Choosing another entry performs a full iframe navigation, rechecks the DeerFlow mapping and read/write permission, then reissues the cookie for the selected base. Never expose the upstream service-admin list directly or permit an arbitrary remote knowledge-base ID. **Embedded native knowledge-base switching:** The original WeKnora breadcrumb picker remains available inside the detail iframe. A signed 15-minute embed cookie carries only the current DeerFlow user's visible mappings (their own plus published bases); the proxied `/api/v1/knowledge-bases` response is filtered to that allowlist. Choosing another entry performs a full iframe navigation, rechecks the DeerFlow mapping and read/write permission, then reissues the cookie for the selected base. Never expose the upstream service-admin list directly or permit an arbitrary remote knowledge-base ID.
@@ -1442,8 +1447,10 @@ detail. Never call `/knowledge-search` from internal search, RAG, or the
`deerflow.integrations.weknora.local_index` and `deerflow.integrations.weknora.local_index` and
`deerflow.persistence.llmwiki_index`; preserve the harness → app import `deerflow.persistence.llmwiki_index`; preserve the harness → app import
firewall. Gateway composition injects the store/search/sync services through firewall. Gateway composition injects the store/search/sync services through
`local_index.runtime`, and the lease-protected scheduler starts only when both `local_index.runtime`. `enabled` turns on local vector search and manual
`enabled` and `sync_enabled` are true. vectorization; `auto_sync` is the separate opt-in for the lease-protected
scheduler. The scheduler sleeps for one full configured interval before its
first scan, so application startup must never trigger immediate vectorization.
Vectors are normalized little-endian float32 BLOBs. A page mirror and all of Vectors are normalized little-endian float32 BLOBs. A page mirror and all of
its vectors are replaced in one transaction only after strict batched its vectors are replaced in one transaction only after strict batched
@@ -1456,6 +1463,10 @@ input, not an outbound API field: fixed-dimension BGE/OpenAI-compatible
endpoints can reject a `dimensions` request parameter with HTTP 400. When the endpoints can reject a `dimensions` request parameter with HTTP 400. When the
value is omitted, the embedding client probes and freezes the first response's value is omitted, the embedding client probes and freezes the first response's
dimension before sync lease/rebuild decisions. dimension before sync lease/rebuild decisions.
Embedding errors must retain the upstream status and bounded response body in
logs, without exposing API keys. SQL vector writes are split into statements
of at most `database_batch_size` rows (default 50) inside the same atomic page
replacement transaction.
During a fingerprint rebuild the mapping uses the Wiki-page API until every During a fingerprint rebuild the mapping uses the Wiki-page API until every
page is successfully rebuilt. `WikiRetrievalService` partitions mixed requests page is successfully rebuilt. `WikiRetrievalService` partitions mixed requests
per mapping, so ready libraries stay local while incomplete libraries use that per mapping, so ready libraries stay local while incomplete libraries use that
+19 -3
View File
@@ -777,15 +777,24 @@ processed Wiki content. A knowledge base uses exact NumPy cosine similarity
only after its whole Wiki has completed one coherent vectorization pass; until only after its whole Wiki has completed one coherent vectorization pass; until
then internal search calls WeKnora's Wiki-page search/list API and reads the then internal search calls WeKnora's Wiki-page search/list API and reads the
matched Wiki pages. It never falls back to raw documents or chunks. The local matched Wiki pages. It never falls back to raw documents or chunks. The local
vector feature is disabled by default. Configure vector feature is disabled by default. Configure the three
`llmwiki.local_wiki_index.embedding` first, then enable `enabled`, `llmwiki.local_wiki_index.embedding` values (`model`, `base_url`, and
`sync_enabled`, and `internal_search_enabled` in `config.yaml`. `api_key`), then change only `enabled` to `true` in `config.yaml`. Manual
vectorization is the default. Set `auto_sync: true` only when periodic
background synchronization is wanted; the scheduler waits one full interval
after startup before its first scan, so boot does not trigger a rebuild.
`database_batch_size` caps each vector insert statement (default `50`) while
the page replacement remains atomic, preventing a large Wiki from flooding
the SQL connection in one statement.
The optional `embedding.dimensions` value is used only to validate the returned The optional `embedding.dimensions` value is used only to validate the returned
vector size and build the index fingerprint; it is deliberately not sent in vector size and build the index fingerprint; it is deliberately not sent in
the embedding request because fixed-dimension BGE/OpenAI-compatible endpoints the embedding request because fixed-dimension BGE/OpenAI-compatible endpoints
may reject that extra field with HTTP 400. Omit it to discover the dimension may reject that extra field with HTTP 400. Omit it to discover the dimension
from the first successful response. from the first successful response.
Embedding failures log the upstream HTTP status and a bounded response body.
The client never sends a `dimensions` request parameter.
The migration creates `llmwiki_wiki_pages`, `llmwiki_wiki_vectors`, and The migration creates `llmwiki_wiki_pages`, `llmwiki_wiki_vectors`, and
`llmwiki_wiki_sync_states`. Wiki Markdown and normalized float32 vectors are `llmwiki_wiki_sync_states`. Wiki Markdown and normalized float32 vectors are
stored in DeerFlow's configured SQL database. Multi-worker sync is protected by stored in DeerFlow's configured SQL database. Multi-worker sync is protected by
@@ -839,6 +848,13 @@ system conversation-deposit knowledge base. Private or unpublished bases are
never exposed by this route. Anonymous consumers can discover the available never exposed by this route. Anonymous consumers can discover the available
public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each
entry deliberately contains only the DeerFlow mapping `id` and display `name`. entry deliberately contains only the DeerFlow mapping `id` and display `name`.
Protected `resource://` and supported object-storage images are fetched through
the current mapping instead of being exposed as upstream URLs. Authenticated
readers use `GET /api/llmwiki/knowledge-bases/{id}/files?file_path=...`; public
readers use the corresponding `/api/public/llmwiki/...` route, which requires a
published mapping. The embedded WeKnora guard applies the same mapping-scoped
proxy to document previews and Markdown images, including images inserted after
the drawer/page first renders.
Wiki management also provides **导出 Excel**. `GET Wiki management also provides **导出 Excel**. `GET
/api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx` exports every processed Wiki /api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx` exports every processed Wiki
@@ -998,7 +998,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
local_wiki_index = app.state.config.llmwiki.local_wiki_index local_wiki_index = app.state.config.llmwiki.local_wiki_index
if ( if (
local_wiki_index.enabled local_wiki_index.enabled
and local_wiki_index.sync_enabled and local_wiki_index.auto_sync
and getattr(app.state, "llmwiki_sync_service", None) is not None and getattr(app.state, "llmwiki_sync_service", None) is not None
): ):
from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop
@@ -321,13 +321,16 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
from deerflow.persistence.llmwiki_index import make_llmwiki_index_store from deerflow.persistence.llmwiki_index import make_llmwiki_index_store
app.state.llmwiki_index_store = make_llmwiki_index_store(sf) local_index_config = config.llmwiki.local_wiki_index
app.state.llmwiki_index_store = make_llmwiki_index_store(
sf,
write_batch_size=local_index_config.database_batch_size,
)
app.state.llmwiki_embedding = None app.state.llmwiki_embedding = None
app.state.llmwiki_vector_cache = None app.state.llmwiki_vector_cache = None
app.state.llmwiki_vector_search = None app.state.llmwiki_vector_search = None
app.state.llmwiki_sync_service = None app.state.llmwiki_sync_service = None
app.state.llmwiki_retrieval_service = None app.state.llmwiki_retrieval_service = None
local_index_config = config.llmwiki.local_wiki_index
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
resolved_llmwiki = get_resolved_llmwiki_runtime(config) resolved_llmwiki = get_resolved_llmwiki_runtime(config)
@@ -48,6 +48,10 @@ async def _sync_once(app: FastAPI) -> None:
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None: async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds
logger.info(
"Local Wiki auto-sync scheduler started; first scan delayed by %s seconds",
interval,
)
while True: while True:
await _sync_once(app)
await asyncio.sleep(interval) await asyncio.sleep(interval)
await _sync_once(app)
@@ -61,6 +61,10 @@ router = APIRouter(tags=["llmwiki"])
proxy_router = APIRouter(tags=["llmwiki-proxy"]) proxy_router = APIRouter(tags=["llmwiki-proxy"])
_MAX_UPLOAD_BYTES = 100 * 1024 * 1024 _MAX_UPLOAD_BYTES = 100 * 1024 * 1024
_PROTECTED_WIKI_FILE_PATH = re.compile(
r"^(?:storage://[0-9A-Za-z_-]+/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs)://",
re.IGNORECASE,
)
_WEKNORA_ADMIN_SESSION_CACHE: dict[str, dict[str, Any]] = {} _WEKNORA_ADMIN_SESSION_CACHE: dict[str, dict[str, Any]] = {}
_WEKNORA_PUBLIC_PREFIX = "/deerflow" _WEKNORA_PUBLIC_PREFIX = "/deerflow"
_WEKNORA_PUBLIC_ROOT_PATHS = ( _WEKNORA_PUBLIC_ROOT_PATHS = (
@@ -242,7 +246,7 @@ def _store(request: Request) -> LlmWikiStore:
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None: def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
service = getattr(request.app.state, "llmwiki_sync_service", None) service = getattr(request.app.state, "llmwiki_sync_service", None)
config = request.app.state.config.llmwiki.local_wiki_index config = request.app.state.config.llmwiki.local_wiki_index
if service is not None and config.enabled and mapping.get("wiki_index_enabled", True): if service is not None and config.enabled and config.auto_sync and mapping.get("wiki_index_enabled", True):
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True)) spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
@@ -1085,6 +1089,8 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
function rewriteApiUrl(value) {{ function rewriteApiUrl(value) {{
try {{ try {{
const protectedUrl = protectedImageApiUrl(value);
if (protectedUrl) return protectedUrl;
const next = new URL(value, location.origin); const next = new URL(value, location.origin);
if (next.origin !== location.origin) return value; if (next.origin !== location.origin) return value;
const publicPrefix = cfg.publicPrefix || ""; const publicPrefix = cfg.publicPrefix || "";
@@ -1104,6 +1110,19 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
return value; return value;
}} }}
}} }}
function protectedImageApiUrl(value) {{
const raw = String(value || "").trim();
if (!/^(?:storage:\\/\\/[0-9A-Za-z_-]+\\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\\/\\//i.test(raw)) {{
return "";
}}
const endpoint =
cfg.apiPrefix +
"/api/v1/knowledge-bases/" +
encodeURIComponent(cfg.weknoraId) +
"/files?file_path=" +
encodeURIComponent(raw);
return new URL(endpoint, location.origin).href;
}}
const nativeFetch = window.fetch.bind(window); const nativeFetch = window.fetch.bind(window);
window.fetch = (input, init) => {{ window.fetch = (input, init) => {{
if (input instanceof Request) {{ if (input instanceof Request) {{
@@ -1116,6 +1135,67 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
return nativeXhrOpen.call(this, method, rewriteApiUrl(url), ...rest); return nativeXhrOpen.call(this, method, rewriteApiUrl(url), ...rest);
}}; }};
// WeKnora renders protected Markdown images as transparent placeholders and
// hydrates them asynchronously. In an embedded page that hydration can run
// before its own API client has picked up our proxy prefix, leaving a 1x1
// GIF forever. Resolve the same protected source through the scoped proxy
// as a fallback. The backend validates the scheme and current KB again.
function hydrateProtectedImage(image) {{
const protectedSrc = String(image?.getAttribute?.("data-protected-src") || "").trim();
const proxyUrl = protectedImageApiUrl(protectedSrc);
if (!proxyUrl) return;
if (
image.dataset.deerflowProtectedSource === protectedSrc &&
String(image.getAttribute("src") || "").startsWith("blob:")
) return;
const attempts = Number(image.dataset.deerflowProtectedAttempts || "0");
if (attempts >= 3 || image.dataset.deerflowProtectedLoading === "1") return;
image.dataset.deerflowProtectedLoading = "1";
image.dataset.deerflowProtectedAttempts = String(attempts + 1);
nativeFetch(proxyUrl, {{ credentials: "same-origin", headers: {{ Accept: "image/*" }} }})
.then((response) => {{
if (!response.ok) throw new Error(`protected image request failed (${{response.status}})`);
return response.blob();
}})
.then((blob) => {{
const objectUrl = URL.createObjectURL(blob);
image.dataset.deerflowProtectedSource = protectedSrc;
image.removeAttribute("data-img-loading");
image.removeAttribute("data-wiki-image-error");
image.src = objectUrl;
image.addEventListener("load", () => URL.revokeObjectURL(objectUrl), {{ once: true }});
}})
.catch(() => {{
image.dataset.wikiImageError = "true";
window.setTimeout(() => {{
delete image.dataset.deerflowProtectedLoading;
hydrateProtectedImage(image);
}}, 1200 * (attempts + 1));
}})
.finally(() => {{
delete image.dataset.deerflowProtectedLoading;
}});
}}
function hydrateProtectedImages(scope = document) {{
if (scope?.matches?.("img[data-protected-src]")) hydrateProtectedImage(scope);
scope?.querySelectorAll?.("img[data-protected-src]").forEach(hydrateProtectedImage);
}}
const protectedImageObserver = new MutationObserver((mutations) => {{
mutations.forEach((mutation) => {{
if (mutation.type === "attributes") hydrateProtectedImage(mutation.target);
mutation.addedNodes?.forEach((node) => {{
if (node?.nodeType === Node.ELEMENT_NODE) hydrateProtectedImages(node);
}});
}});
}});
protectedImageObserver.observe(document.documentElement, {{
childList: true,
subtree: true,
attributes: true,
attributeFilter: ["data-protected-src"]
}});
hydrateProtectedImages();
function tone(color, alpha) {{ function tone(color, alpha) {{
return `color-mix(in srgb, ${{color}} ${{Math.round(alpha * 100)}}%, transparent)`; return `color-mix(in srgb, ${{color}} ${{Math.round(alpha * 100)}}%, transparent)`;
}} }}
@@ -1519,6 +1599,7 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
enforceLocation(); enforceLocation();
purgeGuides(); purgeGuides();
hideWeKnoraShell(); hideWeKnoraShell();
hydrateProtectedImages();
if (tickSourceDetail() && sourceDetailMode) {{ if (tickSourceDetail() && sourceDetailMode) {{
clearInterval(maintenanceTimer); clearInterval(maintenanceTimer);
try {{ readyObserver.disconnect(); }} catch (_) {{}} try {{ readyObserver.disconnect(); }} catch (_) {{}}
@@ -1820,6 +1901,32 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
return row return row
def _validate_protected_wiki_file_path(file_path: str) -> str:
value = file_path.strip()
if not value or "\x00" in value or not _PROTECTED_WIKI_FILE_PATH.match(value):
raise HTTPException(status_code=422, detail="Unsupported Wiki image path")
return value
async def _wiki_file_response(row: dict[str, Any], file_path: str, *, public: bool) -> Response:
try:
content, media_type = await _client_or_503().get_knowledge_base_file(
str(row["weknora_id"]),
_validate_protected_wiki_file_path(file_path),
)
except WeKnoraError as exc:
_raise_upstream(exc)
return Response(
content=content,
media_type=media_type,
headers={
"Cache-Control": "public, max-age=300" if public else "private, max-age=60",
"Content-Disposition": "inline",
"X-Content-Type-Options": "nosniff",
},
)
async def _verify_document(client: WeKnoraClient, row: dict[str, Any], document_id: str) -> dict[str, Any]: async def _verify_document(client: WeKnoraClient, row: dict[str, Any], document_id: str) -> dict[str, Any]:
document = await client.get_document(document_id) document = await client.get_document(document_id)
if str(document.get("knowledge_base_id") or "") != str(row["weknora_id"]): if str(document.get("knowledge_base_id") or "") != str(row["weknora_id"]):
@@ -2510,6 +2617,18 @@ async def list_public_wiki_pages(
return {**result, "pages": pages} return {**result, "pages": pages}
@router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}/files")
async def get_public_wiki_file(
request: Request,
mapping_id: str,
file_path: str = Query(..., min_length=1, max_length=8192),
) -> Response:
"""Serve one protected image belonging to a published Wiki."""
row = await _public_mapping(request, mapping_id)
return await _wiki_file_response(row, file_path, public=True)
_PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison") _PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison")
@@ -2820,6 +2939,18 @@ async def preview_document(request: Request, mapping_id: str, document_id: str)
) )
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/files")
async def get_wiki_file(
request: Request,
mapping_id: str,
file_path: str = Query(..., min_length=1, max_length=8192),
) -> Response:
"""Serve one protected Wiki image after normal knowledge-base authorization."""
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
return await _wiki_file_response(row, file_path, public=False)
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph") @router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph")
async def get_knowledge_base_graph( async def get_knowledge_base_graph(
request: Request, request: Request,
@@ -2,6 +2,7 @@
from __future__ import annotations from __future__ import annotations
import logging
from typing import Any from typing import Any
from uuid import uuid4 from uuid import uuid4
@@ -14,6 +15,7 @@ from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectori
from deerflow.integrations.weknora.local_index.search import WikiIndexError from deerflow.integrations.weknora.local_index.search import WikiIndexError
router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"]) router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"])
logger = logging.getLogger(__name__)
class WikiVectorSearchRequest(BaseModel): class WikiVectorSearchRequest(BaseModel):
@@ -181,6 +183,7 @@ async def _run_sync(app, mapping_ids: list[str], force: bool) -> None:
try: try:
await service.sync_mapping(mapping, force=force) await service.sync_mapping(mapping, force=force)
except Exception: except Exception:
logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id"))
continue continue
@@ -191,6 +194,7 @@ async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
try: try:
await service.sync_mapping(mapping, force=force) await service.sync_mapping(mapping, force=force)
except Exception: except Exception:
logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id"))
return return
+5 -35
View File
@@ -556,47 +556,17 @@ llmwiki:
web_base_url: "http://119.254.86.251:8088" web_base_url: "http://119.254.86.251:8088"
admin_email: "lqq@qq.com" admin_email: "lqq@qq.com"
admin_password: "1qaz@WSX" admin_password: "1qaz@WSX"
# DeerFlow-local Wiki mirror/vector index. Keep disabled until the # 本地 Wiki 向量化:只需把 enabled 改为 true,并填写下面 3 个 embedding 值。
# OpenAI-compatible embedding endpoint below is configured and reachable. # 默认只在页面点击“向量化”时执行;auto_sync=true 才会后台自动增量同步,
# 且服务启动时只启动定时器,不会立刻全量向量化。
local_wiki_index: local_wiki_index:
enabled: false enabled: false
sync_enabled: false auto_sync: false
internal_search_enabled: false database_batch_size: 50
reference_ui_enabled: false
sync_interval_seconds: 120
chunk_max_chars: 1000
chunk_overlap_chars: 120
chunk_min_chars: 80
embedding: embedding:
enabled: true
provider: openai
model: bge-m3 model: bge-m3
base_url: "" base_url: ""
api_key: "" api_key: ""
# Expected BGE-M3 response size for local validation/indexing only.
# This value is not sent as an embedding API request parameter.
dimensions: 1024
batch_size: 32
timeout_seconds: 30
max_retries: 3
engine: numpy_exact
strict_vector: true
top_k_sections: 30
top_k_pages: 8
max_sections_per_page: 2
min_similarity: 0.35
cache_max_knowledge_bases: 20
cache_revision_check_seconds: 5
external_api:
enabled: false
api_key: ""
previous_api_key: ""
max_query_chars: 1000
max_knowledge_bases: 20
max_results: 50
default_results: 10
include_content_max_chars: 12000
requests_per_minute: 60
log_system: log_system:
enabled: true enabled: true
@@ -279,65 +279,25 @@ llmwiki:
local_wiki_index: local_wiki_index:
enabled: true enabled: true
# 默认只允许手动向量化;需要定时增量同步时再开启。
# 同步 auto_sync: false
sync_enabled: true # 每条 SQL 最多插入的向量行数;整页替换仍在同一事务内完成。
sync_interval_seconds: 120 database_batch_size: 50
sync_page_size: 100
sync_concurrency: 3
sync_lease_seconds: 600
sync_request_timeout_seconds: 30
sync_max_retries: 3
sync_retry_backoff_seconds: 5
# 切分
chunk_max_chars: 1000
chunk_overlap_chars: 120
chunk_min_chars: 80
title_prefix_enabled: true
# Embedding:OpenAI-compatible /v1/embeddings
embedding: embedding:
enabled: true
provider: openai
model: bge-m3 model: bge-m3
base_url: "$WIKI_EMBEDDING_BASE_URL" base_url: "$WIKI_EMBEDDING_BASE_URL"
api_key: "$WIKI_EMBEDDING_API_KEY" api_key: "$WIKI_EMBEDDING_API_KEY"
dimensions: 1024
batch_size: 32
timeout_seconds: 30
max_retries: 3
# 检索
engine: numpy_exact
strict_vector: true
top_k_sections: 30
top_k_pages: 8
max_sections_per_page: 2
min_similarity: 0.35
keyword_boost_enabled: false
keyword_boost_weight: 0.15
cache_max_knowledge_bases: 20
cache_revision_check_seconds: 5
# 外部接口
external_api:
enabled: false
api_key: "$LLMWIKI_EXTERNAL_SEARCH_API_KEY"
max_query_chars: 1000
max_knowledge_bases: 20
max_results: 50
default_results: 10
include_content_max_chars: 12000
requests_per_minute: 60
``` ```
日常部署只需配置上面三个 Embedding 值并将 `enabled` 改为 `true`。
`auto_sync=false` 时仅由页面“手动向量化”触发;即使设为 `true`,定时器也会先等待一个完整同步周期,不会在服务刚启动时立即重建索引。其余切分、检索、超时与重试参数继续使用代码默认值,只有确有调优需要时才覆盖。
### 6.1 配置校验规则 ### 6.1 配置校验规则
启动时应执行: 启动时应执行:
1. `enabled=true` 时,Embedding `base_url` 和 `model` 必须非空; 1. `enabled=true` 时,Embedding `base_url` 和 `model` 必须非空;
2. `dimensions` 可省略,但第一次成功 Embedding 后必须固定并写入索引 profile; 2. 不向上游发送 `dimensions` 参数;第一次成功 Embedding 后从响应自动发现维度,并固定到索引 profile;
3. `chunk_overlap_chars < chunk_max_chars`; 3. `chunk_overlap_chars < chunk_max_chars`;
4. `top_k_pages <= top_k_sections`; 4. `top_k_pages <= top_k_sections`;
5. `external_api.enabled=true` 时必须配置非空 API Key; 5. `external_api.enabled=true` 时必须配置非空 API Key;
@@ -1885,9 +1845,7 @@ class WikiVectorEngine(ABC):
```text ```text
local_wiki_index.enabled local_wiki_index.enabled
local_wiki_index.sync_enabled local_wiki_index.auto_sync
local_wiki_index.internal_search_enabled
local_wiki_index.reference_ui_enabled
local_wiki_index.external_api.enabled local_wiki_index.external_api.enabled
``` ```
@@ -59,7 +59,7 @@ class LocalWikiEmbeddingConfig(BaseModel):
base_url: str = "" base_url: str = ""
api_key: str = Field(default="", repr=False) api_key: str = Field(default="", repr=False)
dimensions: int | None = Field( dimensions: int | None = Field(
default=1024, default=None,
ge=1, ge=1,
le=65536, le=65536,
description="Expected response vector dimensions; not sent to the embedding endpoint.", description="Expected response vector dimensions; not sent to the embedding endpoint.",
@@ -108,9 +108,7 @@ class LocalWikiIndexConfig(BaseModel):
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages.""" """DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
enabled: bool = False enabled: bool = False
sync_enabled: bool = False auto_sync: bool = False
internal_search_enabled: bool = False
reference_ui_enabled: bool = False
sync_interval_seconds: int = Field(default=120, ge=10, le=86400) sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
sync_page_size: int = Field(default=100, ge=1, le=500) sync_page_size: int = Field(default=100, ge=1, le=500)
sync_concurrency: int = Field(default=3, ge=1, le=32) sync_concurrency: int = Field(default=3, ge=1, le=32)
@@ -133,9 +131,31 @@ class LocalWikiIndexConfig(BaseModel):
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1) keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000) cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600) cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
database_batch_size: int = Field(default=50, ge=1, le=500)
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig) external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
model_config = ConfigDict(extra="ignore") model_config = ConfigDict(extra="ignore")
@model_validator(mode="before")
@classmethod
def _migrate_legacy_flags(cls, value):
if isinstance(value, dict) and "auto_sync" not in value and "sync_enabled" in value:
value = {**value, "auto_sync": value.get("sync_enabled")}
return value
@property
def sync_enabled(self) -> bool:
"""Backward-compatible status name; automatic work is one explicit switch."""
return self.auto_sync
@property
def internal_search_enabled(self) -> bool:
return self.enabled
@property
def reference_ui_enabled(self) -> bool:
return self.enabled
@model_validator(mode="after") @model_validator(mode="after")
def _validate_index(self): def _validate_index(self):
if self.chunk_overlap_chars >= self.chunk_max_chars: if self.chunk_overlap_chars >= self.chunk_max_chars:
@@ -2,8 +2,8 @@
from __future__ import annotations from __future__ import annotations
from collections.abc import Mapping, Sequence
import time import time
from collections.abc import Mapping, Sequence
from typing import Any from typing import Any
from urllib.parse import quote from urllib.parse import quote
@@ -430,6 +430,51 @@ class WeKnoraClient:
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip() media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
return response.content, media_type or "application/octet-stream" return response.content, media_type or "application/octet-stream"
async def get_knowledge_base_file(
self,
knowledge_base_id: str,
file_path: str,
) -> tuple[bytes, str]:
"""Return one protected Wiki attachment using DeerFlow's WeKnora identity."""
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=True,
trust_env=False,
)
try:
for attempt in range(2):
response = await client.request(
"GET",
f"{self.base_url}/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/files",
params={"file_path": file_path},
headers=await self._headers(force_refresh=attempt > 0, accept="*/*"),
follow_redirects=True,
)
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
break
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora Wiki image request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora Wiki image is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code == 404:
raise WeKnoraError("WeKnora Wiki image was not found", status_code=404)
if response.status_code in {401, 403}:
if self._has_admin_credentials:
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated Wiki image request", status_code=502)
raise WeKnoraError(
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
status_code=502,
)
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502)
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
return response.content, media_type or "application/octet-stream"
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]: async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse")) value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
return dict(value) if isinstance(value, dict) else {} return dict(value) if isinstance(value, dict) else {}
@@ -15,6 +15,21 @@ from deerflow.integrations.weknora.local_index.vector_codec import normalize_vec
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def _upstream_error_detail(response: httpx.Response) -> str:
"""Return a bounded upstream reason without hiding the useful response body."""
try:
payload = response.json()
if isinstance(payload, dict):
detail = payload.get("detail") or payload.get("message") or payload.get("error") or payload
else:
detail = payload
value = str(detail)
except (ValueError, TypeError):
value = response.text
return " ".join(value.split())[:2000] or "no response body"
class WikiEmbeddingError(RuntimeError): class WikiEmbeddingError(RuntimeError):
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None: def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
super().__init__(message) super().__init__(message)
@@ -76,11 +91,15 @@ class StrictWikiEmbeddingClient:
started = monotonic() started = monotonic()
try: try:
response = await (await self._get_client()).post(url, headers=headers, json=payload) response = await (await self._get_client()).post(url, headers=headers, json=payload)
if response.status_code in {401, 403}: if response.status_code >= 400:
raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False) detail = _upstream_error_detail(response)
if response.status_code == 429 or response.status_code >= 500: category = "authentication" if response.status_code in {401, 403} else "upstream"
raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True) retryable = response.status_code == 429 or response.status_code >= 500
response.raise_for_status() raise WikiEmbeddingError(
f"Embedding service returned HTTP {response.status_code}: {detail}",
category=category,
retryable=retryable,
)
data = response.json() data = response.json()
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else [] items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
if len(items) != len(texts): if len(items) != len(texts):
@@ -98,8 +117,20 @@ class StrictWikiEmbeddingClient:
except httpx.TimeoutException: except httpx.TimeoutException:
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True) last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
except (httpx.HTTPError, ValueError, TypeError) as exc: except (httpx.HTTPError, ValueError, TypeError) as exc:
last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError)) reason = " ".join(str(exc).split())[:2000] or type(exc).__name__
last_error = WikiEmbeddingError(
f"Embedding request failed ({type(exc).__name__}): {reason}",
category="request",
retryable=isinstance(exc, httpx.HTTPError),
)
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries): if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
logger.error(
"Wiki embedding batch failed model=%s batch=%d category=%s reason=%s",
self.config.model,
len(texts),
last_error.category,
last_error,
)
raise last_error raise last_error
await asyncio.sleep(min(30.0, 0.5 * (2**attempt))) await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
raise last_error or WikiEmbeddingError("Embedding request failed") raise last_error or WikiEmbeddingError("Embedding request failed")
@@ -6,8 +6,12 @@ from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWiki
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
def make_llmwiki_index_store(session_factory): def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory) return (
MemoryLlmWikiIndexStore()
if session_factory is None
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
)
__all__ = [ __all__ = [
@@ -6,7 +6,7 @@ from datetime import UTC, datetime, timedelta
from typing import Any from typing import Any
from uuid import uuid4 from uuid import uuid4
from sqlalchemy import case, delete, func, or_, select, update from sqlalchemy import case, delete, func, insert, or_, select, update
from sqlalchemy.exc import IntegrityError from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -27,8 +27,9 @@ def _dict(row) -> dict[str, Any]:
class SqlLlmWikiIndexStore(LlmWikiIndexStore): class SqlLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None:
self._sf = session_factory self._sf = session_factory
self.write_batch_size = max(1, min(int(write_batch_size), 500))
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow: async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id) row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
@@ -82,24 +83,29 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row.updated_at = now row.updated_at = now
await session.flush() await session.flush()
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id)) await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
session.add_all( vector_rows = [
[ {
LlmWikiWikiVectorRow( "id": str(uuid4()),
id=str(uuid4()), "page_id": row.id,
page_id=row.id, "knowledge_base_mapping_id": mapping_id,
knowledge_base_mapping_id=mapping_id, "section_index": int(item["section_index"]),
section_index=int(item["section_index"]), "heading": item.get("heading"),
heading=item.get("heading"), "section_content": str(item["section_content"]),
section_content=str(item["section_content"]), "content_hash": str(item["content_hash"]),
content_hash=str(item["content_hash"]), "embedding_fingerprint": fingerprint,
embedding_fingerprint=fingerprint, "embedding_dimensions": int(item["embedding_dimensions"]),
embedding_dimensions=int(item["embedding_dimensions"]), "vector_blob": bytes(item["vector_blob"]),
vector_blob=bytes(item["vector_blob"]), "created_at": now,
created_at=now, }
) for item in vectors
for item in vectors ]
] # Keep the page replacement atomic, but cap each SQL statement so
) # large Wiki pages cannot flood the database with one huge insert.
for start in range(0, len(vector_rows), self.write_batch_size):
await session.execute(
insert(LlmWikiWikiVectorRow),
vector_rows[start : start + self.write_batch_size],
)
state = await self._ensure_state(session, mapping_id) state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1 state.index_revision = int(state.index_revision or 0) + 1
state.embedding_fingerprint = fingerprint state.embedding_fingerprint = fingerprint
@@ -40,20 +40,31 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
) )
await session.commit() await session.commit()
store = SqlLlmWikiIndexStore(session_factory) store = SqlLlmWikiIndexStore(session_factory, write_batch_size=1)
assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp") assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp")
assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp") assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp")
page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"}) page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"})
blob, dimensions = encode_vector([3.0, 4.0], dimensions=2) blob, dimensions = encode_vector([3.0, 4.0], dimensions=2)
saved = await store.replace_page_vectors( saved = await store.replace_page_vectors(
page=page, page=page,
vectors=[{"section_index": 0, "heading": "History", "section_content": "Alice founded Acme.", "content_hash": "section", "embedding_dimensions": dimensions, "vector_blob": blob}], vectors=[
{
"section_index": index,
"heading": "History",
"section_content": f"Alice founded Acme. Part {index}",
"content_hash": f"section-{index}",
"embedding_dimensions": dimensions,
"vector_blob": blob,
}
for index in range(3)
],
fingerprint="fp", fingerprint="fp",
sync_id="sync-1", sync_id="sync-1",
) )
snapshot = await store.load_vector_snapshot("mapping-1", "fp") snapshot = await store.load_vector_snapshot("mapping-1", "fp")
assert saved["slug"] == "people/alice" assert saved["slug"] == "people/alice"
assert len(snapshot["rows"]) == 1 assert store.write_batch_size == 1
assert len(snapshot["rows"]) == 3
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8]) assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1 assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
@@ -690,6 +690,24 @@ async def test_weknora_client_proxies_original_document_preview_without_credenti
assert media_type == "text/plain" assert media_type == "text/plain"
@pytest.mark.asyncio
async def test_weknora_client_fetches_protected_wiki_image() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/api/v1/knowledge-bases/kb-one/files"
assert request.url.params["file_path"] == "resource://wiki/figure.png"
return httpx.Response(200, content=b"png-bytes", headers={"Content-Type": "image/png"})
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
content, media_type = await client.get_knowledge_base_file(
"kb-one",
"resource://wiki/figure.png",
)
assert content == b"png-bytes"
assert media_type == "image/png"
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_memory_store_enforces_owner_and_public_visibility() -> None: async def test_memory_store_enforces_owner_and_public_visibility() -> None:
store = MemoryLlmWikiStore() store = MemoryLlmWikiStore()
@@ -1358,6 +1376,10 @@ def test_weknora_iframe_html_rewrites_public_prefix_paths() -> None:
assert '"allowedWeKnoraIds":["remote-kb","other-visible-kb"]' in injected assert '"allowedWeKnoraIds":["remote-kb","other-visible-kb"]' in injected
assert ".breadcrumb-link.dropdown" not in injected assert ".breadcrumb-link.dropdown" not in injected
assert "reloadForKnowledgeBaseSwitch" in injected assert "reloadForKnowledgeBaseSwitch" in injected
assert "function protectedImageApiUrl" in injected
assert 'img[data-protected-src]' in injected
assert 'encodeURIComponent(cfg.weknoraId)' in injected
assert 'credentials: "same-origin"' in injected
def test_weknora_iframe_filters_kb_list_to_signed_visible_mappings() -> None: def test_weknora_iframe_filters_kb_list_to_signed_visible_mappings() -> None:
@@ -1,10 +1,13 @@
from __future__ import annotations from __future__ import annotations
import asyncio
import json import json
from types import SimpleNamespace
import httpx import httpx
import pytest import pytest
from app.gateway import llmwiki_index_scheduler
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index.sync import WikiSyncService from deerflow.integrations.weknora.local_index.sync import WikiSyncService
@@ -85,6 +88,24 @@ async def test_configured_dimensions_are_validated_but_not_sent_to_bge() -> None
assert requests == [{"model": "bge-embedding-m3", "input": ["Wiki page"]}] assert requests == [{"model": "bge-embedding-m3", "input": ["Wiki page"]}]
@pytest.mark.asyncio
async def test_embedding_error_includes_upstream_reason(caplog) -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(400, json={"detail": "unsupported request parameter: dimensions"})
config = LocalWikiEmbeddingConfig(
model="bge-embedding-m3",
base_url="http://embedding.test/v1",
max_retries=0,
)
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
client = StrictWikiEmbeddingClient(config, http_client=http_client)
with pytest.raises(RuntimeError, match="unsupported request parameter: dimensions"):
await client.embed_query("Wiki page")
assert "unsupported request parameter: dimensions" in caplog.text
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_omitted_dimensions_are_discovered_from_first_response() -> None: async def test_omitted_dimensions_are_discovered_from_first_response() -> None:
def handler(request: httpx.Request) -> httpx.Response: def handler(request: httpx.Request) -> httpx.Response:
@@ -167,3 +188,39 @@ async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> Non
assert store.states["kb"]["state"] == "idle" assert store.states["kb"]["state"] == "idle"
assert store.states["kb"]["embedding_fingerprint"] == "fp-new" assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"} assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}
@pytest.mark.asyncio
async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
calls: list[str] = []
async def fake_sleep(seconds: float) -> None:
calls.append(f"sleep:{seconds}")
raise asyncio.CancelledError
async def fake_sync_once(app) -> None:
calls.append("sync")
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
app = SimpleNamespace(
state=SimpleNamespace(
config=SimpleNamespace(
llmwiki=SimpleNamespace(
local_wiki_index=SimpleNamespace(sync_interval_seconds=120),
)
)
)
)
with pytest.raises(asyncio.CancelledError):
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
assert calls == ["sleep:120"]
def test_local_wiki_index_accepts_legacy_sync_flag() -> None:
config = LocalWikiIndexConfig.model_validate({"sync_enabled": True})
assert config.auto_sync is True
assert config.sync_enabled is True
@@ -177,6 +177,37 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None:
assert payload["pages"][0]["aliases"] == [] assert payload["pages"][0]["aliases"] == []
@pytest.mark.asyncio
async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None:
store = PublicStore(_published_row())
class Client:
async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str):
assert knowledge_base_id == "remote-one"
assert file_path == "storage://tenant/resource://wiki/figure.png"
return b"png-bytes", "image/png"
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
response = await llmwiki.get_public_wiki_file(
_request(store),
"kb-one",
"storage://tenant/resource://wiki/figure.png",
)
assert store.calls == [("kb-one", "__public_wiki__", False, False)]
assert response.body == b"png-bytes"
assert response.media_type == "image/png"
assert response.headers["cache-control"] == "public, max-age=300"
def test_wiki_file_proxy_rejects_non_weknora_urls() -> None:
with pytest.raises(HTTPException) as exc_info:
llmwiki._validate_protected_wiki_file_path("https://example.test/private.png")
assert exc_info.value.status_code == 422
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None: async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None:
store = PublicStore(_published_row()) store = PublicStore(_published_row())