From 4b86106a2417ccbc6ca7c21f527ccbe189369326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=89=E9=91=AB=E5=8D=87?= Date: Tue, 25 Aug 2026 11:24:01 +0800 Subject: [PATCH] fix: restore embedded wiki images and optimize vector sync --- .../src/core/llmwiki/wiki-page-markdown.ts | 28 ++++ .../src/strategy-components/api/llmwiki.ts | 17 +++ .../llmwiki/WeKnoraSourceDrawer.tsx | 22 ++- .../llmwiki/WeKnoraWikiPageView.tsx | 15 +- .../llmwiki/WikiProtectedImage.tsx | 75 ++++++++++ offline-backend-20260512/backend/CLAUDE.md | 15 +- offline-backend-20260512/backend/README.md | 22 ++- .../backend/app/gateway/app.py | 2 +- .../backend/app/gateway/deps.py | 7 +- .../app/gateway/llmwiki_index_scheduler.py | 6 +- .../backend/app/gateway/routers/llmwiki.py | 133 +++++++++++++++++- .../app/gateway/routers/llmwiki_index.py | 4 + offline-backend-20260512/backend/config.yaml | 40 +----- .../LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md | 60 ++------ .../harness/deerflow/config/llmwiki_config.py | 28 +++- .../deerflow/integrations/weknora/client.py | 47 ++++++- .../weknora/local_index/embedding.py | 43 +++++- .../persistence/llmwiki_index/__init__.py | 8 +- .../deerflow/persistence/llmwiki_index/sql.py | 46 +++--- .../test_llmwiki_local_index_repository.py | 17 ++- .../backend/tests/test_llmwiki_weknora.py | 22 +++ .../backend/tests/test_llmwiki_wiki_sync.py | 57 ++++++++ .../tests/test_public_llmwiki_embed.py | 31 ++++ 23 files changed, 609 insertions(+), 136 deletions(-) create mode 100644 frontend-web/src/strategy-components/components/resource-management/llmwiki/WikiProtectedImage.tsx diff --git a/frontend-web/src/core/llmwiki/wiki-page-markdown.ts b/frontend-web/src/core/llmwiki/wiki-page-markdown.ts index 4ee93185..17a2a60f 100644 --- a/frontend-web/src/core/llmwiki/wiki-page-markdown.ts +++ b/frontend-web/src/core/llmwiki/wiki-page-markdown.ts @@ -37,6 +37,34 @@ const WIKI_INDEX_TYPE_LABELS: Record = { const WIKI_INDEX_TYPE_ORDER = ["summary", "entity", "concept"]; +export const WIKI_PROTECTED_IMAGE_MARKER = "/__llmwiki-protected-image__"; + +const PROTECTED_WIKI_IMAGE_SOURCE = + /(?:storage:\/\/[0-9A-Za-z_-]+\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\/\/[^\s"'<>\])]+/gi; + +/** + * Browsers cannot load WeKnora's protected resource:// and object-storage + * schemes directly. Replace them with a safe marker before Markdown + * sanitization; the image component resolves that marker through DeerFlow. + */ +export function rewriteProtectedWikiImageSources(markdown: string): string { + return markdown.replace( + PROTECTED_WIKI_IMAGE_SOURCE, + (filePath) => `${WIKI_PROTECTED_IMAGE_MARKER}?file_path=${encodeURIComponent(filePath)}`, + ); +} + +export function parseProtectedWikiImageSource(source: string | undefined): string | null { + if (!source) return null; + try { + const url = new URL(source, window.location.origin); + if (url.pathname !== WIKI_PROTECTED_IMAGE_MARKER) return null; + return url.searchParams.get("file_path")?.trim() || null; + } catch { + return null; + } +} + export function stripWikiFrontmatter(value: string) { return value.replace(/^\s*---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, "").trim(); } diff --git a/frontend-web/src/strategy-components/api/llmwiki.ts b/frontend-web/src/strategy-components/api/llmwiki.ts index d7df1d9f..5b8f16c9 100644 --- a/frontend-web/src/strategy-components/api/llmwiki.ts +++ b/frontend-web/src/strategy-components/api/llmwiki.ts @@ -444,6 +444,23 @@ export async function getLlmWikiDocumentPreview( return response.blob(); } +export async function getLlmWikiProtectedFile( + knowledgeBaseId: string, + filePath: string, + publicAccess = false, +): Promise { + const base = publicAccess ? PUBLIC_API() : API(); + const params = new URLSearchParams({ file_path: filePath }); + const response = await apiFetch( + `${base}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/files?${params.toString()}`, + ); + if (!response.ok) { + await readJson(response, "读取 Wiki 图片"); + throw new Error("读取 Wiki 图片失败"); + } + return response.blob(); +} + export async function getLlmWikiGraph(knowledgeBaseId: string): Promise { return readJson( await apiFetch(`${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/graph?limit=200`), diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx index 272457d2..d013fa9c 100644 --- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraSourceDrawer.tsx @@ -33,6 +33,7 @@ import { reasoningPlugins } from "@/core/streamdown"; import { exportableWikiMarkdown, renderableWikiMarkdown, + rewriteProtectedWikiImageSources, safeWikiFileName, stripWikiFrontmatter, wikiLinkMarkdown, @@ -49,6 +50,7 @@ import { type LlmWikiDocument, type LlmWikiWikiPage, } from "@/strategy-components/api/llmwiki"; +import { WikiProtectedImage } from "./WikiProtectedImage"; type SourceContextResponse = { success?: boolean; @@ -926,10 +928,18 @@ export function WeKnoraSourceDrawer({ ( + + ), + }} /> @@ -964,10 +974,18 @@ export function WeKnoraSourceDrawer({
( + + ), + }} linkClassName={(href) => parseWeKnoraPreviewUrl(href)?.wikiSlug ? "wiki-content-link font-medium text-[#be2323] decoration-[#be2323]/55 decoration-dashed hover:decoration-solid" diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx index 1a6bd740..b2af1866 100644 --- a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView.tsx @@ -35,6 +35,7 @@ import { markdownToDocxBlob } from "@/core/scheduled-tasks/markdown-docx"; import { exportableWikiMarkdown, renderableWikiMarkdown, + rewriteProtectedWikiImageSources, safeWikiFileName, stripWikiFrontmatter, wikiLinkMarkdown, @@ -49,6 +50,7 @@ import { usePublicLlmWikiWikiPage, usePublicLlmWikiWikiPages, } from "@/strategy-components/api/llmwiki"; +import { WikiProtectedImage } from "./WikiProtectedImage"; const LIST_PATH = "/page/workspace/knowledge"; @@ -209,7 +211,7 @@ export function WeKnoraWikiPageView({ : pageData?.content, )); const displayMarkdown = pageData - ? renderableWikiMarkdown( + ? rewriteProtectedWikiImageSources(renderableWikiMarkdown( title, content, noteId, @@ -222,7 +224,7 @@ export function WeKnoraWikiPageView({ : undefined, publicEmbed ? pagesList : [], publicEmbed ? publicWikiIndex.data?.groups || [] : [], - ) + )) : ""; const exportMarkdown = pageData ? wikiLinkMarkdown(exportableWikiMarkdown(title, summary, content), noteId) @@ -497,6 +499,15 @@ export function WeKnoraWikiPageView({ isLoading={false} remarkPlugins={reasoningPlugins.remarkPlugins} rehypePlugins={reasoningPlugins.rehypePlugins} + components={{ + img: (props) => ( + + ), + }} linkClassName={(href) => publicEmbed && publicWikiTargetFromHref(href, noteId) ? "wiki-content-link border-b border-dashed !border-[var(--primary)] font-medium !text-primary !no-underline hover:border-solid focus-visible:rounded-sm focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary" diff --git a/frontend-web/src/strategy-components/components/resource-management/llmwiki/WikiProtectedImage.tsx b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WikiProtectedImage.tsx new file mode 100644 index 00000000..816aeecf --- /dev/null +++ b/frontend-web/src/strategy-components/components/resource-management/llmwiki/WikiProtectedImage.tsx @@ -0,0 +1,75 @@ +import { ImageOff, Loader2 } from "lucide-react"; +import { type ImgHTMLAttributes, useEffect, useState } from "react"; + +import { parseProtectedWikiImageSource } from "@/core/llmwiki/wiki-page-markdown"; +import { getLlmWikiProtectedFile } from "@/strategy-components/api/llmwiki"; + +type WikiProtectedImageProps = ImgHTMLAttributes & { + knowledgeBaseId: string; + publicAccess?: boolean; +}; + +export function WikiProtectedImage({ + knowledgeBaseId, + publicAccess = false, + src, + alt, + className, + ...props +}: WikiProtectedImageProps) { + const filePath = parseProtectedWikiImageSource(src); + const [objectUrl, setObjectUrl] = useState(""); + const [error, setError] = useState(""); + + useEffect(() => { + if (!filePath || !knowledgeBaseId) { + setObjectUrl(""); + setError(""); + return; + } + let cancelled = false; + let nextObjectUrl = ""; + setObjectUrl(""); + setError(""); + void getLlmWikiProtectedFile(knowledgeBaseId, filePath, publicAccess) + .then((blob) => { + if (cancelled) return; + nextObjectUrl = URL.createObjectURL(blob); + setObjectUrl(nextObjectUrl); + }) + .catch((reason: unknown) => { + if (!cancelled) { + setError(reason instanceof Error ? reason.message : "Wiki 图片加载失败"); + } + }); + return () => { + cancelled = true; + if (nextObjectUrl) URL.revokeObjectURL(nextObjectUrl); + }; + }, [filePath, knowledgeBaseId, publicAccess]); + + if (!filePath) { + return {alt}; + } + if (error) { + return ( + + + {alt || "Wiki 图片加载失败"} + + ); + } + if (!objectUrl) { + return ( + + + + ); + } + return {alt}; +} diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index 806119ea..6b29a8cf 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -1018,6 +1018,11 @@ Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` me `llmwiki.weknora.api_base_url` is the provider switch: non-empty enables WeKnora, empty preserves the legacy LLMWiki UI. WeKnora is independently deployed and owns PostgreSQL/Redis/object storage/vector storage/parsers/models; DeerFlow config contains only `api_base_url` and optional `web_base_url`. DeerFlow deliberately sends no WeKnora credential, so the WeKnora endpoint must be restricted to a trusted network and allow DeerFlow traffic itself or through its reverse proxy. There is intentionally no `default_embedding_model_id`: `WeKnoraClient` queries `/api/v1/models` and uses WeKnora's active Embedding model when creating a KB; Wiki creation additionally resolves an active KnowledgeQA model inside WeKnora for synthesis. The native BFF is `app/gateway/routers/llmwiki.py`; ownership/publication mappings live in `deerflow.persistence.llmwiki`; the browser only sees DeerFlow mapping IDs. Knowledge-base owners publish immediately (`private` -> `published`) without a review queue, and can unpublish ordinary bases. The conversation-deposit KB is system-owned, always published, shared by all DeerFlow users, excluded from My Knowledge, and immutable through user/admin detail operations; recognition requires both the reserved name and `system` owner. Existing same-name remote mappings are adopted in place as the system deposit, preserving mapping ids and document links. Its user-facing description, generated Markdown source and tags use `cmzs` branding; list/detail reads synchronize stale remote descriptions, and search results, source previews and embedded-detail JSON recursively replace legacy brand text in historical conversation-deposit payloads. Deposit requests must reference a thread accessible to the current account. Personal knowledge-base list/search views are scoped to the current DeerFlow account even for admins; frontend LLMWiki caches include the live account id, and admin write power does not make other users' private KBs appear as owned. Only `system_role === "admin"` sees the WeKnora connection-config tab. Knowledge cards navigate to a dedicated WeKnora-style detail route. Wiki-enabled bases render and default to `Wiki -> documents -> graph`; standard bases omit Wiki and render `documents -> graph`. The detail page supports sequential multi-file upload with an aggregate result for writable ordinary bases, hides it for public read-only/system-deposit bases, and uses a generic loading label while the embed session starts. It also supports URL and published manual-Markdown imports, browses real Wiki pages/content, proxies authenticated original-file previews, pages through `/chunks/{knowledge_id}`, and renders the Wiki page graph from `/knowledgebase/{kb_id}/wiki/graph`; document details use a right sheet rather than the removed KB modal. The credential-free proxy uses one WeKnora service identity, so per-user separation is enforced by DeerFlow mappings while all remote KBs physically share that identity's WeKnora workspace. WeKnora iframe detail pages first call `/api/llmwiki/knowledge-bases/{id}/weknora/session` through the normal DeerFlow API auth path, then load `/platform/knowledge-bases/{weknora_id}` with a signed `deerflow_weknora_embed` cookie for `/platform`, `/assets`, `/locales`, root assets, and API calls. Injected fetch/XHR rewriting sends all iframe `/api/v1/*`, including colliding `/api/v1/auth/*`, through `/api/llmwiki/weknora-embed/api/v1/*`; Gateway auth/CSRF bypass only applies to those signed iframe proxy requests, the proxy still revalidates the mapped KB, and cookie security honors `X-Forwarded-Proto` for HTTPS reverse proxies. `llmwiki_search` is conditionally registered in `tools.py`, is read-only, and rechecks the current DeerFlow user, agent `llmwiki_knowledge_base_ids`, and thread-context selection on every call. Source previews prefer the cleaned Wiki/LLMWiki page text when a retrieved chunk maps to a Wiki page, with raw split chunks as the fallback. Agent bindings use an inline WeKnora-style searchable/grouped multi-select in the standard agent create/detail UI; new chats use the same interaction pattern as a compact composer dropdown, and selections are written to thread metadata/context and restored on reopen. The DeerFlow frontend is native React and the WeKnora frontend is not modified or embedded. Full deployment and behavior notes: `docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`. Tests: `tests/test_llmwiki_weknora.py`. The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. Do not add an HTML `sandbox` attribute to this trusted, permission-filtered detail iframe: a sandboxed ancestor causes Chromium's built-in Blob PDF viewer to render the browser-blocked page. The Gateway follows upstream redirects only for `/knowledge/{id}/preview` and `/knowledge-bases/{id}/files`, keeping PDF bytes and protected Wiki images inside the signed proxy instead of sending the browser to WeKnora object storage. +Native/private and public Wiki readers must rewrite protected Markdown image +sources to their mapping-scoped DeerFlow `/files?file_path=` endpoints and load +them as Blob URLs. The embedded guard also rewrites protected fetch/XHR values +and hydrates late `img[data-protected-src]` placeholders; never expose the +service token or relax the current-KB authorization to make images work. **Embedded native knowledge-base switching:** The original WeKnora breadcrumb picker remains available inside the detail iframe. A signed 15-minute embed cookie carries only the current DeerFlow user's visible mappings (their own plus published bases); the proxied `/api/v1/knowledge-bases` response is filtered to that allowlist. Choosing another entry performs a full iframe navigation, rechecks the DeerFlow mapping and read/write permission, then reissues the cookie for the selected base. Never expose the upstream service-admin list directly or permit an arbitrary remote knowledge-base ID. @@ -1444,8 +1449,10 @@ detail. Never call `/knowledge-search` from internal search, RAG, or the `deerflow.integrations.weknora.local_index` and `deerflow.persistence.llmwiki_index`; preserve the harness → app import firewall. Gateway composition injects the store/search/sync services through -`local_index.runtime`, and the lease-protected scheduler starts only when both -`enabled` and `sync_enabled` are true. +`local_index.runtime`. `enabled` turns on local vector search and manual +vectorization; `auto_sync` is the separate opt-in for the lease-protected +scheduler. The scheduler sleeps for one full configured interval before its +first scan, so application startup must never trigger immediate vectorization. Vectors are normalized little-endian float32 BLOBs. A page mirror and all of its vectors are replaced in one transaction only after strict batched @@ -1458,6 +1465,10 @@ input, not an outbound API field: fixed-dimension BGE/OpenAI-compatible endpoints can reject a `dimensions` request parameter with HTTP 400. When the value is omitted, the embedding client probes and freezes the first response's dimension before sync lease/rebuild decisions. +Embedding errors must retain the upstream status and bounded response body in +logs, without exposing API keys. SQL vector writes are split into statements +of at most `database_batch_size` rows (default 50) inside the same atomic page +replacement transaction. During a fingerprint rebuild the mapping uses the Wiki-page API until every page is successfully rebuilt. `WikiRetrievalService` partitions mixed requests per mapping, so ready libraries stay local while incomplete libraries use that diff --git a/offline-backend-20260512/backend/README.md b/offline-backend-20260512/backend/README.md index 04547446..e02dacc4 100644 --- a/offline-backend-20260512/backend/README.md +++ b/offline-backend-20260512/backend/README.md @@ -779,15 +779,24 @@ processed Wiki content. A knowledge base uses exact NumPy cosine similarity only after its whole Wiki has completed one coherent vectorization pass; until then internal search calls WeKnora's Wiki-page search/list API and reads the matched Wiki pages. It never falls back to raw documents or chunks. The local -vector feature is disabled by default. Configure -`llmwiki.local_wiki_index.embedding` first, then enable `enabled`, -`sync_enabled`, and `internal_search_enabled` in `config.yaml`. +vector feature is disabled by default. Configure the three +`llmwiki.local_wiki_index.embedding` values (`model`, `base_url`, and +`api_key`), then change only `enabled` to `true` in `config.yaml`. Manual +vectorization is the default. Set `auto_sync: true` only when periodic +background synchronization is wanted; the scheduler waits one full interval +after startup before its first scan, so boot does not trigger a rebuild. +`database_batch_size` caps each vector insert statement (default `50`) while +the page replacement remains atomic, preventing a large Wiki from flooding +the SQL connection in one statement. The optional `embedding.dimensions` value is used only to validate the returned vector size and build the index fingerprint; it is deliberately not sent in the embedding request because fixed-dimension BGE/OpenAI-compatible endpoints may reject that extra field with HTTP 400. Omit it to discover the dimension from the first successful response. +Embedding failures log the upstream HTTP status and a bounded response body. +The client never sends a `dimensions` request parameter. + The migration creates `llmwiki_wiki_pages`, `llmwiki_wiki_vectors`, and `llmwiki_wiki_sync_states`. Wiki Markdown and normalized float32 vectors are stored in DeerFlow's configured SQL database. Multi-worker sync is protected by @@ -841,6 +850,13 @@ system conversation-deposit knowledge base. Private or unpublished bases are never exposed by this route. Anonymous consumers can discover the available public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each entry deliberately contains only the DeerFlow mapping `id` and display `name`. +Protected `resource://` and supported object-storage images are fetched through +the current mapping instead of being exposed as upstream URLs. Authenticated +readers use `GET /api/llmwiki/knowledge-bases/{id}/files?file_path=...`; public +readers use the corresponding `/api/public/llmwiki/...` route, which requires a +published mapping. The embedded WeKnora guard applies the same mapping-scoped +proxy to document previews and Markdown images, including images inserted after +the drawer/page first renders. Wiki management also provides **导出 Excel**. `GET /api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx` exports every processed Wiki diff --git a/offline-backend-20260512/backend/app/gateway/app.py b/offline-backend-20260512/backend/app/gateway/app.py index 64c6be09..697c16c2 100644 --- a/offline-backend-20260512/backend/app/gateway/app.py +++ b/offline-backend-20260512/backend/app/gateway/app.py @@ -998,7 +998,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: local_wiki_index = app.state.config.llmwiki.local_wiki_index if ( local_wiki_index.enabled - and local_wiki_index.sync_enabled + and local_wiki_index.auto_sync and getattr(app.state, "llmwiki_sync_service", None) is not None ): from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop diff --git a/offline-backend-20260512/backend/app/gateway/deps.py b/offline-backend-20260512/backend/app/gateway/deps.py index e76c54ea..1a1287e7 100644 --- a/offline-backend-20260512/backend/app/gateway/deps.py +++ b/offline-backend-20260512/backend/app/gateway/deps.py @@ -321,13 +321,16 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]: from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime from deerflow.persistence.llmwiki_index import make_llmwiki_index_store - app.state.llmwiki_index_store = make_llmwiki_index_store(sf) + local_index_config = config.llmwiki.local_wiki_index + app.state.llmwiki_index_store = make_llmwiki_index_store( + sf, + write_batch_size=local_index_config.database_batch_size, + ) app.state.llmwiki_embedding = None app.state.llmwiki_vector_cache = None app.state.llmwiki_vector_search = None app.state.llmwiki_sync_service = None app.state.llmwiki_retrieval_service = None - local_index_config = config.llmwiki.local_wiki_index from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime resolved_llmwiki = get_resolved_llmwiki_runtime(config) diff --git a/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py b/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py index 8964378e..a5363b21 100644 --- a/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py +++ b/offline-backend-20260512/backend/app/gateway/llmwiki_index_scheduler.py @@ -48,6 +48,10 @@ async def _sync_once(app: FastAPI) -> None: async def local_wiki_index_scheduler_loop(app: FastAPI) -> None: interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds + logger.info( + "Local Wiki auto-sync scheduler started; first scan delayed by %s seconds", + interval, + ) while True: - await _sync_once(app) await asyncio.sleep(interval) + await _sync_once(app) diff --git a/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py b/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py index cee7b04a..560dadfd 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py +++ b/offline-backend-20260512/backend/app/gateway/routers/llmwiki.py @@ -61,6 +61,10 @@ router = APIRouter(tags=["llmwiki"]) proxy_router = APIRouter(tags=["llmwiki-proxy"]) _MAX_UPLOAD_BYTES = 100 * 1024 * 1024 +_PROTECTED_WIKI_FILE_PATH = re.compile( + r"^(?:storage://[0-9A-Za-z_-]+/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs)://", + re.IGNORECASE, +) _WEKNORA_ADMIN_SESSION_CACHE: dict[str, dict[str, Any]] = {} _WEKNORA_PUBLIC_PREFIX = "/deerflow" _WEKNORA_PUBLIC_ROOT_PATHS = ( @@ -242,7 +246,7 @@ def _store(request: Request) -> LlmWikiStore: def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None: service = getattr(request.app.state, "llmwiki_sync_service", None) config = request.app.state.config.llmwiki.local_wiki_index - if service is not None and config.enabled and mapping.get("wiki_index_enabled", True): + if service is not None and config.enabled and config.auto_sync and mapping.get("wiki_index_enabled", True): from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True)) @@ -1085,6 +1089,8 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla function rewriteApiUrl(value) {{ try {{ + const protectedUrl = protectedImageApiUrl(value); + if (protectedUrl) return protectedUrl; const next = new URL(value, location.origin); if (next.origin !== location.origin) return value; const publicPrefix = cfg.publicPrefix || ""; @@ -1104,6 +1110,19 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla return value; }} }} + function protectedImageApiUrl(value) {{ + const raw = String(value || "").trim(); + if (!/^(?:storage:\\/\\/[0-9A-Za-z_-]+\\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\\/\\//i.test(raw)) {{ + return ""; + }} + const endpoint = + cfg.apiPrefix + + "/api/v1/knowledge-bases/" + + encodeURIComponent(cfg.weknoraId) + + "/files?file_path=" + + encodeURIComponent(raw); + return new URL(endpoint, location.origin).href; + }} const nativeFetch = window.fetch.bind(window); window.fetch = (input, init) => {{ if (input instanceof Request) {{ @@ -1116,6 +1135,67 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla return nativeXhrOpen.call(this, method, rewriteApiUrl(url), ...rest); }}; + // WeKnora renders protected Markdown images as transparent placeholders and + // hydrates them asynchronously. In an embedded page that hydration can run + // before its own API client has picked up our proxy prefix, leaving a 1x1 + // GIF forever. Resolve the same protected source through the scoped proxy + // as a fallback. The backend validates the scheme and current KB again. + function hydrateProtectedImage(image) {{ + const protectedSrc = String(image?.getAttribute?.("data-protected-src") || "").trim(); + const proxyUrl = protectedImageApiUrl(protectedSrc); + if (!proxyUrl) return; + if ( + image.dataset.deerflowProtectedSource === protectedSrc && + String(image.getAttribute("src") || "").startsWith("blob:") + ) return; + const attempts = Number(image.dataset.deerflowProtectedAttempts || "0"); + if (attempts >= 3 || image.dataset.deerflowProtectedLoading === "1") return; + image.dataset.deerflowProtectedLoading = "1"; + image.dataset.deerflowProtectedAttempts = String(attempts + 1); + nativeFetch(proxyUrl, {{ credentials: "same-origin", headers: {{ Accept: "image/*" }} }}) + .then((response) => {{ + if (!response.ok) throw new Error(`protected image request failed (${{response.status}})`); + return response.blob(); + }}) + .then((blob) => {{ + const objectUrl = URL.createObjectURL(blob); + image.dataset.deerflowProtectedSource = protectedSrc; + image.removeAttribute("data-img-loading"); + image.removeAttribute("data-wiki-image-error"); + image.src = objectUrl; + image.addEventListener("load", () => URL.revokeObjectURL(objectUrl), {{ once: true }}); + }}) + .catch(() => {{ + image.dataset.wikiImageError = "true"; + window.setTimeout(() => {{ + delete image.dataset.deerflowProtectedLoading; + hydrateProtectedImage(image); + }}, 1200 * (attempts + 1)); + }}) + .finally(() => {{ + delete image.dataset.deerflowProtectedLoading; + }}); + }} + function hydrateProtectedImages(scope = document) {{ + if (scope?.matches?.("img[data-protected-src]")) hydrateProtectedImage(scope); + scope?.querySelectorAll?.("img[data-protected-src]").forEach(hydrateProtectedImage); + }} + const protectedImageObserver = new MutationObserver((mutations) => {{ + mutations.forEach((mutation) => {{ + if (mutation.type === "attributes") hydrateProtectedImage(mutation.target); + mutation.addedNodes?.forEach((node) => {{ + if (node?.nodeType === Node.ELEMENT_NODE) hydrateProtectedImages(node); + }}); + }}); + }}); + protectedImageObserver.observe(document.documentElement, {{ + childList: true, + subtree: true, + attributes: true, + attributeFilter: ["data-protected-src"] + }}); + hydrateProtectedImages(); + function tone(color, alpha) {{ return `color-mix(in srgb, ${{color}} ${{Math.round(alpha * 100)}}%, transparent)`; }} @@ -1519,6 +1599,7 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla enforceLocation(); purgeGuides(); hideWeKnoraShell(); + hydrateProtectedImages(); if (tickSourceDetail() && sourceDetailMode) {{ clearInterval(maintenanceTimer); try {{ readyObserver.disconnect(); }} catch (_) {{}} @@ -1820,6 +1901,32 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]: return row +def _validate_protected_wiki_file_path(file_path: str) -> str: + value = file_path.strip() + if not value or "\x00" in value or not _PROTECTED_WIKI_FILE_PATH.match(value): + raise HTTPException(status_code=422, detail="Unsupported Wiki image path") + return value + + +async def _wiki_file_response(row: dict[str, Any], file_path: str, *, public: bool) -> Response: + try: + content, media_type = await _client_or_503().get_knowledge_base_file( + str(row["weknora_id"]), + _validate_protected_wiki_file_path(file_path), + ) + except WeKnoraError as exc: + _raise_upstream(exc) + return Response( + content=content, + media_type=media_type, + headers={ + "Cache-Control": "public, max-age=300" if public else "private, max-age=60", + "Content-Disposition": "inline", + "X-Content-Type-Options": "nosniff", + }, + ) + + async def _verify_document(client: WeKnoraClient, row: dict[str, Any], document_id: str) -> dict[str, Any]: document = await client.get_document(document_id) if str(document.get("knowledge_base_id") or "") != str(row["weknora_id"]): @@ -2510,6 +2617,18 @@ async def list_public_wiki_pages( return {**result, "pages": pages} +@router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}/files") +async def get_public_wiki_file( + request: Request, + mapping_id: str, + file_path: str = Query(..., min_length=1, max_length=8192), +) -> Response: + """Serve one protected image belonging to a published Wiki.""" + + row = await _public_mapping(request, mapping_id) + return await _wiki_file_response(row, file_path, public=True) + + _PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison") @@ -2820,6 +2939,18 @@ async def preview_document(request: Request, mapping_id: str, document_id: str) ) +@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/files") +async def get_wiki_file( + request: Request, + mapping_id: str, + file_path: str = Query(..., min_length=1, max_length=8192), +) -> Response: + """Serve one protected Wiki image after normal knowledge-base authorization.""" + + row, _, _ = await _authorized_mapping(request, mapping_id, write=False) + return await _wiki_file_response(row, file_path, public=False) + + @router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph") async def get_knowledge_base_graph( request: Request, diff --git a/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py b/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py index 313522ee..8c69ccea 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py +++ b/offline-backend-20260512/backend/app/gateway/routers/llmwiki_index.py @@ -2,6 +2,7 @@ from __future__ import annotations +import logging from typing import Any from uuid import uuid4 @@ -14,6 +15,7 @@ from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectori from deerflow.integrations.weknora.local_index.search import WikiIndexError router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"]) +logger = logging.getLogger(__name__) class WikiVectorSearchRequest(BaseModel): @@ -181,6 +183,7 @@ async def _run_sync(app, mapping_ids: list[str], force: bool) -> None: try: await service.sync_mapping(mapping, force=force) except Exception: + logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id")) continue @@ -191,6 +194,7 @@ async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None: try: await service.sync_mapping(mapping, force=force) except Exception: + logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id")) return diff --git a/offline-backend-20260512/backend/config.yaml b/offline-backend-20260512/backend/config.yaml index a8990137..31606d1a 100644 --- a/offline-backend-20260512/backend/config.yaml +++ b/offline-backend-20260512/backend/config.yaml @@ -556,47 +556,17 @@ llmwiki: web_base_url: "http://119.254.86.251:8088" admin_email: "lqq@qq.com" admin_password: "1qaz@WSX" - # DeerFlow-local Wiki mirror/vector index. Keep disabled until the - # OpenAI-compatible embedding endpoint below is configured and reachable. + # 本地 Wiki 向量化：只需把 enabled 改为 true，并填写下面 3 个 embedding 值。 + # 默认只在页面点击“向量化”时执行；auto_sync=true 才会后台自动增量同步， + # 且服务启动时只启动定时器，不会立刻全量向量化。 local_wiki_index: enabled: false - sync_enabled: false - internal_search_enabled: false - reference_ui_enabled: false - sync_interval_seconds: 120 - chunk_max_chars: 1000 - chunk_overlap_chars: 120 - chunk_min_chars: 80 + auto_sync: false + database_batch_size: 50 embedding: - enabled: true - provider: openai model: bge-m3 base_url: "" api_key: "" - # Expected BGE-M3 response size for local validation/indexing only. - # This value is not sent as an embedding API request parameter. - dimensions: 1024 - batch_size: 32 - timeout_seconds: 30 - max_retries: 3 - engine: numpy_exact - strict_vector: true - top_k_sections: 30 - top_k_pages: 8 - max_sections_per_page: 2 - min_similarity: 0.35 - cache_max_knowledge_bases: 20 - cache_revision_check_seconds: 5 - external_api: - enabled: false - api_key: "" - previous_api_key: "" - max_query_chars: 1000 - max_knowledge_bases: 20 - max_results: 50 - default_results: 10 - include_content_max_chars: 12000 - requests_per_minute: 60 log_system: enabled: true diff --git a/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md b/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md index 0a166d91..02ed80c8 100644 --- a/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md +++ b/offline-backend-20260512/backend/docs/LLMWIKI_DEERFLOW_LOCAL_VECTOR_INDEX_ZH.md @@ -279,65 +279,25 @@ llmwiki: local_wiki_index: enabled: true - - # 同步 - sync_enabled: true - sync_interval_seconds: 120 - sync_page_size: 100 - sync_concurrency: 3 - sync_lease_seconds: 600 - sync_request_timeout_seconds: 30 - sync_max_retries: 3 - sync_retry_backoff_seconds: 5 - - # 切分 - chunk_max_chars: 1000 - chunk_overlap_chars: 120 - chunk_min_chars: 80 - title_prefix_enabled: true - - # Embedding：OpenAI-compatible /v1/embeddings + # 默认只允许手动向量化；需要定时增量同步时再开启。 + auto_sync: false + # 每条 SQL 最多插入的向量行数；整页替换仍在同一事务内完成。 + database_batch_size: 50 embedding: - enabled: true - provider: openai model: bge-m3 base_url: "$WIKI_EMBEDDING_BASE_URL" api_key: "$WIKI_EMBEDDING_API_KEY" - dimensions: 1024 - batch_size: 32 - timeout_seconds: 30 - max_retries: 3 - - # 检索 - engine: numpy_exact - strict_vector: true - top_k_sections: 30 - top_k_pages: 8 - max_sections_per_page: 2 - min_similarity: 0.35 - keyword_boost_enabled: false - keyword_boost_weight: 0.15 - cache_max_knowledge_bases: 20 - cache_revision_check_seconds: 5 - - # 外部接口 - external_api: - enabled: false - api_key: "$LLMWIKI_EXTERNAL_SEARCH_API_KEY" - max_query_chars: 1000 - max_knowledge_bases: 20 - max_results: 50 - default_results: 10 - include_content_max_chars: 12000 - requests_per_minute: 60 ``` +日常部署只需配置上面三个 Embedding 值并将 `enabled` 改为 `true`。 +`auto_sync=false` 时仅由页面“手动向量化”触发；即使设为 `true`，定时器也会先等待一个完整同步周期，不会在服务刚启动时立即重建索引。其余切分、检索、超时与重试参数继续使用代码默认值，只有确有调优需要时才覆盖。 + ### 6.1 配置校验规则 启动时应执行： 1. `enabled=true` 时，Embedding `base_url` 和 `model` 必须非空； -2. `dimensions` 可省略，但第一次成功 Embedding 后必须固定并写入索引 profile； +2. 不向上游发送 `dimensions` 参数；第一次成功 Embedding 后从响应自动发现维度，并固定到索引 profile； 3. `chunk_overlap_chars < chunk_max_chars`； 4. `top_k_pages <= top_k_sections`； 5. `external_api.enabled=true` 时必须配置非空 API Key； @@ -1885,9 +1845,7 @@ class WikiVectorEngine(ABC): ```text local_wiki_index.enabled -local_wiki_index.sync_enabled -local_wiki_index.internal_search_enabled -local_wiki_index.reference_ui_enabled +local_wiki_index.auto_sync local_wiki_index.external_api.enabled ``` diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py b/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py index 45776f0b..ae2427af 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/config/llmwiki_config.py @@ -59,7 +59,7 @@ class LocalWikiEmbeddingConfig(BaseModel): base_url: str = "" api_key: str = Field(default="", repr=False) dimensions: int | None = Field( - default=1024, + default=None, ge=1, le=65536, description="Expected response vector dimensions; not sent to the embedding endpoint.", @@ -108,9 +108,7 @@ class LocalWikiIndexConfig(BaseModel): """DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages.""" enabled: bool = False - sync_enabled: bool = False - internal_search_enabled: bool = False - reference_ui_enabled: bool = False + auto_sync: bool = False sync_interval_seconds: int = Field(default=120, ge=10, le=86400) sync_page_size: int = Field(default=100, ge=1, le=500) sync_concurrency: int = Field(default=3, ge=1, le=32) @@ -133,9 +131,31 @@ class LocalWikiIndexConfig(BaseModel): keyword_boost_weight: float = Field(default=0.15, ge=0, le=1) cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000) cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600) + database_batch_size: int = Field(default=50, ge=1, le=500) external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig) model_config = ConfigDict(extra="ignore") + @model_validator(mode="before") + @classmethod + def _migrate_legacy_flags(cls, value): + if isinstance(value, dict) and "auto_sync" not in value and "sync_enabled" in value: + value = {**value, "auto_sync": value.get("sync_enabled")} + return value + + @property + def sync_enabled(self) -> bool: + """Backward-compatible status name; automatic work is one explicit switch.""" + + return self.auto_sync + + @property + def internal_search_enabled(self) -> bool: + return self.enabled + + @property + def reference_ui_enabled(self) -> bool: + return self.enabled + @model_validator(mode="after") def _validate_index(self): if self.chunk_overlap_chars >= self.chunk_max_chars: diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py index 2bd618c1..1589dc55 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/client.py @@ -2,8 +2,8 @@ from __future__ import annotations -from collections.abc import Mapping, Sequence import time +from collections.abc import Mapping, Sequence from typing import Any from urllib.parse import quote @@ -430,6 +430,51 @@ class WeKnoraClient: media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip() return response.content, media_type or "application/octet-stream" + async def get_knowledge_base_file( + self, + knowledge_base_id: str, + file_path: str, + ) -> tuple[bytes, str]: + """Return one protected Wiki attachment using DeerFlow's WeKnora identity.""" + + owns_client = self._http_client is None + client = self._http_client or httpx.AsyncClient( + timeout=httpx.Timeout(self.timeout_seconds), + follow_redirects=True, + trust_env=False, + ) + try: + for attempt in range(2): + response = await client.request( + "GET", + f"{self.base_url}/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/files", + params={"file_path": file_path}, + headers=await self._headers(force_refresh=attempt > 0, accept="*/*"), + follow_redirects=True, + ) + if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0: + break + except httpx.TimeoutException as exc: + raise WeKnoraError("WeKnora Wiki image request timed out", status_code=504) from exc + except httpx.HTTPError as exc: + raise WeKnoraError("WeKnora Wiki image is unavailable", status_code=502) from exc + finally: + if owns_client: + await client.aclose() + if response.status_code == 404: + raise WeKnoraError("WeKnora Wiki image was not found", status_code=404) + if response.status_code in {401, 403}: + if self._has_admin_credentials: + raise WeKnoraError("WeKnora rejected DeerFlow's authenticated Wiki image request", status_code=502) + raise WeKnoraError( + "WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy", + status_code=502, + ) + if response.status_code >= 400: + raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502) + media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip() + return response.content, media_type or "application/octet-stream" + async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]: value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse")) return dict(value) if isinstance(value, dict) else {} diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py index f4b6b8ef..ca654111 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/integrations/weknora/local_index/embedding.py @@ -15,6 +15,21 @@ from deerflow.integrations.weknora.local_index.vector_codec import normalize_vec logger = logging.getLogger(__name__) +def _upstream_error_detail(response: httpx.Response) -> str: + """Return a bounded upstream reason without hiding the useful response body.""" + + try: + payload = response.json() + if isinstance(payload, dict): + detail = payload.get("detail") or payload.get("message") or payload.get("error") or payload + else: + detail = payload + value = str(detail) + except (ValueError, TypeError): + value = response.text + return " ".join(value.split())[:2000] or "no response body" + + class WikiEmbeddingError(RuntimeError): def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None: super().__init__(message) @@ -76,11 +91,15 @@ class StrictWikiEmbeddingClient: started = monotonic() try: response = await (await self._get_client()).post(url, headers=headers, json=payload) - if response.status_code in {401, 403}: - raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False) - if response.status_code == 429 or response.status_code >= 500: - raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True) - response.raise_for_status() + if response.status_code >= 400: + detail = _upstream_error_detail(response) + category = "authentication" if response.status_code in {401, 403} else "upstream" + retryable = response.status_code == 429 or response.status_code >= 500 + raise WikiEmbeddingError( + f"Embedding service returned HTTP {response.status_code}: {detail}", + category=category, + retryable=retryable, + ) data = response.json() items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else [] if len(items) != len(texts): @@ -98,8 +117,20 @@ class StrictWikiEmbeddingClient: except httpx.TimeoutException: last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True) except (httpx.HTTPError, ValueError, TypeError) as exc: - last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError)) + reason = " ".join(str(exc).split())[:2000] or type(exc).__name__ + last_error = WikiEmbeddingError( + f"Embedding request failed ({type(exc).__name__}): {reason}", + category="request", + retryable=isinstance(exc, httpx.HTTPError), + ) if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries): + logger.error( + "Wiki embedding batch failed model=%s batch=%d category=%s reason=%s", + self.config.model, + len(texts), + last_error.category, + last_error, + ) raise last_error await asyncio.sleep(min(30.0, 0.5 * (2**attempt))) raise last_error or WikiEmbeddingError("Embedding request failed") diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py index 71f2b522..fb8da3da 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/__init__.py @@ -6,8 +6,12 @@ from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWiki from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore -def make_llmwiki_index_store(session_factory): - return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory) +def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50): + return ( + MemoryLlmWikiIndexStore() + if session_factory is None + else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size) + ) __all__ = [ diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py index c9d87b27..75f9ff68 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/persistence/llmwiki_index/sql.py @@ -6,7 +6,7 @@ from datetime import UTC, datetime, timedelta from typing import Any from uuid import uuid4 -from sqlalchemy import case, delete, func, or_, select, update +from sqlalchemy import case, delete, func, insert, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -27,8 +27,9 @@ def _dict(row) -> dict[str, Any]: class SqlLlmWikiIndexStore(LlmWikiIndexStore): - def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None: + def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None: self._sf = session_factory + self.write_batch_size = max(1, min(int(write_batch_size), 500)) async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow: row = await session.get(LlmWikiWikiSyncStateRow, mapping_id) @@ -82,24 +83,29 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore): row.updated_at = now await session.flush() await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id)) - session.add_all( - [ - LlmWikiWikiVectorRow( - id=str(uuid4()), - page_id=row.id, - knowledge_base_mapping_id=mapping_id, - section_index=int(item["section_index"]), - heading=item.get("heading"), - section_content=str(item["section_content"]), - content_hash=str(item["content_hash"]), - embedding_fingerprint=fingerprint, - embedding_dimensions=int(item["embedding_dimensions"]), - vector_blob=bytes(item["vector_blob"]), - created_at=now, - ) - for item in vectors - ] - ) + vector_rows = [ + { + "id": str(uuid4()), + "page_id": row.id, + "knowledge_base_mapping_id": mapping_id, + "section_index": int(item["section_index"]), + "heading": item.get("heading"), + "section_content": str(item["section_content"]), + "content_hash": str(item["content_hash"]), + "embedding_fingerprint": fingerprint, + "embedding_dimensions": int(item["embedding_dimensions"]), + "vector_blob": bytes(item["vector_blob"]), + "created_at": now, + } + for item in vectors + ] + # Keep the page replacement atomic, but cap each SQL statement so + # large Wiki pages cannot flood the database with one huge insert. + for start in range(0, len(vector_rows), self.write_batch_size): + await session.execute( + insert(LlmWikiWikiVectorRow), + vector_rows[start : start + self.write_batch_size], + ) state = await self._ensure_state(session, mapping_id) state.index_revision = int(state.index_revision or 0) + 1 state.embedding_fingerprint = fingerprint diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py b/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py index d33a1309..ad93a117 100644 --- a/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py +++ b/offline-backend-20260512/backend/tests/test_llmwiki_local_index_repository.py @@ -40,20 +40,31 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t ) await session.commit() - store = SqlLlmWikiIndexStore(session_factory) + store = SqlLlmWikiIndexStore(session_factory, write_batch_size=1) assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp") assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp") page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"}) blob, dimensions = encode_vector([3.0, 4.0], dimensions=2) saved = await store.replace_page_vectors( page=page, - vectors=[{"section_index": 0, "heading": "History", "section_content": "Alice founded Acme.", "content_hash": "section", "embedding_dimensions": dimensions, "vector_blob": blob}], + vectors=[ + { + "section_index": index, + "heading": "History", + "section_content": f"Alice founded Acme. Part {index}", + "content_hash": f"section-{index}", + "embedding_dimensions": dimensions, + "vector_blob": blob, + } + for index in range(3) + ], fingerprint="fp", sync_id="sync-1", ) snapshot = await store.load_vector_snapshot("mapping-1", "fp") assert saved["slug"] == "people/alice" - assert len(snapshot["rows"]) == 1 + assert store.write_batch_size == 1 + assert len(snapshot["rows"]) == 3 assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8]) assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1 diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py b/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py index 667a1dfe..b5c3ee1f 100644 --- a/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py +++ b/offline-backend-20260512/backend/tests/test_llmwiki_weknora.py @@ -690,6 +690,24 @@ async def test_weknora_client_proxies_original_document_preview_without_credenti assert media_type == "text/plain" +@pytest.mark.asyncio +async def test_weknora_client_fetches_protected_wiki_image() -> None: + async def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v1/knowledge-bases/kb-one/files" + assert request.url.params["file_path"] == "resource://wiki/figure.png" + return httpx.Response(200, content=b"png-bytes", headers={"Content-Type": "image/png"}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client: + client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client) + content, media_type = await client.get_knowledge_base_file( + "kb-one", + "resource://wiki/figure.png", + ) + + assert content == b"png-bytes" + assert media_type == "image/png" + + @pytest.mark.asyncio async def test_memory_store_enforces_owner_and_public_visibility() -> None: store = MemoryLlmWikiStore() @@ -1358,6 +1376,10 @@ def test_weknora_iframe_html_rewrites_public_prefix_paths() -> None: assert '"allowedWeKnoraIds":["remote-kb","other-visible-kb"]' in injected assert ".breadcrumb-link.dropdown" not in injected assert "reloadForKnowledgeBaseSwitch" in injected + assert "function protectedImageApiUrl" in injected + assert 'img[data-protected-src]' in injected + assert 'encodeURIComponent(cfg.weknoraId)' in injected + assert 'credentials: "same-origin"' in injected def test_weknora_iframe_filters_kb_list_to_signed_visible_mappings() -> None: diff --git a/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py b/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py index f4e2f725..a93a573f 100644 --- a/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py +++ b/offline-backend-20260512/backend/tests/test_llmwiki_wiki_sync.py @@ -1,10 +1,13 @@ from __future__ import annotations +import asyncio import json +from types import SimpleNamespace import httpx import pytest +from app.gateway import llmwiki_index_scheduler from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient from deerflow.integrations.weknora.local_index.sync import WikiSyncService @@ -85,6 +88,24 @@ async def test_configured_dimensions_are_validated_but_not_sent_to_bge() -> None assert requests == [{"model": "bge-embedding-m3", "input": ["Wiki page"]}] +@pytest.mark.asyncio +async def test_embedding_error_includes_upstream_reason(caplog) -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(400, json={"detail": "unsupported request parameter: dimensions"}) + + config = LocalWikiEmbeddingConfig( + model="bge-embedding-m3", + base_url="http://embedding.test/v1", + max_retries=0, + ) + async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client: + client = StrictWikiEmbeddingClient(config, http_client=http_client) + with pytest.raises(RuntimeError, match="unsupported request parameter: dimensions"): + await client.embed_query("Wiki page") + + assert "unsupported request parameter: dimensions" in caplog.text + + @pytest.mark.asyncio async def test_omitted_dimensions_are_discovered_from_first_response() -> None: def handler(request: httpx.Request) -> httpx.Response: @@ -167,3 +188,39 @@ async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> Non assert store.states["kb"]["state"] == "idle" assert store.states["kb"]["embedding_fingerprint"] == "fp-new" assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"} + + +@pytest.mark.asyncio +async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None: + calls: list[str] = [] + + async def fake_sleep(seconds: float) -> None: + calls.append(f"sleep:{seconds}") + raise asyncio.CancelledError + + async def fake_sync_once(app) -> None: + calls.append("sync") + + monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep) + monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once) + app = SimpleNamespace( + state=SimpleNamespace( + config=SimpleNamespace( + llmwiki=SimpleNamespace( + local_wiki_index=SimpleNamespace(sync_interval_seconds=120), + ) + ) + ) + ) + + with pytest.raises(asyncio.CancelledError): + await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app) + + assert calls == ["sleep:120"] + + +def test_local_wiki_index_accepts_legacy_sync_flag() -> None: + config = LocalWikiIndexConfig.model_validate({"sync_enabled": True}) + + assert config.auto_sync is True + assert config.sync_enabled is True diff --git a/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py b/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py index 10c63e8b..944b94c2 100644 --- a/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py +++ b/offline-backend-20260512/backend/tests/test_public_llmwiki_embed.py @@ -177,6 +177,37 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None: assert payload["pages"][0]["aliases"] == [] +@pytest.mark.asyncio +async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None: + store = PublicStore(_published_row()) + + class Client: + async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str): + assert knowledge_base_id == "remote-one" + assert file_path == "storage://tenant/resource://wiki/figure.png" + return b"png-bytes", "image/png" + + monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client()) + + response = await llmwiki.get_public_wiki_file( + _request(store), + "kb-one", + "storage://tenant/resource://wiki/figure.png", + ) + + assert store.calls == [("kb-one", "__public_wiki__", False, False)] + assert response.body == b"png-bytes" + assert response.media_type == "image/png" + assert response.headers["cache-control"] == "public, max-age=300" + + +def test_wiki_file_proxy_rejects_non_weknora_urls() -> None: + with pytest.raises(HTTPException) as exc_info: + llmwiki._validate_protected_wiki_file_path("https://example.test/private.png") + + assert exc_info.value.status_code == 422 + + @pytest.mark.asyncio async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None: store = PublicStore(_published_row())