fix: restore embedded wiki images and optimize vector sync

This commit is contained in:
吉鑫升 committed 2026-08-25 11:24:01 +08:00
1 parent 30943159bf
commit 0c5e2e8cb0
23 files changed
+609 -136

No files matched your search

@@ -37,6 +37,34 @@ const WIKI_INDEX_TYPE_LABELS: Record<string, string> = {
const WIKI_INDEX_TYPE_ORDER = ["summary", "entity", "concept"];
export const WIKI_PROTECTED_IMAGE_MARKER = "/__llmwiki-protected-image__";
const PROTECTED_WIKI_IMAGE_SOURCE =
/(?:storage:\/\/[0-9A-Za-z_-]+\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\/\/[^\s"'<>\])]+/gi;
/**
* Browsers cannot load WeKnora's protected resource:// and object-storage
* schemes directly. Replace them with a safe marker before Markdown
* sanitization; the image component resolves that marker through DeerFlow.
*/
export function rewriteProtectedWikiImageSources(markdown: string): string {
return markdown.replace(
PROTECTED_WIKI_IMAGE_SOURCE,
(filePath) => `${WIKI_PROTECTED_IMAGE_MARKER}?file_path=${encodeURIComponent(filePath)}`,
);
}
export function parseProtectedWikiImageSource(source: string | undefined): string | null {
if (!source) return null;
try {
const url = new URL(source, window.location.origin);
if (url.pathname !== WIKI_PROTECTED_IMAGE_MARKER) return null;
return url.searchParams.get("file_path")?.trim() || null;
} catch {
return null;
}
}
export function stripWikiFrontmatter(value: string) {
return value.replace(/^\s*---\s*\n[\s\S]*?\n---\s*(?:\n|$)/, "").trim();
}
@@ -444,6 +444,23 @@ export async function getLlmWikiDocumentPreview(
return response.blob();
}
export async function getLlmWikiProtectedFile(
knowledgeBaseId: string,
filePath: string,
publicAccess = false,
): Promise<Blob> {
const base = publicAccess ? PUBLIC_API() : API();
const params = new URLSearchParams({ file_path: filePath });
const response = await apiFetch(
`${base}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/files?${params.toString()}`,
);
if (!response.ok) {
await readJson<void>(response, "读取 Wiki 图片");
throw new Error("读取 Wiki 图片失败");
}
return response.blob();
}
export async function getLlmWikiGraph(knowledgeBaseId: string): Promise<LlmWikiGraph> {
return readJson(
await apiFetch(`${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/graph?limit=200`),
@@ -33,6 +33,7 @@ import { reasoningPlugins } from "@/core/streamdown";
import {
exportableWikiMarkdown,
renderableWikiMarkdown,
rewriteProtectedWikiImageSources,
safeWikiFileName,
stripWikiFrontmatter,
wikiLinkMarkdown,
@@ -49,6 +50,7 @@ import {
type LlmWikiDocument,
type LlmWikiWikiPage,
} from "@/strategy-components/api/llmwiki";
import { WikiProtectedImage } from "./WikiProtectedImage";
type SourceContextResponse = {
success?: boolean;
@@ -926,10 +928,18 @@ export function WeKnoraSourceDrawer({
</div>
<MarkdownContent
className="max-w-none"
content={preview.text}
content={rewriteProtectedWikiImageSources(preview.text)}
isLoading={false}
remarkPlugins={reasoningPlugins.remarkPlugins}
rehypePlugins={reasoningPlugins.rehypePlugins}
components={{
img: (props) => (
<WikiProtectedImage
{...props}
knowledgeBaseId={target?.knowledgeBaseId || ""}
/>
),
}}
/>
</article>
</section>
@@ -964,10 +974,18 @@ export function WeKnoraSourceDrawer({
<div className="rounded-lg border bg-background p-5">
<MarkdownContent
className="max-w-none"
content={wikiDisplayContent}
content={rewriteProtectedWikiImageSources(wikiDisplayContent)}
isLoading={false}
remarkPlugins={reasoningPlugins.remarkPlugins}
rehypePlugins={reasoningPlugins.rehypePlugins}
components={{
img: (props) => (
<WikiProtectedImage
{...props}
knowledgeBaseId={target?.knowledgeBaseId || ""}
/>
),
}}
linkClassName={(href) =>
parseWeKnoraPreviewUrl(href)?.wikiSlug
? "wiki-content-link font-medium text-[#be2323] decoration-[#be2323]/55 decoration-dashed hover:decoration-solid"
@@ -35,6 +35,7 @@ import { markdownToDocxBlob } from "@/core/scheduled-tasks/markdown-docx";
import {
exportableWikiMarkdown,
renderableWikiMarkdown,
rewriteProtectedWikiImageSources,
safeWikiFileName,
stripWikiFrontmatter,
wikiLinkMarkdown,
@@ -49,6 +50,7 @@ import {
usePublicLlmWikiWikiPage,
usePublicLlmWikiWikiPages,
} from "@/strategy-components/api/llmwiki";
import { WikiProtectedImage } from "./WikiProtectedImage";
const LIST_PATH = "/page/workspace/knowledge";
@@ -209,7 +211,7 @@ export function WeKnoraWikiPageView({
: pageData?.content,
));
const displayMarkdown = pageData
? renderableWikiMarkdown(
? rewriteProtectedWikiImageSources(renderableWikiMarkdown(
title,
content,
noteId,
@@ -222,7 +224,7 @@ export function WeKnoraWikiPageView({
: undefined,
publicEmbed ? pagesList : [],
publicEmbed ? publicWikiIndex.data?.groups || [] : [],
)
))
: "";
const exportMarkdown = pageData
? wikiLinkMarkdown(exportableWikiMarkdown(title, summary, content), noteId)
@@ -497,6 +499,15 @@ export function WeKnoraWikiPageView({
isLoading={false}
remarkPlugins={reasoningPlugins.remarkPlugins}
rehypePlugins={reasoningPlugins.rehypePlugins}
components={{
img: (props) => (
<WikiProtectedImage
{...props}
knowledgeBaseId={noteId}
publicAccess={publicEmbed}
/>
),
}}
linkClassName={(href) =>
publicEmbed && publicWikiTargetFromHref(href, noteId)
? "wiki-content-link border-b border-dashed !border-[var(--primary)] font-medium !text-primary !no-underline hover:border-solid focus-visible:rounded-sm focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-primary"
@@ -0,0 +1,75 @@
import { ImageOff, Loader2 } from "lucide-react";
import { type ImgHTMLAttributes, useEffect, useState } from "react";
import { parseProtectedWikiImageSource } from "@/core/llmwiki/wiki-page-markdown";
import { getLlmWikiProtectedFile } from "@/strategy-components/api/llmwiki";
type WikiProtectedImageProps = ImgHTMLAttributes<HTMLImageElement> & {
knowledgeBaseId: string;
publicAccess?: boolean;
};
export function WikiProtectedImage({
knowledgeBaseId,
publicAccess = false,
src,
alt,
className,
...props
}: WikiProtectedImageProps) {
const filePath = parseProtectedWikiImageSource(src);
const [objectUrl, setObjectUrl] = useState("");
const [error, setError] = useState("");
useEffect(() => {
if (!filePath || !knowledgeBaseId) {
setObjectUrl("");
setError("");
return;
}
let cancelled = false;
let nextObjectUrl = "";
setObjectUrl("");
setError("");
void getLlmWikiProtectedFile(knowledgeBaseId, filePath, publicAccess)
.then((blob) => {
if (cancelled) return;
nextObjectUrl = URL.createObjectURL(blob);
setObjectUrl(nextObjectUrl);
})
.catch((reason: unknown) => {
if (!cancelled) {
setError(reason instanceof Error ? reason.message : "Wiki 图片加载失败");
}
});
return () => {
cancelled = true;
if (nextObjectUrl) URL.revokeObjectURL(nextObjectUrl);
};
}, [filePath, knowledgeBaseId, publicAccess]);
if (!filePath) {
return <img {...props} src={src} alt={alt} className={className} />;
}
if (error) {
return (
<span
className="my-3 flex min-h-24 items-center justify-center gap-2 rounded-lg border border-dashed bg-muted/40 px-4 text-sm text-muted-foreground"
data-wiki-image-error={error}
role="img"
aria-label={alt || "Wiki 图片加载失败"}
>
<ImageOff className="size-5" />
{alt || "Wiki 图片加载失败"}
</span>
);
}
if (!objectUrl) {
return (
<span className="my-3 flex min-h-24 items-center justify-center rounded-lg bg-muted/30 text-muted-foreground">
<Loader2 className="size-5 animate-spin" aria-label="正在加载 Wiki 图片" />
</span>
);
}
return <img {...props} src={objectUrl} alt={alt} className={className} />;
}
+13 -2
View File
@@ -1016,6 +1016,11 @@ Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` me
`llmwiki.weknora.api_base_url` is the provider switch: non-empty enables WeKnora, empty preserves the legacy LLMWiki UI. WeKnora is independently deployed and owns PostgreSQL/Redis/object storage/vector storage/parsers/models; DeerFlow config contains only `api_base_url` and optional `web_base_url`. DeerFlow deliberately sends no WeKnora credential, so the WeKnora endpoint must be restricted to a trusted network and allow DeerFlow traffic itself or through its reverse proxy. There is intentionally no `default_embedding_model_id`: `WeKnoraClient` queries `/api/v1/models` and uses WeKnora's active Embedding model when creating a KB; Wiki creation additionally resolves an active KnowledgeQA model inside WeKnora for synthesis. The native BFF is `app/gateway/routers/llmwiki.py`; ownership/publication mappings live in `deerflow.persistence.llmwiki`; the browser only sees DeerFlow mapping IDs. Knowledge-base owners publish immediately (`private` -> `published`) without a review queue, and can unpublish ordinary bases. The conversation-deposit KB is system-owned, always published, shared by all DeerFlow users, excluded from My Knowledge, and immutable through user/admin detail operations; recognition requires both the reserved name and `system` owner. Existing same-name remote mappings are adopted in place as the system deposit, preserving mapping ids and document links. Its user-facing description, generated Markdown source and tags use `cmzs` branding; list/detail reads synchronize stale remote descriptions, and search results, source previews and embedded-detail JSON recursively replace legacy brand text in historical conversation-deposit payloads. Deposit requests must reference a thread accessible to the current account. Personal knowledge-base list/search views are scoped to the current DeerFlow account even for admins; frontend LLMWiki caches include the live account id, and admin write power does not make other users' private KBs appear as owned. Only `system_role === "admin"` sees the WeKnora connection-config tab. Knowledge cards navigate to a dedicated WeKnora-style detail route. Wiki-enabled bases render and default to `Wiki -> documents -> graph`; standard bases omit Wiki and render `documents -> graph`. The detail page supports sequential multi-file upload with an aggregate result for writable ordinary bases, hides it for public read-only/system-deposit bases, and uses a generic loading label while the embed session starts. It also supports URL and published manual-Markdown imports, browses real Wiki pages/content, proxies authenticated original-file previews, pages through `/chunks/{knowledge_id}`, and renders the Wiki page graph from `/knowledgebase/{kb_id}/wiki/graph`; document details use a right sheet rather than the removed KB modal. The credential-free proxy uses one WeKnora service identity, so per-user separation is enforced by DeerFlow mappings while all remote KBs physically share that identity's WeKnora workspace. WeKnora iframe detail pages first call `/api/llmwiki/knowledge-bases/{id}/weknora/session` through the normal DeerFlow API auth path, then load `/platform/knowledge-bases/{weknora_id}` with a signed `deerflow_weknora_embed` cookie for `/platform`, `/assets`, `/locales`, root assets, and API calls. Injected fetch/XHR rewriting sends all iframe `/api/v1/*`, including colliding `/api/v1/auth/*`, through `/api/llmwiki/weknora-embed/api/v1/*`; Gateway auth/CSRF bypass only applies to those signed iframe proxy requests, the proxy still revalidates the mapped KB, and cookie security honors `X-Forwarded-Proto` for HTTPS reverse proxies. `llmwiki_search` is conditionally registered in `tools.py`, is read-only, and rechecks the current DeerFlow user, agent `llmwiki_knowledge_base_ids`, and thread-context selection on every call. Source previews prefer the cleaned Wiki/LLMWiki page text when a retrieved chunk maps to a Wiki page, with raw split chunks as the fallback. Agent bindings use an inline WeKnora-style searchable/grouped multi-select in the standard agent create/detail UI; new chats use the same interaction pattern as a compact composer dropdown, and selections are written to thread metadata/context and restored on reopen. The DeerFlow frontend is native React and the WeKnora frontend is not modified or embedded. Full deployment and behavior notes: `docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`. Tests: `tests/test_llmwiki_weknora.py`.
The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. Do not add an HTML `sandbox` attribute to this trusted, permission-filtered detail iframe: a sandboxed ancestor causes Chromium's built-in Blob PDF viewer to render the browser-blocked page. The Gateway follows upstream redirects only for `/knowledge/{id}/preview` and `/knowledge-bases/{id}/files`, keeping PDF bytes and protected Wiki images inside the signed proxy instead of sending the browser to WeKnora object storage.
Native/private and public Wiki readers must rewrite protected Markdown image
sources to their mapping-scoped DeerFlow `/files?file_path=` endpoints and load
them as Blob URLs. The embedded guard also rewrites protected fetch/XHR values
and hydrates late `img[data-protected-src]` placeholders; never expose the
service token or relax the current-KB authorization to make images work.
**Embedded native knowledge-base switching:** The original WeKnora breadcrumb picker remains available inside the detail iframe. A signed 15-minute embed cookie carries only the current DeerFlow user's visible mappings (their own plus published bases); the proxied `/api/v1/knowledge-bases` response is filtered to that allowlist. Choosing another entry performs a full iframe navigation, rechecks the DeerFlow mapping and read/write permission, then reissues the cookie for the selected base. Never expose the upstream service-admin list directly or permit an arbitrary remote knowledge-base ID.
@@ -1442,8 +1447,10 @@ detail. Never call `/knowledge-search` from internal search, RAG, or the
`deerflow.integrations.weknora.local_index` and
`deerflow.persistence.llmwiki_index`; preserve the harness → app import
firewall. Gateway composition injects the store/search/sync services through
`local_index.runtime`, and the lease-protected scheduler starts only when both
`enabled` and `sync_enabled` are true.
`local_index.runtime`. `enabled` turns on local vector search and manual
vectorization; `auto_sync` is the separate opt-in for the lease-protected
scheduler. The scheduler sleeps for one full configured interval before its
first scan, so application startup must never trigger immediate vectorization.
Vectors are normalized little-endian float32 BLOBs. A page mirror and all of
its vectors are replaced in one transaction only after strict batched
@@ -1456,6 +1463,10 @@ input, not an outbound API field: fixed-dimension BGE/OpenAI-compatible
endpoints can reject a `dimensions` request parameter with HTTP 400. When the
value is omitted, the embedding client probes and freezes the first response's
dimension before sync lease/rebuild decisions.
Embedding errors must retain the upstream status and bounded response body in
logs, without exposing API keys. SQL vector writes are split into statements
of at most `database_batch_size` rows (default 50) inside the same atomic page
replacement transaction.
During a fingerprint rebuild the mapping uses the Wiki-page API until every
page is successfully rebuilt. `WikiRetrievalService` partitions mixed requests
per mapping, so ready libraries stay local while incomplete libraries use that
+19 -3
View File
@@ -777,15 +777,24 @@ processed Wiki content. A knowledge base uses exact NumPy cosine similarity
only after its whole Wiki has completed one coherent vectorization pass; until
then internal search calls WeKnora's Wiki-page search/list API and reads the
matched Wiki pages. It never falls back to raw documents or chunks. The local
vector feature is disabled by default. Configure
`llmwiki.local_wiki_index.embedding` first, then enable `enabled`,
`sync_enabled`, and `internal_search_enabled` in `config.yaml`.
vector feature is disabled by default. Configure the three
`llmwiki.local_wiki_index.embedding` values (`model`, `base_url`, and
`api_key`), then change only `enabled` to `true` in `config.yaml`. Manual
vectorization is the default. Set `auto_sync: true` only when periodic
background synchronization is wanted; the scheduler waits one full interval
after startup before its first scan, so boot does not trigger a rebuild.
`database_batch_size` caps each vector insert statement (default `50`) while
the page replacement remains atomic, preventing a large Wiki from flooding
the SQL connection in one statement.
The optional `embedding.dimensions` value is used only to validate the returned
vector size and build the index fingerprint; it is deliberately not sent in
the embedding request because fixed-dimension BGE/OpenAI-compatible endpoints
may reject that extra field with HTTP 400. Omit it to discover the dimension
from the first successful response.
Embedding failures log the upstream HTTP status and a bounded response body.
The client never sends a `dimensions` request parameter.
The migration creates `llmwiki_wiki_pages`, `llmwiki_wiki_vectors`, and
`llmwiki_wiki_sync_states`. Wiki Markdown and normalized float32 vectors are
stored in DeerFlow's configured SQL database. Multi-worker sync is protected by
@@ -839,6 +848,13 @@ system conversation-deposit knowledge base. Private or unpublished bases are
never exposed by this route. Anonymous consumers can discover the available
public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each
entry deliberately contains only the DeerFlow mapping `id` and display `name`.
Protected `resource://` and supported object-storage images are fetched through
the current mapping instead of being exposed as upstream URLs. Authenticated
readers use `GET /api/llmwiki/knowledge-bases/{id}/files?file_path=...`; public
readers use the corresponding `/api/public/llmwiki/...` route, which requires a
published mapping. The embedded WeKnora guard applies the same mapping-scoped
proxy to document previews and Markdown images, including images inserted after
the drawer/page first renders.
Wiki management also provides **导出 Excel**. `GET
/api/llmwiki/knowledge-bases/{id}/wiki/export.xlsx` exports every processed Wiki
@@ -998,7 +998,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
local_wiki_index = app.state.config.llmwiki.local_wiki_index
if (
local_wiki_index.enabled
and local_wiki_index.sync_enabled
and local_wiki_index.auto_sync
and getattr(app.state, "llmwiki_sync_service", None) is not None
):
from app.gateway.llmwiki_index_scheduler import local_wiki_index_scheduler_loop
@@ -321,13 +321,16 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
from deerflow.persistence.llmwiki_index import make_llmwiki_index_store
app.state.llmwiki_index_store = make_llmwiki_index_store(sf)
local_index_config = config.llmwiki.local_wiki_index
app.state.llmwiki_index_store = make_llmwiki_index_store(
sf,
write_batch_size=local_index_config.database_batch_size,
)
app.state.llmwiki_embedding = None
app.state.llmwiki_vector_cache = None
app.state.llmwiki_vector_search = None
app.state.llmwiki_sync_service = None
app.state.llmwiki_retrieval_service = None
local_index_config = config.llmwiki.local_wiki_index
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
resolved_llmwiki = get_resolved_llmwiki_runtime(config)
@@ -48,6 +48,10 @@ async def _sync_once(app: FastAPI) -> None:
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds
logger.info(
"Local Wiki auto-sync scheduler started; first scan delayed by %s seconds",
interval,
)
while True:
await _sync_once(app)
await asyncio.sleep(interval)
await _sync_once(app)
@@ -61,6 +61,10 @@ router = APIRouter(tags=["llmwiki"])
proxy_router = APIRouter(tags=["llmwiki-proxy"])
_MAX_UPLOAD_BYTES = 100 * 1024 * 1024
_PROTECTED_WIKI_FILE_PATH = re.compile(
r"^(?:storage://[0-9A-Za-z_-]+/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs)://",
re.IGNORECASE,
)
_WEKNORA_ADMIN_SESSION_CACHE: dict[str, dict[str, Any]] = {}
_WEKNORA_PUBLIC_PREFIX = "/deerflow"
_WEKNORA_PUBLIC_ROOT_PATHS = (
@@ -242,7 +246,7 @@ def _store(request: Request) -> LlmWikiStore:
def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None:
service = getattr(request.app.state, "llmwiki_sync_service", None)
config = request.app.state.config.llmwiki.local_wiki_index
if service is not None and config.enabled and mapping.get("wiki_index_enabled", True):
if service is not None and config.enabled and config.auto_sync and mapping.get("wiki_index_enabled", True):
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
@@ -1085,6 +1089,8 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
function rewriteApiUrl(value) {{
try {{
const protectedUrl = protectedImageApiUrl(value);
if (protectedUrl) return protectedUrl;
const next = new URL(value, location.origin);
if (next.origin !== location.origin) return value;
const publicPrefix = cfg.publicPrefix || "";
@@ -1104,6 +1110,19 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
return value;
}}
}}
function protectedImageApiUrl(value) {{
const raw = String(value || "").trim();
if (!/^(?:storage:\\/\\/[0-9A-Za-z_-]+\\/)?(?:resource|local|minio|cos|tos|s3|oss|ks3|obs):\\/\\//i.test(raw)) {{
return "";
}}
const endpoint =
cfg.apiPrefix +
"/api/v1/knowledge-bases/" +
encodeURIComponent(cfg.weknoraId) +
"/files?file_path=" +
encodeURIComponent(raw);
return new URL(endpoint, location.origin).href;
}}
const nativeFetch = window.fetch.bind(window);
window.fetch = (input, init) => {{
if (input instanceof Request) {{
@@ -1116,6 +1135,67 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
return nativeXhrOpen.call(this, method, rewriteApiUrl(url), ...rest);
}};
// WeKnora renders protected Markdown images as transparent placeholders and
// hydrates them asynchronously. In an embedded page that hydration can run
// before its own API client has picked up our proxy prefix, leaving a 1x1
// GIF forever. Resolve the same protected source through the scoped proxy
// as a fallback. The backend validates the scheme and current KB again.
function hydrateProtectedImage(image) {{
const protectedSrc = String(image?.getAttribute?.("data-protected-src") || "").trim();
const proxyUrl = protectedImageApiUrl(protectedSrc);
if (!proxyUrl) return;
if (
image.dataset.deerflowProtectedSource === protectedSrc &&
String(image.getAttribute("src") || "").startsWith("blob:")
) return;
const attempts = Number(image.dataset.deerflowProtectedAttempts || "0");
if (attempts >= 3 || image.dataset.deerflowProtectedLoading === "1") return;
image.dataset.deerflowProtectedLoading = "1";
image.dataset.deerflowProtectedAttempts = String(attempts + 1);
nativeFetch(proxyUrl, {{ credentials: "same-origin", headers: {{ Accept: "image/*" }} }})
.then((response) => {{
if (!response.ok) throw new Error(`protected image request failed (${{response.status}})`);
return response.blob();
}})
.then((blob) => {{
const objectUrl = URL.createObjectURL(blob);
image.dataset.deerflowProtectedSource = protectedSrc;
image.removeAttribute("data-img-loading");
image.removeAttribute("data-wiki-image-error");
image.src = objectUrl;
image.addEventListener("load", () => URL.revokeObjectURL(objectUrl), {{ once: true }});
}})
.catch(() => {{
image.dataset.wikiImageError = "true";
window.setTimeout(() => {{
delete image.dataset.deerflowProtectedLoading;
hydrateProtectedImage(image);
}}, 1200 * (attempts + 1));
}})
.finally(() => {{
delete image.dataset.deerflowProtectedLoading;
}});
}}
function hydrateProtectedImages(scope = document) {{
if (scope?.matches?.("img[data-protected-src]")) hydrateProtectedImage(scope);
scope?.querySelectorAll?.("img[data-protected-src]").forEach(hydrateProtectedImage);
}}
const protectedImageObserver = new MutationObserver((mutations) => {{
mutations.forEach((mutation) => {{
if (mutation.type === "attributes") hydrateProtectedImage(mutation.target);
mutation.addedNodes?.forEach((node) => {{
if (node?.nodeType === Node.ELEMENT_NODE) hydrateProtectedImages(node);
}});
}});
}});
protectedImageObserver.observe(document.documentElement, {{
childList: true,
subtree: true,
attributes: true,
attributeFilter: ["data-protected-src"]
}});
hydrateProtectedImages();
function tone(color, alpha) {{
return `color-mix(in srgb, ${{color}} ${{Math.round(alpha * 100)}}%, transparent)`;
}}
@@ -1519,6 +1599,7 @@ html.deerflow-source-detail-open [data-deerflow-source-detail-panel="true"] [cla
enforceLocation();
purgeGuides();
hideWeKnoraShell();
hydrateProtectedImages();
if (tickSourceDetail() && sourceDetailMode) {{
clearInterval(maintenanceTimer);
try {{ readyObserver.disconnect(); }} catch (_) {{}}
@@ -1820,6 +1901,32 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
return row
def _validate_protected_wiki_file_path(file_path: str) -> str:
value = file_path.strip()
if not value or "\x00" in value or not _PROTECTED_WIKI_FILE_PATH.match(value):
raise HTTPException(status_code=422, detail="Unsupported Wiki image path")
return value
async def _wiki_file_response(row: dict[str, Any], file_path: str, *, public: bool) -> Response:
try:
content, media_type = await _client_or_503().get_knowledge_base_file(
str(row["weknora_id"]),
_validate_protected_wiki_file_path(file_path),
)
except WeKnoraError as exc:
_raise_upstream(exc)
return Response(
content=content,
media_type=media_type,
headers={
"Cache-Control": "public, max-age=300" if public else "private, max-age=60",
"Content-Disposition": "inline",
"X-Content-Type-Options": "nosniff",
},
)
async def _verify_document(client: WeKnoraClient, row: dict[str, Any], document_id: str) -> dict[str, Any]:
document = await client.get_document(document_id)
if str(document.get("knowledge_base_id") or "") != str(row["weknora_id"]):
@@ -2510,6 +2617,18 @@ async def list_public_wiki_pages(
return {**result, "pages": pages}
@router.get("/api/public/llmwiki/knowledge-bases/{mapping_id}/files")
async def get_public_wiki_file(
request: Request,
mapping_id: str,
file_path: str = Query(..., min_length=1, max_length=8192),
) -> Response:
"""Serve one protected image belonging to a published Wiki."""
row = await _public_mapping(request, mapping_id)
return await _wiki_file_response(row, file_path, public=True)
_PUBLIC_WIKI_INDEX_TYPES = ("summary", "entity", "concept", "synthesis", "comparison")
@@ -2820,6 +2939,18 @@ async def preview_document(request: Request, mapping_id: str, document_id: str)
)
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/files")
async def get_wiki_file(
request: Request,
mapping_id: str,
file_path: str = Query(..., min_length=1, max_length=8192),
) -> Response:
"""Serve one protected Wiki image after normal knowledge-base authorization."""
row, _, _ = await _authorized_mapping(request, mapping_id, write=False)
return await _wiki_file_response(row, file_path, public=False)
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/graph")
async def get_knowledge_base_graph(
request: Request,
@@ -2,6 +2,7 @@
from __future__ import annotations
import logging
from typing import Any
from uuid import uuid4
@@ -14,6 +15,7 @@ from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectori
from deerflow.integrations.weknora.local_index.search import WikiIndexError
router = APIRouter(prefix="/api/llmwiki", tags=["llmwiki-index"])
logger = logging.getLogger(__name__)
class WikiVectorSearchRequest(BaseModel):
@@ -181,6 +183,7 @@ async def _run_sync(app, mapping_ids: list[str], force: bool) -> None:
try:
await service.sync_mapping(mapping, force=force)
except Exception:
logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id"))
continue
@@ -191,6 +194,7 @@ async def _run_one_sync(app, mapping: dict[str, Any], *, force: bool) -> None:
try:
await service.sync_mapping(mapping, force=force)
except Exception:
logger.exception("Manual Wiki vectorization failed mapping_id=%s", mapping.get("id"))
return
+5 -35
View File
@@ -556,47 +556,17 @@ llmwiki:
web_base_url: "http://119.254.86.251:8088"
admin_email: "lqq@qq.com"
admin_password: "1qaz@WSX"
# DeerFlow-local Wiki mirror/vector index. Keep disabled until the
# OpenAI-compatible embedding endpoint below is configured and reachable.
# 本地 Wiki 向量化:只需把 enabled 改为 true,并填写下面 3 个 embedding 值。
# 默认只在页面点击“向量化”时执行;auto_sync=true 才会后台自动增量同步,
# 且服务启动时只启动定时器,不会立刻全量向量化。
local_wiki_index:
enabled: false
sync_enabled: false
internal_search_enabled: false
reference_ui_enabled: false
sync_interval_seconds: 120
chunk_max_chars: 1000
chunk_overlap_chars: 120
chunk_min_chars: 80
auto_sync: false
database_batch_size: 50
embedding:
enabled: true
provider: openai
model: bge-m3
base_url: ""
api_key: ""
# Expected BGE-M3 response size for local validation/indexing only.
# This value is not sent as an embedding API request parameter.
dimensions: 1024
batch_size: 32
timeout_seconds: 30
max_retries: 3
engine: numpy_exact
strict_vector: true
top_k_sections: 30
top_k_pages: 8
max_sections_per_page: 2
min_similarity: 0.35
cache_max_knowledge_bases: 20
cache_revision_check_seconds: 5
external_api:
enabled: false
api_key: ""
previous_api_key: ""
max_query_chars: 1000
max_knowledge_bases: 20
max_results: 50
default_results: 10
include_content_max_chars: 12000
requests_per_minute: 60
log_system:
enabled: true
@@ -279,65 +279,25 @@ llmwiki:
local_wiki_index:
enabled: true
# 同步
sync_enabled: true
sync_interval_seconds: 120
sync_page_size: 100
sync_concurrency: 3
sync_lease_seconds: 600
sync_request_timeout_seconds: 30
sync_max_retries: 3
sync_retry_backoff_seconds: 5
# 切分
chunk_max_chars: 1000
chunk_overlap_chars: 120
chunk_min_chars: 80
title_prefix_enabled: true
# Embedding:OpenAI-compatible /v1/embeddings
# 默认只允许手动向量化;需要定时增量同步时再开启。
auto_sync: false
# 每条 SQL 最多插入的向量行数;整页替换仍在同一事务内完成。
database_batch_size: 50
embedding:
enabled: true
provider: openai
model: bge-m3
base_url: "$WIKI_EMBEDDING_BASE_URL"
api_key: "$WIKI_EMBEDDING_API_KEY"
dimensions: 1024
batch_size: 32
timeout_seconds: 30
max_retries: 3
# 检索
engine: numpy_exact
strict_vector: true
top_k_sections: 30
top_k_pages: 8
max_sections_per_page: 2
min_similarity: 0.35
keyword_boost_enabled: false
keyword_boost_weight: 0.15
cache_max_knowledge_bases: 20
cache_revision_check_seconds: 5
# 外部接口
external_api:
enabled: false
api_key: "$LLMWIKI_EXTERNAL_SEARCH_API_KEY"
max_query_chars: 1000
max_knowledge_bases: 20
max_results: 50
default_results: 10
include_content_max_chars: 12000
requests_per_minute: 60
```
日常部署只需配置上面三个 Embedding 值并将 `enabled` 改为 `true`。
`auto_sync=false` 时仅由页面“手动向量化”触发;即使设为 `true`,定时器也会先等待一个完整同步周期,不会在服务刚启动时立即重建索引。其余切分、检索、超时与重试参数继续使用代码默认值,只有确有调优需要时才覆盖。
### 6.1 配置校验规则
启动时应执行:
1. `enabled=true` 时,Embedding `base_url` 和 `model` 必须非空;
2. `dimensions` 可省略,但第一次成功 Embedding 后必须固定并写入索引 profile;
2. 不向上游发送 `dimensions` 参数;第一次成功 Embedding 后从响应自动发现维度,并固定到索引 profile;
3. `chunk_overlap_chars < chunk_max_chars`;
4. `top_k_pages <= top_k_sections`;
5. `external_api.enabled=true` 时必须配置非空 API Key;
@@ -1885,9 +1845,7 @@ class WikiVectorEngine(ABC):
```text
local_wiki_index.enabled
local_wiki_index.sync_enabled
local_wiki_index.internal_search_enabled
local_wiki_index.reference_ui_enabled
local_wiki_index.auto_sync
local_wiki_index.external_api.enabled
```
@@ -59,7 +59,7 @@ class LocalWikiEmbeddingConfig(BaseModel):
base_url: str = ""
api_key: str = Field(default="", repr=False)
dimensions: int | None = Field(
default=1024,
default=None,
ge=1,
le=65536,
description="Expected response vector dimensions; not sent to the embedding endpoint.",
@@ -108,9 +108,7 @@ class LocalWikiIndexConfig(BaseModel):
"""DeerFlow-owned mirror and exact-vector index for WeKnora Wiki pages."""
enabled: bool = False
sync_enabled: bool = False
internal_search_enabled: bool = False
reference_ui_enabled: bool = False
auto_sync: bool = False
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
sync_page_size: int = Field(default=100, ge=1, le=500)
sync_concurrency: int = Field(default=3, ge=1, le=32)
@@ -133,9 +131,31 @@ class LocalWikiIndexConfig(BaseModel):
keyword_boost_weight: float = Field(default=0.15, ge=0, le=1)
cache_max_knowledge_bases: int = Field(default=20, ge=1, le=1000)
cache_revision_check_seconds: float = Field(default=5, ge=0, le=3600)
database_batch_size: int = Field(default=50, ge=1, le=500)
external_api: LocalWikiExternalApiConfig = Field(default_factory=LocalWikiExternalApiConfig)
model_config = ConfigDict(extra="ignore")
@model_validator(mode="before")
@classmethod
def _migrate_legacy_flags(cls, value):
if isinstance(value, dict) and "auto_sync" not in value and "sync_enabled" in value:
value = {**value, "auto_sync": value.get("sync_enabled")}
return value
@property
def sync_enabled(self) -> bool:
"""Backward-compatible status name; automatic work is one explicit switch."""
return self.auto_sync
@property
def internal_search_enabled(self) -> bool:
return self.enabled
@property
def reference_ui_enabled(self) -> bool:
return self.enabled
@model_validator(mode="after")
def _validate_index(self):
if self.chunk_overlap_chars >= self.chunk_max_chars:
@@ -2,8 +2,8 @@
from __future__ import annotations
from collections.abc import Mapping, Sequence
import time
from collections.abc import Mapping, Sequence
from typing import Any
from urllib.parse import quote
@@ -430,6 +430,51 @@ class WeKnoraClient:
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
return response.content, media_type or "application/octet-stream"
async def get_knowledge_base_file(
self,
knowledge_base_id: str,
file_path: str,
) -> tuple[bytes, str]:
"""Return one protected Wiki attachment using DeerFlow's WeKnora identity."""
owns_client = self._http_client is None
client = self._http_client or httpx.AsyncClient(
timeout=httpx.Timeout(self.timeout_seconds),
follow_redirects=True,
trust_env=False,
)
try:
for attempt in range(2):
response = await client.request(
"GET",
f"{self.base_url}/api/v1/knowledge-bases/{quote(knowledge_base_id, safe='')}/files",
params={"file_path": file_path},
headers=await self._headers(force_refresh=attempt > 0, accept="*/*"),
follow_redirects=True,
)
if response.status_code not in {401, 403} or not self._has_admin_credentials or attempt > 0:
break
except httpx.TimeoutException as exc:
raise WeKnoraError("WeKnora Wiki image request timed out", status_code=504) from exc
except httpx.HTTPError as exc:
raise WeKnoraError("WeKnora Wiki image is unavailable", status_code=502) from exc
finally:
if owns_client:
await client.aclose()
if response.status_code == 404:
raise WeKnoraError("WeKnora Wiki image was not found", status_code=404)
if response.status_code in {401, 403}:
if self._has_admin_credentials:
raise WeKnoraError("WeKnora rejected DeerFlow's authenticated Wiki image request", status_code=502)
raise WeKnoraError(
"WeKnora rejected DeerFlow's anonymous internal request; allow DeerFlow in WeKnora or its reverse proxy",
status_code=502,
)
if response.status_code >= 400:
raise WeKnoraError(f"WeKnora returned HTTP {response.status_code}", status_code=502)
media_type = response.headers.get("content-type", "application/octet-stream").split(";", 1)[0].strip()
return response.content, media_type or "application/octet-stream"
async def reprocess_document(self, knowledge_id: str) -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledge/{quote(knowledge_id, safe='')}/reparse"))
return dict(value) if isinstance(value, dict) else {}
@@ -15,6 +15,21 @@ from deerflow.integrations.weknora.local_index.vector_codec import normalize_vec
logger = logging.getLogger(__name__)
def _upstream_error_detail(response: httpx.Response) -> str:
"""Return a bounded upstream reason without hiding the useful response body."""
try:
payload = response.json()
if isinstance(payload, dict):
detail = payload.get("detail") or payload.get("message") or payload.get("error") or payload
else:
detail = payload
value = str(detail)
except (ValueError, TypeError):
value = response.text
return " ".join(value.split())[:2000] or "no response body"
class WikiEmbeddingError(RuntimeError):
def __init__(self, message: str, *, category: str = "request", retryable: bool = True) -> None:
super().__init__(message)
@@ -76,11 +91,15 @@ class StrictWikiEmbeddingClient:
started = monotonic()
try:
response = await (await self._get_client()).post(url, headers=headers, json=payload)
if response.status_code in {401, 403}:
raise WikiEmbeddingError("Embedding authentication failed", category="authentication", retryable=False)
if response.status_code == 429 or response.status_code >= 500:
raise WikiEmbeddingError(f"Embedding service returned HTTP {response.status_code}", category="upstream", retryable=True)
response.raise_for_status()
if response.status_code >= 400:
detail = _upstream_error_detail(response)
category = "authentication" if response.status_code in {401, 403} else "upstream"
retryable = response.status_code == 429 or response.status_code >= 500
raise WikiEmbeddingError(
f"Embedding service returned HTTP {response.status_code}: {detail}",
category=category,
retryable=retryable,
)
data = response.json()
items = sorted(data.get("data", []), key=lambda item: int(item.get("index", 0))) if isinstance(data, dict) else []
if len(items) != len(texts):
@@ -98,8 +117,20 @@ class StrictWikiEmbeddingClient:
except httpx.TimeoutException:
last_error = WikiEmbeddingError("Embedding request timed out", category="timeout", retryable=True)
except (httpx.HTTPError, ValueError, TypeError) as exc:
last_error = WikiEmbeddingError(f"Embedding request failed ({type(exc).__name__})", category="request", retryable=isinstance(exc, httpx.HTTPError))
reason = " ".join(str(exc).split())[:2000] or type(exc).__name__
last_error = WikiEmbeddingError(
f"Embedding request failed ({type(exc).__name__}): {reason}",
category="request",
retryable=isinstance(exc, httpx.HTTPError),
)
if last_error is not None and (not last_error.retryable or attempt >= self.config.max_retries):
logger.error(
"Wiki embedding batch failed model=%s batch=%d category=%s reason=%s",
self.config.model,
len(texts),
last_error.category,
last_error,
)
raise last_error
await asyncio.sleep(min(30.0, 0.5 * (2**attempt)))
raise last_error or WikiEmbeddingError("Embedding request failed")
@@ -6,8 +6,12 @@ from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWiki
from deerflow.persistence.llmwiki_index.sql import SqlLlmWikiIndexStore
def make_llmwiki_index_store(session_factory):
return MemoryLlmWikiIndexStore() if session_factory is None else SqlLlmWikiIndexStore(session_factory)
def make_llmwiki_index_store(session_factory, *, write_batch_size: int = 50):
return (
MemoryLlmWikiIndexStore()
if session_factory is None
else SqlLlmWikiIndexStore(session_factory, write_batch_size=write_batch_size)
)
__all__ = [
@@ -6,7 +6,7 @@ from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import case, delete, func, or_, select, update
from sqlalchemy import case, delete, func, insert, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
@@ -27,8 +27,9 @@ def _dict(row) -> dict[str, Any]:
class SqlLlmWikiIndexStore(LlmWikiIndexStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
def __init__(self, session_factory: async_sessionmaker[AsyncSession], *, write_batch_size: int = 50) -> None:
self._sf = session_factory
self.write_batch_size = max(1, min(int(write_batch_size), 500))
async def _ensure_state(self, session: AsyncSession, mapping_id: str) -> LlmWikiWikiSyncStateRow:
row = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
@@ -82,24 +83,29 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
row.updated_at = now
await session.flush()
await session.execute(delete(LlmWikiWikiVectorRow).where(LlmWikiWikiVectorRow.page_id == row.id))
session.add_all(
[
LlmWikiWikiVectorRow(
id=str(uuid4()),
page_id=row.id,
knowledge_base_mapping_id=mapping_id,
section_index=int(item["section_index"]),
heading=item.get("heading"),
section_content=str(item["section_content"]),
content_hash=str(item["content_hash"]),
embedding_fingerprint=fingerprint,
embedding_dimensions=int(item["embedding_dimensions"]),
vector_blob=bytes(item["vector_blob"]),
created_at=now,
)
for item in vectors
]
)
vector_rows = [
{
"id": str(uuid4()),
"page_id": row.id,
"knowledge_base_mapping_id": mapping_id,
"section_index": int(item["section_index"]),
"heading": item.get("heading"),
"section_content": str(item["section_content"]),
"content_hash": str(item["content_hash"]),
"embedding_fingerprint": fingerprint,
"embedding_dimensions": int(item["embedding_dimensions"]),
"vector_blob": bytes(item["vector_blob"]),
"created_at": now,
}
for item in vectors
]
# Keep the page replacement atomic, but cap each SQL statement so
# large Wiki pages cannot flood the database with one huge insert.
for start in range(0, len(vector_rows), self.write_batch_size):
await session.execute(
insert(LlmWikiWikiVectorRow),
vector_rows[start : start + self.write_batch_size],
)
state = await self._ensure_state(session, mapping_id)
state.index_revision = int(state.index_revision or 0) + 1
state.embedding_fingerprint = fingerprint
@@ -40,20 +40,31 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
)
await session.commit()
store = SqlLlmWikiIndexStore(session_factory)
store = SqlLlmWikiIndexStore(session_factory, write_batch_size=1)
assert await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-1", lease_seconds=60, fingerprint="fp")
assert not await store.try_acquire_sync_lease("mapping-1", owner="worker", sync_id="sync-2", lease_seconds=60, fingerprint="fp")
page = normalize_wiki_page("mapping-1", {"id": "p1", "slug": "people/alice", "title": "Alice", "content": "Alice founded Acme.", "status": "published"})
blob, dimensions = encode_vector([3.0, 4.0], dimensions=2)
saved = await store.replace_page_vectors(
page=page,
vectors=[{"section_index": 0, "heading": "History", "section_content": "Alice founded Acme.", "content_hash": "section", "embedding_dimensions": dimensions, "vector_blob": blob}],
vectors=[
{
"section_index": index,
"heading": "History",
"section_content": f"Alice founded Acme. Part {index}",
"content_hash": f"section-{index}",
"embedding_dimensions": dimensions,
"vector_blob": blob,
}
for index in range(3)
],
fingerprint="fp",
sync_id="sync-1",
)
snapshot = await store.load_vector_snapshot("mapping-1", "fp")
assert saved["slug"] == "people/alice"
assert len(snapshot["rows"]) == 1
assert store.write_batch_size == 1
assert len(snapshot["rows"]) == 3
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
assert await store.mark_missing_pages_deleted("mapping-1", "sync-2") == 1
@@ -690,6 +690,24 @@ async def test_weknora_client_proxies_original_document_preview_without_credenti
assert media_type == "text/plain"
@pytest.mark.asyncio
async def test_weknora_client_fetches_protected_wiki_image() -> None:
async def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/api/v1/knowledge-bases/kb-one/files"
assert request.url.params["file_path"] == "resource://wiki/figure.png"
return httpx.Response(200, content=b"png-bytes", headers={"Content-Type": "image/png"})
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
client = WeKnoraClient(base_url="http://weknora.local:8080", http_client=http_client)
content, media_type = await client.get_knowledge_base_file(
"kb-one",
"resource://wiki/figure.png",
)
assert content == b"png-bytes"
assert media_type == "image/png"
@pytest.mark.asyncio
async def test_memory_store_enforces_owner_and_public_visibility() -> None:
store = MemoryLlmWikiStore()
@@ -1358,6 +1376,10 @@ def test_weknora_iframe_html_rewrites_public_prefix_paths() -> None:
assert '"allowedWeKnoraIds":["remote-kb","other-visible-kb"]' in injected
assert ".breadcrumb-link.dropdown" not in injected
assert "reloadForKnowledgeBaseSwitch" in injected
assert "function protectedImageApiUrl" in injected
assert 'img[data-protected-src]' in injected
assert 'encodeURIComponent(cfg.weknoraId)' in injected
assert 'credentials: "same-origin"' in injected
def test_weknora_iframe_filters_kb_list_to_signed_visible_mappings() -> None:
@@ -1,10 +1,13 @@
from __future__ import annotations
import asyncio
import json
from types import SimpleNamespace
import httpx
import pytest
from app.gateway import llmwiki_index_scheduler
from deerflow.config.llmwiki_config import LocalWikiEmbeddingConfig, LocalWikiIndexConfig
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
@@ -85,6 +88,24 @@ async def test_configured_dimensions_are_validated_but_not_sent_to_bge() -> None
assert requests == [{"model": "bge-embedding-m3", "input": ["Wiki page"]}]
@pytest.mark.asyncio
async def test_embedding_error_includes_upstream_reason(caplog) -> None:
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(400, json={"detail": "unsupported request parameter: dimensions"})
config = LocalWikiEmbeddingConfig(
model="bge-embedding-m3",
base_url="http://embedding.test/v1",
max_retries=0,
)
async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as http_client:
client = StrictWikiEmbeddingClient(config, http_client=http_client)
with pytest.raises(RuntimeError, match="unsupported request parameter: dimensions"):
await client.embed_query("Wiki page")
assert "unsupported request parameter: dimensions" in caplog.text
@pytest.mark.asyncio
async def test_omitted_dimensions_are_discovered_from_first_response() -> None:
def handler(request: httpx.Request) -> httpx.Response:
@@ -167,3 +188,39 @@ async def test_fingerprint_change_automatically_rebuilds_entire_mapping() -> Non
assert store.states["kb"]["state"] == "idle"
assert store.states["kb"]["embedding_fingerprint"] == "fp-new"
assert {vector["embedding_fingerprint"] for vectors in store.vectors.values() for vector in vectors} == {"fp-new"}
@pytest.mark.asyncio
async def test_auto_sync_scheduler_waits_before_first_scan(monkeypatch) -> None:
calls: list[str] = []
async def fake_sleep(seconds: float) -> None:
calls.append(f"sleep:{seconds}")
raise asyncio.CancelledError
async def fake_sync_once(app) -> None:
calls.append("sync")
monkeypatch.setattr(llmwiki_index_scheduler.asyncio, "sleep", fake_sleep)
monkeypatch.setattr(llmwiki_index_scheduler, "_sync_once", fake_sync_once)
app = SimpleNamespace(
state=SimpleNamespace(
config=SimpleNamespace(
llmwiki=SimpleNamespace(
local_wiki_index=SimpleNamespace(sync_interval_seconds=120),
)
)
)
)
with pytest.raises(asyncio.CancelledError):
await llmwiki_index_scheduler.local_wiki_index_scheduler_loop(app)
assert calls == ["sleep:120"]
def test_local_wiki_index_accepts_legacy_sync_flag() -> None:
config = LocalWikiIndexConfig.model_validate({"sync_enabled": True})
assert config.auto_sync is True
assert config.sync_enabled is True
@@ -177,6 +177,37 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None:
assert payload["pages"][0]["aliases"] == []
@pytest.mark.asyncio
async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None:
store = PublicStore(_published_row())
class Client:
async def get_knowledge_base_file(self, knowledge_base_id: str, file_path: str):
assert knowledge_base_id == "remote-one"
assert file_path == "storage://tenant/resource://wiki/figure.png"
return b"png-bytes", "image/png"
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
response = await llmwiki.get_public_wiki_file(
_request(store),
"kb-one",
"storage://tenant/resource://wiki/figure.png",
)
assert store.calls == [("kb-one", "__public_wiki__", False, False)]
assert response.body == b"png-bytes"
assert response.media_type == "image/png"
assert response.headers["cache-control"] == "public, max-age=300"
def test_wiki_file_proxy_rejects_non_weknora_urls() -> None:
with pytest.raises(HTTPException) as exc_info:
llmwiki._validate_protected_wiki_file_path("https://example.test/private.png")
assert exc_info.value.status_code == 422
@pytest.mark.asyncio
async def test_public_wiki_index_preserves_weknora_group_and_item_order(monkeypatch) -> None:
store = PublicStore(_published_row())