Merge branch 'ji' into master

This commit is contained in:
吉鑫升 committed 2026-09-03 10:25:57 +08:00
commit 49d9ced04f
83 files changed
+12909 -313

No files matched your search

+4
View File
@@ -88,10 +88,14 @@ skills/
hermes-agent-main/
.gitignore
offline-backend-20260512/backend/tests/*
# Keep the openGauss URL/dialect/schema compatibility regression coverage versioned.
!offline-backend-20260512/backend/tests/test_gauss_database.py
!offline-backend-20260512/backend/tests/test_llmwiki_weknora.py
!offline-backend-20260512/backend/tests/test_health_auth_bypass.py
# Keep formal Word OOXML and embedded-font regression coverage versioned.
!offline-backend-20260512/backend/tests/test_word_export.py
# Keep generated-file library aggregation and filtering regression coverage versioned.
!offline-backend-20260512/backend/tests/test_artifact_library.py
# Keep the model configuration regression coverage versioned even though
# this deployment repository otherwise ignores newly added backend tests.
!offline-backend-20260512/backend/tests/test_model_management.py
@@ -0,0 +1,7 @@
# 技能知识归纳与“知识梳理”助手知识库
管理员在「技能管理 → 技能知识归纳」中选择一个或多个技能、一个或多个目标,系统按“技能 × 目标”创建后台任务。界面提供总体进度、单项失败重试、实体/关系审核、绑定重归纳、自动同步和两种解绑动作。技能列表通过一次 bindings 查询展示归纳目标,避免逐技能请求。
扫描仅处理知识资料。代码、脚本、锁文件、密钥文件以及 Markdown 代码块不会进入上传、摘要、提示词或知识指纹;ZIP 只读取逐项验证通过的知识成员。自动监听只比较该知识指纹,所以纯代码改动不会产生 stale 状态。
知识库页面新增「普通知识库 / 助手知识库」并行入口。管理员可将系统默认入口设为 v1(普通 WeKnora)或 v2(全局“知识梳理”)。助手库无需 WeKnora 即可初始化,所有用户可只读查看和搜索 Wiki;管理员可上传本地文件、从普通库导入、重新导入或删除来源、查看版本并回滚。来源删除只移除对应贡献,共享实体与关系继续保留。
File diff suppressed because it is too large. Load diff
@@ -1,7 +1,7 @@
import type { Message } from "@langchain/langgraph-sdk";
import { useQuery } from "@tanstack/react-query";
import { FilesIcon, XIcon } from "lucide-react";
import { usePathname } from "next/navigation";
import { usePathname, useSearchParams } from "next/navigation";
import { useEffect, useMemo, useRef, useState } from "react";
import type { GroupImperativeHandle } from "react-resizable-panels";
@@ -15,6 +15,7 @@ import {
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { apiFetch } from "@/core/api";
import {
artifactPathname,
listThreadArtifacts,
mergeArtifactPaths,
} from "@/core/artifacts/listing";
@@ -94,7 +95,13 @@ const ChatBox: React.FC<{
knowledgeReferencesEnabled = false,
} = useThread();
const pathname = usePathname();
const searchParams = useSearchParams();
const requestedArtifact = useMemo(() => {
const value = searchParams.get("artifact")?.trim();
return value ? artifactPathname(value) : null;
}, [searchParams]);
const threadIdRef = useRef(threadId);
const openedRequestedArtifactRef = useRef<string | null>(null);
const artifactsCountRef = useRef(0);
const layoutRef = useRef<GroupImperativeHandle>(null);
const latestReferenceBatchIdRef = useRef<string | null>(null);
@@ -135,7 +142,7 @@ const ChatBox: React.FC<{
const { skills } = useSkills();
const canScanArtifactDirectory =
artifactDirectoryModeEnabled &&
(artifactDirectoryModeEnabled || !!requestedArtifact) &&
!disableArtifacts &&
!isMock &&
!!threadId &&
@@ -153,17 +160,21 @@ const ChatBox: React.FC<{
);
const checkpointArtifactSignature = checkpointArtifacts.join("\n");
const availableArtifacts = useMemo(
() =>
artifactDirectoryModeEnabled
? mergeArtifactPaths(
checkpointArtifacts,
(artifactListingQuery.data?.files ?? []).map((file) => file.path),
)
: checkpointArtifacts,
() => {
if (!artifactDirectoryModeEnabled && !requestedArtifact) {
return checkpointArtifacts;
}
const discoveredArtifacts = (artifactListingQuery.data?.files ?? []).map(
(file) => file.path,
);
if (requestedArtifact) discoveredArtifacts.push(requestedArtifact);
return mergeArtifactPaths(checkpointArtifacts, discoveredArtifacts);
},
[
artifactDirectoryModeEnabled,
artifactListingQuery.data?.files,
checkpointArtifacts,
requestedArtifact,
],
);
@@ -207,6 +218,29 @@ const ChatBox: React.FC<{
thread.isLoading,
);
}, [referencesEnabled, skillDisplayMap, thread.isLoading, thread.values.messages]);
useEffect(() => {
if (
disableArtifacts ||
!requestedArtifact ||
!threadId ||
threadId === "new"
) {
return;
}
const requestKey = `${threadId}:${artifactPathname(requestedArtifact)}`;
if (openedRequestedArtifactRef.current === requestKey) return;
openedRequestedArtifactRef.current = requestKey;
selectArtifact(requestedArtifact);
setArtifactsOpen(true);
setSideTab("document");
}, [
disableArtifacts,
requestedArtifact,
selectArtifact,
setArtifactsOpen,
threadId,
]);
useEffect(() => {
if (!shouldFetchBackendReferenceBatches) {
setBackendReferenceBatches([]);
@@ -0,0 +1,57 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { BookOpen, Database } from "lucide-react";
import { useLocation, useNavigate } from "react-router-dom";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { getStoredAuth } from "@/core/auth";
import {
loadKnowledgeBaseProductVersion,
type KnowledgeBaseProductVersion,
updateKnowledgeBaseProductVersion,
} from "@/core/system-settings/api";
export const KNOWLEDGE_PRODUCT_KEY = ["system-settings", "knowledge-base-product-version"] as const;
export const KNOWLEDGE_NORMAL_VIEW_VALUE = "normal";
export const KNOWLEDGE_NORMAL_VIEW_SEARCH = `?view=${KNOWLEDGE_NORMAL_VIEW_VALUE}`;
export function useKnowledgeBaseProductVersion() {
return useQuery({ queryKey: KNOWLEDGE_PRODUCT_KEY, queryFn: loadKnowledgeBaseProductVersion, staleTime: 60_000 });
}
export function KnowledgeProductNav() {
const location = useLocation();
const navigate = useNavigate();
const client = useQueryClient();
const version = useKnowledgeBaseProductVersion();
const isAdmin = getStoredAuth()?.system_role === "admin";
const save = useMutation({
mutationFn: updateKnowledgeBaseProductVersion,
onSuccess: (value) => {
client.setQueryData(KNOWLEDGE_PRODUCT_KEY, value);
toast.success(`默认知识库体验已切换为 ${value}`);
},
});
const assistant = location.pathname.includes("/knowledge/assistant");
return (
<div className="flex items-center gap-2 border-b bg-card px-6 py-2">
<Button size="sm" variant={assistant ? "ghost" : "secondary"} onClick={() => navigate(`/page/workspace/knowledge${KNOWLEDGE_NORMAL_VIEW_SEARCH}`)}>
<BookOpen />普通知识库
</Button>
<Button size="sm" variant={assistant ? "secondary" : "ghost"} onClick={() => navigate("/page/workspace/knowledge/assistant")}>
<Database />助手知识库
</Button>
{isAdmin && (
<div className="ml-auto flex items-center gap-2 text-xs text-muted-foreground">
默认产品版本
<Select value={version.data ?? "v1"} onValueChange={(value) => save.mutate(value as KnowledgeBaseProductVersion)}>
<SelectTrigger className="h-8 w-40"><SelectValue /></SelectTrigger>
<SelectContent><SelectItem value="v1">v1 普通知识库</SelectItem><SelectItem value="v2">v2 助手知识库</SelectItem></SelectContent>
</Select>
</div>
)}
</div>
);
}
@@ -22,6 +22,10 @@ import {
} from "@/components/ui/dropdown-menu";
import { Input } from "@/components/ui/input";
import { cn } from "@/lib/utils";
import {
type AssistantKnowledgeBase,
useAssistantKnowledgeBases,
} from "@/strategy-components/api/assistant-knowledge";
import {
type LlmWikiKnowledgeBase,
useAgentLlmWikiKnowledgeBases,
@@ -66,10 +70,26 @@ function SpaceIcon({ item }: { item: LlmWikiKnowledgeBase }) {
);
}
function AssistantSpaceIcon({ item }: { item: AssistantKnowledgeBase }) {
return (
<span
className={cn(
"flex size-7 shrink-0 items-center justify-center rounded-md",
item.is_global ? "bg-amber-500/10 text-amber-600 dark:text-amber-400" : "bg-sky-500/10 text-sky-600 dark:text-sky-400",
)}
>
<Database className="size-3.5" />
</span>
);
}
function SelectionContent({
rows,
assistantRows,
value,
assistantValue,
onChange,
onAssistantChange,
search,
onSearchChange,
loading,
@@ -79,8 +99,11 @@ function SelectionContent({
readOnly,
}: {
rows: LlmWikiKnowledgeBase[];
assistantRows: AssistantKnowledgeBase[];
value: string[];
assistantValue: string[];
onChange: (value: string[]) => void;
onAssistantChange: (value: string[]) => void;
search: string;
onSearchChange: (value: string) => void;
loading: boolean;
@@ -90,6 +113,7 @@ function SelectionContent({
readOnly: boolean;
}) {
const selected = useMemo(() => new Set(value), [value]);
const selectedAssistant = useMemo(() => new Set(assistantValue), [assistantValue]);
const filtered = useMemo(() => {
const query = search.trim().toLocaleLowerCase();
if (!query) return rows;
@@ -97,11 +121,28 @@ function SelectionContent({
`${item.name} ${item.description}`.toLocaleLowerCase().includes(query),
);
}, [rows, search]);
const filteredAssistant = useMemo(() => {
const query = search.trim().toLocaleLowerCase();
if (!query) return assistantRows;
return assistantRows.filter((item) =>
`${item.name} ${item.description ?? ""}`.toLocaleLowerCase().includes(query),
);
}, [assistantRows, search]);
const groups = useMemo(() => groupRows(filtered), [filtered]);
const toggle = (id: string) => {
if (readOnly) return;
onChange(selected.has(id) ? value.filter((item) => item !== id) : [...value, id]);
};
const toggleAssistant = (id: string) => {
if (readOnly) return;
onAssistantChange(
selectedAssistant.has(id)
? assistantValue.filter((item) => item !== id)
: [...assistantValue, id],
);
};
const selectedCount = value.length + assistantValue.length;
const hasRows = groups.length > 0 || filteredAssistant.length > 0;
return (
<>
@@ -127,7 +168,7 @@ function SelectionContent({
<div className="m-2 rounded-lg border border-destructive/30 bg-destructive/5 p-3 text-xs text-destructive">
{error instanceof Error ? error.message : "知识空间读取失败"}
</div>
) : groups.length === 0 ? (
) : !hasRows ? (
<div className="flex h-28 items-center justify-center px-4 text-center text-xs text-muted-foreground">
{search
? "没有匹配的知识空间"
@@ -136,91 +177,181 @@ function SelectionContent({
: "当前账号暂无可用知识空间"}
</div>
) : (
groups.map((group, groupIndex) => (
<div key={group.key}>
{groupIndex > 0 && (
variant === "popover"
? <DropdownMenuSeparator className="my-1.5" />
: <div className="my-1.5 border-t" />
)}
{variant === "popover" ? (
<DropdownMenuLabel className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
{group.key === "personal" ? <Folder className="size-3" /> : <Globe2 className="size-3" />}{group.label}
</DropdownMenuLabel>
) : (
<div className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
{group.key === "personal" ? <Folder className="size-3" /> : <Globe2 className="size-3" />}{group.label}
</div>
)}
{group.rows.map((item) => {
const rowContent = (
<>
<SpaceIcon item={item} />
<div className="min-w-0 flex-1">
<div className="truncate text-xs font-medium">{item.name}</div>
<div className="mt-0.5 truncate text-[10px] text-muted-foreground">
{item.type === "faq" ? "FAQ" : "文档库"} · {item.knowledge_count ?? 0} 项知识
</div>
</div>
{item.publication_status === "published" && (
<Badge variant="outline" className="h-4 px-1 text-[9px]">公共</Badge>
)}
</>
);
return variant === "popover" ? (
<DropdownMenuCheckboxItem
key={item.id}
checked={selected.has(item.id)}
disabled={readOnly}
onCheckedChange={() => toggle(item.id)}
onSelect={(event) => event.preventDefault()}
className={cn(
"my-0.5 min-h-10 rounded-md py-1.5 pr-2 pl-7",
selected.has(item.id) && "bg-primary/8 focus:bg-primary/12",
)}
>
{rowContent}
</DropdownMenuCheckboxItem>
<>
{groups.map((group, groupIndex) => (
<div key={group.key}>
{groupIndex > 0 && (
variant === "popover"
? <DropdownMenuSeparator className="my-1.5" />
: <div className="my-1.5 border-t" />
)}
{variant === "popover" ? (
<DropdownMenuLabel className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
{group.key === "personal" ? <Folder className="size-3" /> : <Globe2 className="size-3" />}{group.label}
</DropdownMenuLabel>
) : (
<button
key={item.id}
type="button"
role="checkbox"
aria-checked={selected.has(item.id)}
disabled={readOnly}
onClick={() => toggle(item.id)}
className={cn(
"my-0.5 flex min-h-11 w-full items-center gap-2 rounded-md px-2 py-1.5 text-left outline-none transition-colors hover:bg-muted/60 focus-visible:ring-2 focus-visible:ring-primary",
selected.has(item.id) && "bg-primary/8",
)}
>
<span className={cn(
"flex size-4 shrink-0 items-center justify-center rounded border",
selected.has(item.id) && "border-primary bg-primary text-primary-foreground",
)}>
{selected.has(item.id) && <Check className="size-3" />}
</span>
{rowContent}
</button>
);
})}
</div>
))
<div className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
{group.key === "personal" ? <Folder className="size-3" /> : <Globe2 className="size-3" />}{group.label}
</div>
)}
{group.rows.map((item) => {
const rowContent = (
<>
<SpaceIcon item={item} />
<div className="min-w-0 flex-1">
<div className="truncate text-xs font-medium">{item.name}</div>
<div className="mt-0.5 truncate text-[10px] text-muted-foreground">
{item.type === "faq" ? "FAQ" : "文档库"} · {item.knowledge_count ?? 0} 项知识
</div>
</div>
{item.publication_status === "published" && (
<Badge variant="outline" className="h-4 px-1 text-[9px]">公共</Badge>
)}
</>
);
return variant === "popover" ? (
<DropdownMenuCheckboxItem
key={item.id}
checked={selected.has(item.id)}
disabled={readOnly}
onCheckedChange={() => toggle(item.id)}
onSelect={(event) => event.preventDefault()}
className={cn(
"my-0.5 min-h-10 rounded-md py-1.5 pr-2 pl-7",
selected.has(item.id) && "bg-primary/8 focus:bg-primary/12",
)}
>
{rowContent}
</DropdownMenuCheckboxItem>
) : (
<button
key={item.id}
type="button"
role="checkbox"
aria-checked={selected.has(item.id)}
disabled={readOnly}
onClick={() => toggle(item.id)}
className={cn(
"my-0.5 flex min-h-11 w-full items-center gap-2 rounded-md px-2 py-1.5 text-left outline-none transition-colors hover:bg-muted/60 focus-visible:ring-2 focus-visible:ring-primary",
selected.has(item.id) && "bg-primary/8",
)}
>
<span className={cn(
"flex size-4 shrink-0 items-center justify-center rounded border",
selected.has(item.id) && "border-primary bg-primary text-primary-foreground",
)}>
{selected.has(item.id) && <Check className="size-3" />}
</span>
{rowContent}
</button>
);
})}
</div>
))}
{filteredAssistant.length > 0 && (
<div>
{(groups.length > 0) && (
variant === "popover"
? <DropdownMenuSeparator className="my-1.5" />
: <div className="my-1.5 border-t" />
)}
{variant === "popover" ? (
<DropdownMenuLabel className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
<Database className="size-3" />助手知识库
</DropdownMenuLabel>
) : (
<div className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
<Database className="size-3" />助手知识库
</div>
)}
{filteredAssistant.map((item) => {
const rowContent = (
<>
<AssistantSpaceIcon item={item} />
<div className="min-w-0 flex-1">
<div className="truncate text-xs font-medium">{item.name}</div>
<div className="mt-0.5 truncate text-[10px] text-muted-foreground">
Wiki · {item.page_count ?? 0} 页 · {item.entity_count ?? 0} 实体
</div>
</div>
<Badge variant="outline" className="h-4 px-1 text-[9px]">
{item.is_global ? "总库" : "助手"}
</Badge>
</>
);
return variant === "popover" ? (
<DropdownMenuCheckboxItem
key={`assistant:${item.id}`}
checked={selectedAssistant.has(item.id)}
disabled={readOnly}
onCheckedChange={() => toggleAssistant(item.id)}
onSelect={(event) => event.preventDefault()}
className={cn(
"my-0.5 min-h-10 rounded-md py-1.5 pr-2 pl-7",
selectedAssistant.has(item.id) && "bg-primary/8 focus:bg-primary/12",
)}
>
{rowContent}
</DropdownMenuCheckboxItem>
) : (
<button
key={`assistant:${item.id}`}
type="button"
role="checkbox"
aria-checked={selectedAssistant.has(item.id)}
disabled={readOnly}
onClick={() => toggleAssistant(item.id)}
className={cn(
"my-0.5 flex min-h-11 w-full items-center gap-2 rounded-md px-2 py-1.5 text-left outline-none transition-colors hover:bg-muted/60 focus-visible:ring-2 focus-visible:ring-primary",
selectedAssistant.has(item.id) && "bg-primary/8",
)}
>
<span className={cn(
"flex size-4 shrink-0 items-center justify-center rounded border",
selectedAssistant.has(item.id) && "border-primary bg-primary text-primary-foreground",
)}>
{selectedAssistant.has(item.id) && <Check className="size-3" />}
</span>
{rowContent}
</button>
);
})}
</div>
)}
</>
)}
</div>
<div className="flex items-center justify-between gap-2 border-t bg-muted/20 px-2.5 py-2">
<span className="text-[11px] text-muted-foreground">
{readOnly ? `本对话已选 ${value.length} 个` : `已选择 ${value.length} 个`}
{readOnly ? `本对话已选 ${selectedCount} 个` : `已选择 ${selectedCount} 个`}
</span>
{readOnly ? (
<span className="text-[11px] text-muted-foreground">对话开始后不可更改</span>
) : (
<div className="flex items-center gap-1">
<Button type="button" variant="ghost" size="sm" className="h-7 px-2 text-xs" onClick={() => onChange(filtered.map((item) => item.id))}>
<Button
type="button"
variant="ghost"
size="sm"
className="h-7 px-2 text-xs"
onClick={() => {
onChange(filtered.map((item) => item.id));
onAssistantChange(filteredAssistant.map((item) => item.id));
}}
>
全选
</Button>
<Button type="button" variant="ghost" size="sm" className="h-7 px-2 text-xs" onClick={() => onChange([])}>
<Button
type="button"
variant="ghost"
size="sm"
className="h-7 px-2 text-xs"
onClick={() => {
onChange([]);
onAssistantChange([]);
}}
>
清空
</Button>
</div>
@@ -232,7 +363,9 @@ function SelectionContent({
export function LlmWikiKnowledgeSelector({
value,
assistantValue = [],
onChange,
onAssistantChange = () => undefined,
source = "all",
agentId,
autoSelectAll = false,
@@ -243,7 +376,9 @@ export function LlmWikiKnowledgeSelector({
variant = "popover",
}: {
value: string[];
assistantValue?: string[];
onChange: (value: string[]) => void;
onAssistantChange?: (value: string[]) => void;
source?: SelectorSource;
agentId?: string | null;
autoSelectAll?: boolean;
@@ -257,28 +392,39 @@ export function LlmWikiKnowledgeSelector({
const [open, setOpen] = useState(false);
const [search, setSearch] = useState("");
const runtime = useLlmWikiRuntime();
const weknoraEnabled = runtime.data?.provider === "weknora";
const all = useLlmWikiKnowledgeBases(
"all",
runtime.data?.provider === "weknora",
weknoraEnabled,
);
const agent = useAgentLlmWikiKnowledgeBases(
agentId,
runtime.data?.provider === "weknora" && source === "agent" && !!agentId,
weknoraEnabled && source === "agent" && !!agentId,
);
const rows = all.data ?? [];
const assistantBases = useAssistantKnowledgeBases();
const rows = weknoraEnabled ? (all.data ?? []) : [];
const assistantRows = assistantBases.data ?? [];
const defaultIds = useMemo(() => {
if (source !== "agent") {
return rows.map((item) => item.id);
}
return (agent.data ?? []).map((item) => item.id);
}, [agent.data, rows, source]);
const loading = all.isLoading || (source === "agent" && !!agentId && agent.isLoading);
const error = all.error ?? (source === "agent" ? agent.error : null);
const loading = runtime.isLoading || (weknoraEnabled && all.isLoading) || assistantBases.isLoading || (source === "agent" && !!agentId && agent.isLoading);
const error = (weknoraEnabled ? all.error ?? (source === "agent" ? agent.error : null) : null) ?? assistantBases.error;
const availableIds = useMemo(() => new Set(rows.map((item) => item.id)), [rows]);
const assistantAvailableIds = useMemo(
() => new Set(assistantRows.map((item) => item.id)),
[assistantRows],
);
const initializationKey = `${source}:${agentId ?? "all"}:${defaultIds.join(",")}`;
const initialized = useRef<string | null>(null);
useEffect(() => {
if (!runtime.isLoading && !weknoraEnabled) {
if (value.length > 0) onChange([]);
return;
}
if (!all.data || (source === "agent" && agentId && !agent.data) || initialized.current === initializationKey) return;
initialized.current = initializationKey;
const valid = value.filter((id) => availableIds.has(id));
@@ -288,19 +434,30 @@ export function LlmWikiKnowledgeSelector({
if (next.length !== value.length || next.some((id, index) => id !== value[index])) {
onChange(next);
}
}, [agent.data, agentId, all.data, autoSelectAll, availableIds, defaultIds, initializationKey, onChange, source, value]);
}, [agent.data, agentId, all.data, autoSelectAll, availableIds, defaultIds, initializationKey, onChange, runtime.isLoading, source, value, weknoraEnabled]);
useEffect(() => {
if (!assistantBases.data) return;
const valid = assistantValue.filter((id) => assistantAvailableIds.has(id));
if (valid.length !== assistantValue.length || valid.some((id, index) => id !== assistantValue[index])) {
onAssistantChange(valid);
}
}, [assistantAvailableIds, assistantBases.data, assistantValue, onAssistantChange]);
useEffect(() => {
if (!open) setSearch("");
}, [open]);
if (runtime.isError || runtime.data?.provider !== "weknora") return null;
if ((runtime.isError || !weknoraEnabled) && !assistantBases.isLoading && assistantRows.length === 0) return null;
const content = (
<SelectionContent
rows={rows}
assistantRows={assistantRows}
value={value}
assistantValue={assistantValue}
onChange={onChange}
onAssistantChange={onAssistantChange}
search={search}
onSearchChange={setSearch}
loading={loading}
@@ -323,7 +480,7 @@ export function LlmWikiKnowledgeSelector({
新建该智能体对话时,只能从已绑定空间中选择本次检索范围。
</p>
</div>
<Badge variant="secondary" className="shrink-0">{value.length} 个</Badge>
<Badge variant="secondary" className="shrink-0">{value.length + assistantValue.length} 个</Badge>
</div>
{content}
</div>
@@ -333,10 +490,15 @@ export function LlmWikiKnowledgeSelector({
const selectedNames = value
.map((id) => rows.find((item) => item.id === id)?.name)
.filter((name): name is string => !!name);
const selectedLabel = selectedNames.length > 0
? selectedNames.join("、")
: value.length > 0
? `已选 ${value.length} 个知识空间`
const selectedAssistantNames = assistantValue
.map((id) => assistantRows.find((item) => item.id === id)?.name)
.filter((name): name is string => !!name);
const allSelectedNames = [...selectedNames, ...selectedAssistantNames];
const selectedCount = value.length + assistantValue.length;
const selectedLabel = allSelectedNames.length > 0
? allSelectedNames.join("、")
: selectedCount > 0
? `已选 ${selectedCount} 个知识空间`
: label;
return (
@@ -348,20 +510,20 @@ export function LlmWikiKnowledgeSelector({
size="sm"
className={cn(
"h-8 max-w-52 gap-1.5 rounded-full border-0 bg-transparent px-2.5 text-muted-foreground shadow-none hover:bg-muted/70 hover:text-foreground",
value.length > 0 && "bg-primary/8 text-primary hover:bg-primary/12 hover:text-primary",
selectedCount > 0 && "bg-primary/8 text-primary hover:bg-primary/12 hover:text-primary",
className,
)}
disabled={disabled || loading}
title={value.length > 0 ? selectedLabel : undefined}
title={selectedCount > 0 ? selectedLabel : undefined}
>
{loading ? <Loader2 className="size-3.5 animate-spin" /> : <Database className="size-3.5" />}
<span className="truncate text-xs font-normal">{selectedLabel}</span>
{value.length > 0 && (
{selectedCount > 0 && (
<span className="flex size-4 items-center justify-center rounded-full bg-primary text-[9px] font-semibold text-primary-foreground">
{value.length}
{selectedCount}
</span>
)}
{value.length > 0 && value.length === rows.length && <Check className="size-3" />}
{selectedCount > 0 && selectedCount === rows.length + assistantRows.length && <Check className="size-3" />}
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="start" sideOffset={8} className="w-80 overflow-hidden rounded-xl p-0 shadow-xl">
@@ -93,6 +93,7 @@ import {
useViewMode,
} from "@/core/system-settings/hooks";
import type { Skill } from "@/core/skills/type";
import type { SkillKnowledgeBinding } from "@/strategy-components/api/skill-knowledge";
import { formatDateTime } from "@/core/utils/datetime";
import { env } from "@/env";
import { desensitize } from "@/lib/desensitize";
@@ -601,6 +602,7 @@ export function SkillRow({
isAdmin,
layout = "row",
onPublisherClick,
knowledgeBindings = [],
}: {
skill: Skill;
currentUserId: string | null;
@@ -610,6 +612,8 @@ export function SkillRow({
/** Click the 发布者 line to filter the list to that publisher. Receives the
* owner_user_id + its desensitized display label. */
onPublisherClick?: (publisher: { id: string; label: string }) => void;
/** Admin list can inject its single batched binding query here (never N+1). */
knowledgeBindings?: SkillKnowledgeBinding[];
}) {
const isCard = layout === "card";
const { t } = useI18n();
@@ -858,6 +862,20 @@ export function SkillRow({
常驻
</Badge>
)}
{knowledgeBindings.slice(0, 3).map((binding) => (
<Badge
key={binding.id}
variant="outline"
className="border-indigo-500/30 text-indigo-700 dark:text-indigo-300 text-xs"
title={`${binding.target_name} · ${binding.status}${binding.remote_business_key ? ` · ${binding.remote_business_key}` : ""}`}
>
知识库:{binding.target_name || binding.target_id}
{binding.status === "stale" ? "(待更新)" : ""}
</Badge>
))}
{knowledgeBindings.length > 3 && (
<Badge variant="secondary" className="text-xs">+{knowledgeBindings.length - 3}</Badge>
)}
</div>
);
@@ -0,0 +1,261 @@
import { Database, Loader2, RefreshCw, Search, Sparkles } from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { Input } from "@/components/ui/input";
import { Progress } from "@/components/ui/progress";
import { ScrollArea } from "@/components/ui/scroll-area";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { useAllSkillsAdmin } from "@/core/skills/hooks";
import { useLlmWikiKnowledgeBases, useLlmWikiRuntime } from "@/strategy-components/api/llmwiki";
import {
type SkillKnowledgeTarget,
useSkillKnowledgeActions,
useSkillKnowledgeBindings,
useSkillKnowledgeJobs,
} from "@/strategy-components/api/skill-knowledge";
const PHASE_LABELS: Record<string, string> = {
scanning: "扫描知识文件",
parsing: "解析资料",
extracting: "抽取实体关系",
awaiting_review: "等待审核",
merging: "合并实体关系",
writing_wiki: "写入 Wiki",
activating: "激活新版本",
completed: "完成",
};
export function SkillKnowledgePanel() {
const { skills, isLoading } = useAllSkillsAdmin(true);
const liveSkills = useMemo(() => skills.filter((skill) => (skill.state ?? "active") === "active"), [skills]);
const [selectedSkills, setSelectedSkills] = useState<string[]>([]);
const [skillQuery, setSkillQuery] = useState("");
const [selectedTargetKey, setSelectedTargetKey] = useState<string | null>(null);
const [reviewMode, setReviewMode] = useState<"auto_high_confidence" | "off">("auto_high_confidence");
const runtime = useLlmWikiRuntime();
const knowledgeBases = useLlmWikiKnowledgeBases("all", runtime.data?.provider === "weknora");
const bindings = useSkillKnowledgeBindings();
const jobs = useSkillKnowledgeJobs();
const actions = useSkillKnowledgeActions();
const targets = useMemo<SkillKnowledgeTarget[]>(() => {
return (knowledgeBases.data ?? [])
.filter((kb) => {
const strategy = kb.indexing_strategy ?? {};
const capabilities = kb.capabilities ?? {};
const wikiEnabled =
kb.type === "wiki" ||
kb.wiki_index_enabled === true ||
strategy.wiki_enabled === true ||
capabilities.wiki_enabled === true ||
(kb.wiki_config && Object.keys(kb.wiki_config).length > 0);
return kb.can_write && kb.remote_status !== "missing" && !kb.is_conversation_deposit && kb.type !== "faq" && wikiEnabled;
})
.map((kb) => ({
target_type: "weknora",
target_mode: "wiki",
target_id: kb.id,
target_name: kb.name,
remote_business_key: `weknora:kb:${kb.id}`,
}));
}, [knowledgeBases.data]);
const filteredSkills = useMemo(() => {
const needle = skillQuery.trim().toLowerCase();
if (!needle) return liveSkills;
return liveSkills.filter((skill) => {
const tagText = (skill.tags ?? []).map((tag) => tag.name).join(" ");
const haystack = [
skill.name,
skill.name_zh,
skill.description,
skill.category,
tagText,
]
.filter(Boolean)
.join("\n")
.toLowerCase();
return haystack.includes(needle);
});
}, [liveSkills, skillQuery]);
const filteredSkillNames = useMemo(() => filteredSkills.map((skill) => skill.name), [filteredSkills]);
const allFilteredSelected = filteredSkillNames.length > 0 && filteredSkillNames.every((name) => selectedSkills.includes(name));
const toggleFilteredSkills = () => {
if (!filteredSkillNames.length) return;
setSelectedSkills((current) => {
if (allFilteredSelected) {
const filtered = new Set(filteredSkillNames);
return current.filter((name) => !filtered.has(name));
}
return [...current, ...filteredSkillNames.filter((name) => !current.includes(name))];
});
};
useEffect(() => {
if (selectedTargetKey && !targets.some((target) => `${target.target_type}:${target.target_id}` === selectedTargetKey)) {
setSelectedTargetKey(null);
}
}, [selectedTargetKey, targets]);
const toggle = (values: string[], value: string, setter: (next: string[]) => void) =>
setter(values.includes(value) ? values.filter((item) => item !== value) : [...values, value]);
const submit = async () => {
const selectedTargets = targets.filter((target) =>
selectedTargetKey === `${target.target_type}:${target.target_id}`,
);
if (!selectedSkills.length || !selectedTargets.length) {
toast.error("请至少选择一个技能和一个目标知识库");
return;
}
try {
const job = await actions.createJob.mutateAsync({
skill_names: selectedSkills,
targets: selectedTargets,
review_mode: reviewMode,
});
toast.success(`已创建 ${job.total_count} 个归纳项`);
setSelectedSkills([]);
} catch (error) {
toast.error(error instanceof Error ? error.message : String(error));
}
};
return (
<div className="grid size-full min-h-0 grid-cols-[minmax(320px,0.9fr)_minmax(420px,1.1fr)] gap-4 overflow-hidden p-6">
<section className="flex min-h-0 flex-col rounded-lg border bg-card">
<div className="border-b p-4">
<h2 className="font-semibold">技能到知识库管理</h2>
<p className="mt-1 text-xs text-muted-foreground">一次选择多个技能归纳到一个 WeKnora Wiki 知识库;代码与脚本不会被读取。</p>
</div>
<ScrollArea className="min-h-0 flex-1">
<div className="space-y-5 p-4">
<div>
<div className="mb-2 flex items-center justify-between">
<span className="text-sm font-medium">1. 选择技能({selectedSkills.length})</span>
<Button variant="ghost" size="sm" onClick={toggleFilteredSkills}>
{allFilteredSelected ? "清空当前结果" : "选择当前结果"}
</Button>
</div>
<div className="relative mb-2">
<Search className="absolute left-2.5 top-1/2 size-4 -translate-y-1/2 text-muted-foreground" />
<Input
value={skillQuery}
onChange={(event) => setSkillQuery(event.target.value)}
placeholder="按技能中文名、英文名、描述、分类或标签搜索"
className="pl-8"
/>
</div>
<div className="max-h-60 space-y-1 overflow-y-auto rounded-md border p-2">
{isLoading ? <Loader2 className="m-4 size-5 animate-spin" /> : filteredSkills.map((skill) => (
<label key={skill.name} className="flex cursor-pointer items-center gap-2 rounded px-2 py-1.5 hover:bg-muted">
<Checkbox checked={selectedSkills.includes(skill.name)} onCheckedChange={() => toggle(selectedSkills, skill.name, setSelectedSkills)} />
<span className="truncate text-sm">{skill.name_zh || skill.name}</span>
<span className="ml-auto truncate text-xs text-muted-foreground">{skill.name}</span>
</label>
))}
{!isLoading && !filteredSkills.length && (
<p className="p-3 text-xs text-muted-foreground">没有匹配的技能,请换个关键词。</p>
)}
</div>
<p className="mt-1 text-xs text-muted-foreground">
当前显示 {filteredSkills.length}/{liveSkills.length} 个技能。
</p>
</div>
<div>
<div className="mb-2 flex items-center justify-between">
<span className="text-sm font-medium">2. 选择目标知识库({selectedTargetKey ? 1 : 0})</span>
</div>
<div className="space-y-1 rounded-md border p-2">
{targets.map((target) => {
const key = `${target.target_type}:${target.target_id}`;
return (
<label key={key} className="flex cursor-pointer items-center gap-2 rounded px-2 py-2 hover:bg-muted">
<Checkbox
checked={selectedTargetKey === key}
onCheckedChange={() => setSelectedTargetKey((current) => (current === key ? null : key))}
/>
<Database className="size-4 text-muted-foreground" />
<span className="text-sm">{target.target_name}</span>
<Badge variant="outline" className="ml-auto text-[10px]">WeKnora Wiki</Badge>
</label>
);
})}
{(runtime.isLoading || knowledgeBases.isLoading) && <Loader2 className="m-4 size-5 animate-spin" />}
{!runtime.isLoading && !knowledgeBases.isLoading && knowledgeBases.error && (
<p className="p-3 text-xs text-destructive">
读取 WeKnora 普通知识库失败:
{knowledgeBases.error instanceof Error ? knowledgeBases.error.message : String(knowledgeBases.error)}
</p>
)}
{!runtime.isLoading && !knowledgeBases.isLoading && !knowledgeBases.error && !targets.length && (
<p className="p-3 text-xs text-muted-foreground">
{runtime.data?.provider !== "weknora"
? "当前未配置 WeKnora 普通知识库地址;技能需要先归纳到普通知识库,再导入助手知识库。"
: "当前没有可写且已开启 Wiki 的普通知识库;请确认该库不是 FAQ/会话沉淀库,并已开启 Wiki。"}
</p>
)}
</div>
</div>
<div className="flex items-center gap-3">
<span className="text-sm font-medium">审核策略</span>
<Select value={reviewMode} onValueChange={(value) => setReviewMode(value as typeof reviewMode)}>
<SelectTrigger className="w-52"><SelectValue /></SelectTrigger>
<SelectContent>
<SelectItem value="auto_high_confidence">高置信自动通过</SelectItem>
<SelectItem value="off">不阻塞写入</SelectItem>
</SelectContent>
</Select>
</div>
<Button className="w-full" disabled={actions.createJob.isPending} onClick={() => void submit()}>
{actions.createJob.isPending ? <Loader2 className="animate-spin" /> : <Sparkles />}
开始归纳({selectedSkills.length * (selectedTargetKey ? 1 : 0)} 项)
</Button>
</div>
</ScrollArea>
</section>
<section className="flex min-h-0 flex-col gap-4 overflow-hidden">
<div className="flex min-h-0 flex-1 flex-col rounded-lg border bg-card">
<div className="flex items-center justify-between border-b p-4">
<div><h2 className="font-semibold">归纳任务</h2><p className="text-xs text-muted-foreground">后台执行、失败可单项重试</p></div>
<Button size="icon" variant="ghost" onClick={() => void jobs.refetch()}><RefreshCw className="size-4" /></Button>
</div>
<ScrollArea className="min-h-0 flex-1">
<div className="space-y-3 p-4">
{(jobs.data ?? []).map((job) => {
const done = job.success_count + job.partial_failed_count + job.failed_count;
const progress = job.total_count ? Math.round((done / job.total_count) * 100) : 0;
return (
<div key={job.id} className="rounded-md border p-3">
<div className="flex items-center gap-2 text-sm"><Badge variant={job.status === "completed" ? "default" : "secondary"}>{job.status}</Badge><span>{done}/{job.total_count} 项</span><span className="ml-auto text-xs text-muted-foreground">{new Date(job.created_at).toLocaleString()}</span></div>
<Progress className="my-2" value={progress} />
{job.items?.map((item) => (
<div key={item.id} className="mt-2 flex items-center gap-2 border-t pt-2 text-xs">
<span className="max-w-36 truncate font-medium">{item.skill_name}</span>
<span className="text-muted-foreground">{PHASE_LABELS[item.phase] || item.phase}</span>
<span className="ml-auto">{item.progress}%</span>
{item.status === "failed" && <Button size="sm" variant="outline" onClick={() => actions.retry.mutate(item.id)}>重试</Button>}
</div>
))}
</div>
);
})}
{!jobs.data?.length && <p className="py-10 text-center text-sm text-muted-foreground">暂无归纳任务</p>}
</div>
</ScrollArea>
</div>
<div className="max-h-80 overflow-auto rounded-lg border bg-card p-3 text-xs">
<div className="mb-2 text-muted-foreground">当前有效绑定:{bindings.data?.length ?? 0};同名同类型实体按来源贡献合并。</div>
<div className="space-y-2">{bindings.data?.map((binding) => <div key={binding.id} className="flex flex-wrap items-center gap-2 rounded border p-2"><span className="max-w-32 truncate font-medium">{binding.skill_name}</span><span className="max-w-40 truncate text-muted-foreground">→ {binding.target_name}</span><Badge variant="outline">{binding.status}</Badge><code className="max-w-40 truncate text-[10px] text-muted-foreground">{binding.remote_business_key}</code><Button size="sm" variant="outline" className="ml-auto" disabled={actions.resync.isPending} onClick={() => actions.resync.mutate(binding.id)}>重归纳</Button><Button size="sm" variant="ghost" onClick={() => { if (window.confirm("解除关联并保留目标知识内容?")) actions.detach.mutate({ id: binding.id, deleteContributions: false }); }}>解绑并保留</Button><Button size="sm" variant="destructive" onClick={() => { if (window.confirm("解除关联并删除此技能贡献?共享知识仍会保留其他来源。")) actions.detach.mutate({ id: binding.id, deleteContributions: true }); }}>解绑并删除贡献</Button></div>)}</div>
</div>
</section>
</div>
);
}
@@ -14,7 +14,7 @@ import {
SidebarMenuItem,
} from "@/components/ui/sidebar";
import { useI18n } from "@/core/i18n/hooks";
import { BookOpen, BotIcon, BrainIcon, FileSearch, ListChecks, MessageSquarePlus, MessagesSquare, PenLine, Plus, SparklesIcon, Telescope, Users } from "lucide-react";
import { BookOpen, BotIcon, BrainIcon, FileSearch, Library, ListChecks, MessageSquarePlus, MessagesSquare, PenLine, Plus, SparklesIcon, Telescope, Users } from "lucide-react";
import { KnowledgeRagToggle } from "./knowledge-rag-toggle";
import { SidebarCollapseButton } from "./workspace-header";
@@ -65,6 +65,13 @@ export const WORKSPACE_COMPACT_NAV_ITEMS: WorkspaceCompactNavItem[] = [
path: "/page/workspace/chats",
match: (pathname) => pathname === "/page/workspace/chats",
},
{
id: "workspace-library",
label: "资料库",
icon: Library,
path: "/page/workspace/library",
match: (pathname) => pathname === "/page/workspace/library",
},
{
id: "workspace-agents",
label: "智能体",
@@ -254,6 +254,14 @@ export function WorkspaceNavMenu() {
技能管理
</DropdownMenuItem>
)}
{isAdmin && (
<DropdownMenuItem
onClick={() => navigate("/page/workspace/admin/skills?tab=knowledge")}
>
<DatabaseZapIcon />
技能到知识库管理
</DropdownMenuItem>
)}
{isAdmin && (
<DropdownMenuItem
onClick={() => navigate("/page/workspace/admin/models")}
@@ -1,4 +1,5 @@
import { apiFetch } from "../api";
import { downloadMarkdownAsDocx } from "../scheduled-tasks/markdown-docx";
import { getFileName } from "../utils/files";
import { urlOfArtifact } from "./utils";
@@ -76,6 +77,24 @@ export async function downloadArtifactFile({
window.setTimeout(() => URL.revokeObjectURL(objectUrl), 30_000);
}
export async function downloadArtifactMarkdownAsWord({
filepath,
threadId,
}: {
filepath: string;
threadId: string;
}): Promise<void> {
const url = urlOfArtifact({ filepath, threadId, download: false });
const res = await apiFetch(url);
if (!res.ok) {
throw new Error(await errorMessageFromResponse(res));
}
const fileName = getFileName(filepath);
const title = fileName.replace(/\.(md|markdown)$/i, "");
await downloadMarkdownAsDocx(await res.text(), fileName, title);
}
export async function openArtifactInNewTab({
filepath,
threadId,
+113
View File
@@ -0,0 +1,113 @@
import { apiFetch } from "@/core/api";
import { getBackendBaseURL } from "@/core/config";
export type ArtifactLibraryKind =
| "all"
| "document"
| "spreadsheet"
| "text_data"
| "archive"
| "other";
export type ArtifactLibrarySource = "all" | "general_qa" | "agent";
export interface ArtifactLibraryItem {
threadId: string;
threadTitle: string;
agentId: string | null;
agentName: string | null;
threadType: string | null;
sourceType: Exclude<ArtifactLibrarySource, "all">;
path: string;
name: string;
sizeBytes: number;
mimeType: string | null;
modifiedAt: string;
fileKind: Exclude<ArtifactLibraryKind, "all">;
}
export interface ArtifactLibraryResponse {
total: number;
offset: number;
limit: number;
items: ArtifactLibraryItem[];
}
export async function fetchArtifactLibrary({
query,
fileKind,
sourceType,
offset,
limit,
refresh = false,
signal,
}: {
query?: string;
fileKind: ArtifactLibraryKind;
sourceType: ArtifactLibrarySource;
offset: number;
limit: number;
refresh?: boolean;
signal?: AbortSignal;
}): Promise<ArtifactLibraryResponse> {
const search = new URLSearchParams({
file_kind: fileKind,
source_type: sourceType,
offset: String(offset),
limit: String(limit),
});
const normalizedQuery = query?.trim();
if (normalizedQuery) search.set("query", normalizedQuery);
if (refresh) search.set("refresh", "true");
const response = await apiFetch(
`${getBackendBaseURL()}/api/artifact-library?${search.toString()}`,
{ method: "GET", signal },
);
if (!response.ok) {
const detail = await response.text().catch(() => "");
throw new Error(
`Failed to load artifact library (${response.status})${detail ? `: ${detail}` : ""}`,
);
}
const data = (await response.json()) as {
total?: number;
offset?: number;
limit?: number;
items?: Array<{
thread_id: string;
thread_title: string;
agent_id?: string | null;
agent_name?: string | null;
thread_type?: string | null;
source_type?: Exclude<ArtifactLibrarySource, "all">;
path: string;
name: string;
size_bytes: number;
mime_type?: string | null;
modified_at: string;
file_kind: Exclude<ArtifactLibraryKind, "all">;
}>;
};
return {
total: data.total ?? 0,
offset: data.offset ?? offset,
limit: data.limit ?? limit,
items: (data.items ?? []).map((item) => ({
threadId: item.thread_id,
threadTitle: item.thread_title,
agentId: item.agent_id ?? null,
agentName: item.agent_name ?? item.agent_id ?? null,
threadType: item.thread_type ?? null,
sourceType: item.source_type ?? (item.agent_id ? "agent" : "general_qa"),
path: item.path,
name: item.name,
sizeBytes: item.size_bytes,
mimeType: item.mime_type ?? null,
modifiedAt: item.modified_at,
fileKind: item.file_kind,
})),
};
}
@@ -149,7 +149,7 @@ export function buildWikiIndexCatalogMarkdown(
buildPageUrl: WikiPageUrlBuilder = buildWeKnoraWikiPageUrl,
indexGroups: WikiIndexGroup[] = [],
): string {
if (indexGroups.length > 0) {
if (indexGroups.some((group) => group.items.length > 0)) {
const sections: string[] = [];
for (const group of indexGroups) {
if (group.items.length === 0) continue;
@@ -61,6 +61,7 @@ export const sidebarMenuItems: MenuItem[] = [
icon: HelpCircle,
children: [
{ id: "general-qa", label: "通用问答", icon: CircleHelp, path: "/page/workspace/chats/new" },
{ id: "artifact-library", label: "资料库", icon: Library, path: "/page/workspace/library" },
{
id: "multi-agent-roundtable",
label: "多智能体会商",
@@ -9,6 +9,31 @@ export interface PromptPrefixSettings {
ordinary_qa_markdown_format_enabled: boolean;
}
export type KnowledgeBaseProductVersion = "v1" | "v2";
export async function loadKnowledgeBaseProductVersion(): Promise<KnowledgeBaseProductVersion> {
const response = await apiFetch(
`${getBackendBaseURL()}/api/system-settings/knowledge-base-product-version`,
);
if (!response.ok) throw new Error(`Failed to load knowledge base product version: ${response.status}`);
return ((await response.json()) as { version: KnowledgeBaseProductVersion }).version;
}
export async function updateKnowledgeBaseProductVersion(
version: KnowledgeBaseProductVersion,
): Promise<KnowledgeBaseProductVersion> {
const response = await apiFetch(
`${getBackendBaseURL()}/api/system-settings/knowledge-base-product-version`,
{
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ version }),
},
);
if (!response.ok) throw new Error(`Failed to update knowledge base product version: ${response.status}`);
return ((await response.json()) as { version: KnowledgeBaseProductVersion }).version;
}
export interface PluginDomainAgentMappingDto {
id: string;
domain: string;
+7 -1
View File
@@ -1512,10 +1512,16 @@ export function useThreadStream({
)
? mergedContext.llmwiki_knowledge_base_ids
: [];
const selectedAssistantKnowledgeIds = Array.isArray(
mergedContext.assistant_knowledge_base_ids,
)
? mergedContext.assistant_knowledge_base_ids
: [];
const ragModeEnabled =
(referenceModeEntryEnabled && localSettings.display.ragModeEnabled) ||
isThreadRagModeLocked(threadId) ||
selectedLlmWikiIds.length > 0;
selectedLlmWikiIds.length > 0 ||
selectedAssistantKnowledgeIds.length > 0;
if (ragModeEnabled) {
lockThreadRagMode(threadId);
}
+2
View File
@@ -27,6 +27,8 @@ export interface AgentThreadContext extends Record<string, unknown> {
knowledge_rag_enabled?: boolean;
/** DeerFlow LLMWiki mapping ids selected for this conversation. */
llmwiki_knowledge_base_ids?: string[];
/** Assistant knowledge-base ids selected for this conversation. */
assistant_knowledge_base_ids?: string[];
/** Direct RAG modes handled in the frontend; these requests never enter Lead Agent. */
execution_profile?: "single_round_rag" | "multi_round_rag";
agent_id?: string;
+32 -1
View File
@@ -3,11 +3,13 @@
import {
ChevronLeftIcon,
ChevronRightIcon,
DatabaseZap,
Replace,
SearchIcon,
SparklesIcon,
} from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import { useSearchParams } from "react-router-dom";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
@@ -18,17 +20,32 @@ import {
WorkspaceHeader,
} from "@/components/workspace/workspace-container";
import { SkillRow } from "@/components/workspace/settings/skill-settings-page";
import { SkillKnowledgePanel } from "@/components/workspace/skills/SkillKnowledgePanel";
import { getStoredAuth } from "@/core/auth";
import { useAllSkillsAdmin } from "@/core/skills/hooks";
import { desensitize } from "@/lib/desensitize";
import { useSkillKnowledgeBindings } from "@/strategy-components/api/skill-knowledge";
import { SkillAddressesPanel } from "./SkillAddressesPage";
const PAGE_SIZE = 10;
export default function AdminSkillsPage() {
const [searchParams, setSearchParams] = useSearchParams();
const [search, setSearch] = useState("");
const [page, setPage] = useState(1);
const { skills, isLoading, error } = useAllSkillsAdmin(true);
const knowledgeBindings = useSkillKnowledgeBindings();
const activeTab = useMemo(() => {
const tab = searchParams.get("tab");
return tab === "addresses" || tab === "knowledge" ? tab : "skills";
}, [searchParams]);
const changeTab = (value: string) => {
const next = new URLSearchParams(searchParams);
if (value === "skills") next.delete("tab");
else next.set("tab", value);
setSearchParams(next, { replace: true });
};
const storedAuth = useMemo(() => getStoredAuth(), []);
const currentUserId = storedAuth?.user_id ?? null;
@@ -56,7 +73,8 @@ export default function AdminSkillsPage() {
<WorkspaceHeader />
<WorkspaceBody>
<Tabs
defaultValue="skills"
value={activeTab}
onValueChange={changeTab}
className="flex size-full min-h-0 flex-col gap-0"
>
<div className="flex items-center justify-between gap-4 border-b px-6 pt-4">
@@ -71,6 +89,10 @@ export default function AdminSkillsPage() {
<Replace />
技能地址管理
</TabsTrigger>
<TabsTrigger value="knowledge">
<DatabaseZap />
技能到知识库管理
</TabsTrigger>
</TabsList>
</div>
</div>
@@ -126,6 +148,9 @@ export default function AdminSkillsPage() {
skill={skill}
currentUserId={currentUserId}
isAdmin
knowledgeBindings={(knowledgeBindings.data ?? []).filter(
(binding) => binding.skill_name === skill.name,
)}
/>
))}
</>
@@ -164,6 +189,12 @@ export default function AdminSkillsPage() {
>
<SkillAddressesPanel />
</TabsContent>
<TabsContent
value="knowledge"
className="mt-0 min-h-0 flex-1 overflow-hidden"
>
<SkillKnowledgePanel />
</TabsContent>
</Tabs>
</WorkspaceBody>
</WorkspaceContainer>
+71 -12
View File
@@ -118,6 +118,26 @@ function readPendingKnowledgeSelection(agentId: string | undefined): string[] |
}
}
function readPendingAssistantKnowledgeSelection(agentId: string | undefined): string[] | null {
if (typeof window === "undefined" || !agentId) return null;
try {
const raw = sessionStorage.getItem("pendingAgentMessage");
if (!raw) return null;
const parsed = JSON.parse(raw) as {
agentId?: string;
assistantKnowledgeBaseIds?: unknown;
};
if (parsed.agentId !== agentId || !Array.isArray(parsed.assistantKnowledgeBaseIds)) {
return null;
}
return parsed.assistantKnowledgeBaseIds
.map((item) => String(item).trim())
.filter(Boolean);
} catch {
return null;
}
}
function stringifyForHiddenContext(value: unknown): string {
if (typeof value === "string") return value;
try {
@@ -334,11 +354,17 @@ export default function AgentChatPage({
const pendingKnowledgeSelection = useRef(
readPendingKnowledgeSelection(agent_id),
);
const pendingAssistantKnowledgeSelection = useRef(
readPendingAssistantKnowledgeSelection(agent_id),
);
const [selectedKnowledgeBaseIds, setSelectedKnowledgeBaseIds] = useState<string[]>(
() => pendingKnowledgeSelection.current ?? [],
);
const [selectedAssistantKnowledgeBaseIds, setSelectedAssistantKnowledgeBaseIds] = useState<string[]>(
() => pendingAssistantKnowledgeSelection.current ?? [],
);
const [knowledgeSelectionReady, setKnowledgeSelectionReady] = useState(
() => pendingKnowledgeSelection.current !== null,
() => pendingKnowledgeSelection.current !== null || pendingAssistantKnowledgeSelection.current !== null,
);
const [knowledgeSelectionLoading, setKnowledgeSelectionLoading] = useState(false);
const knowledgeSelectionThread = useRef<string | null>(null);
@@ -349,6 +375,10 @@ export default function AgentChatPage({
setSelectedKnowledgeBaseIds(ids);
setKnowledgeSelectionReady(true);
}, []);
const changeAssistantKnowledgeSelection = useCallback((ids: string[]) => {
setSelectedAssistantKnowledgeBaseIds(ids);
setKnowledgeSelectionReady(true);
}, []);
useEffect(() => {
activeThreadIdRef.current = threadId;
@@ -358,17 +388,23 @@ export default function AgentChatPage({
if (knowledgeSelectionAgent.current === agent_id) return;
knowledgeSelectionAgent.current = agent_id;
const pending = readPendingKnowledgeSelection(agent_id);
const pendingAssistant = readPendingAssistantKnowledgeSelection(agent_id);
pendingKnowledgeSelection.current = pending;
pendingAssistantKnowledgeSelection.current = pendingAssistant;
knowledgeSelectionThread.current = null;
setSelectedKnowledgeBaseIds(pending ?? []);
setKnowledgeSelectionReady(pending !== null);
setSelectedAssistantKnowledgeBaseIds(pendingAssistant ?? []);
setKnowledgeSelectionReady(pending !== null || pendingAssistant !== null);
}, [agent_id]);
useEffect(() => {
if (isNewThread) {
if (knowledgeSelectionThread.current !== "__new__") {
setSelectedKnowledgeBaseIds(pendingKnowledgeSelection.current ?? []);
setKnowledgeSelectionReady(pendingKnowledgeSelection.current !== null);
setSelectedAssistantKnowledgeBaseIds(pendingAssistantKnowledgeSelection.current ?? []);
setKnowledgeSelectionReady(
pendingKnowledgeSelection.current !== null || pendingAssistantKnowledgeSelection.current !== null,
);
}
knowledgeSelectionThread.current = "__new__";
setKnowledgeSelectionLoading(false);
@@ -383,18 +419,29 @@ export default function AgentChatPage({
if (knowledgeSelectionThread.current !== threadId) return;
const metadata = record.metadata as Record<string, unknown> | undefined;
const raw = metadata?.llmwiki_knowledge_base_ids;
const assistantRaw = metadata?.assistant_knowledge_base_ids;
const llmwikiIds = Array.isArray(raw)
? raw.map((item) => String(item).trim()).filter(Boolean)
: [];
const assistantIds = Array.isArray(assistantRaw)
? assistantRaw.map((item) => String(item).trim()).filter(Boolean)
: [];
if (Array.isArray(raw)) {
setSelectedKnowledgeBaseIds(
raw.map((item) => String(item).trim()).filter(Boolean),
);
setKnowledgeSelectionReady(true);
setSelectedKnowledgeBaseIds(llmwikiIds);
} else {
setSelectedKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
}
if (Array.isArray(assistantRaw)) {
setSelectedAssistantKnowledgeBaseIds(assistantIds);
} else {
setSelectedAssistantKnowledgeBaseIds([]);
}
setKnowledgeSelectionReady(Array.isArray(raw) || Array.isArray(assistantRaw));
})
.catch(() => {
if (knowledgeSelectionThread.current === threadId) {
setSelectedKnowledgeBaseIds([]);
setSelectedAssistantKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
}
})
@@ -417,10 +464,13 @@ export default function AgentChatPage({
() => ({
...(taskId ? taskThreadMetadata(taskContext) : {}),
...(knowledgeSelectionReady
? { llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds }
? {
llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds,
assistant_knowledge_base_ids: selectedAssistantKnowledgeBaseIds,
}
: {}),
}),
[knowledgeSelectionReady, selectedKnowledgeBaseIds, taskContext, taskId],
[knowledgeSelectionReady, selectedAssistantKnowledgeBaseIds, selectedKnowledgeBaseIds, taskContext, taskId],
);
// 任务工作区(taskBind):每次运行都带隐藏任务上下文(任务id/名称/内容),
// 新建对话的消息历史里没有开场信息时模型也能拿到任务id(见 taskBindRunContext)。
@@ -436,7 +486,10 @@ export default function AgentChatPage({
...settings.context,
agent_id,
...(knowledgeSelectionReady
? { llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds }
? {
llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds,
assistant_knowledge_base_ids: selectedAssistantKnowledgeBaseIds,
}
: {}),
...(taskBindRunContextValue ?? {}),
},
@@ -1021,9 +1074,13 @@ export default function AgentChatPage({
source="agent"
agentId={agent_id}
value={selectedKnowledgeBaseIds}
assistantValue={selectedAssistantKnowledgeBaseIds}
onChange={changeKnowledgeSelection}
onAssistantChange={changeAssistantKnowledgeSelection}
autoSelectAll={
isNewThread && pendingKnowledgeSelection.current === null
isNewThread &&
pendingKnowledgeSelection.current === null &&
pendingAssistantKnowledgeSelection.current === null
}
disabled={isDemoMode || knowledgeSelectionLoading}
label="知识空间"
@@ -1108,7 +1165,9 @@ export default function AgentChatPage({
source="agent"
agentId={agent_id}
value={selectedKnowledgeBaseIds}
assistantValue={selectedAssistantKnowledgeBaseIds}
onChange={changeKnowledgeSelection}
onAssistantChange={changeAssistantKnowledgeSelection}
disabled={isDemoMode || knowledgeSelectionLoading}
readOnly
label="知识空间"
@@ -0,0 +1,559 @@
"use client";
import { useQuery } from "@tanstack/react-query";
import {
ArrowDownWideNarrow,
Bot,
ChevronLeft,
ChevronRight,
Download,
FileText,
FileX2,
Library,
Loader2,
MessagesSquare,
MessageSquareShare,
RefreshCw,
Search,
} from "lucide-react";
import Link from "next/link";
import { useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { ScrollArea } from "@/components/ui/scroll-area";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Skeleton } from "@/components/ui/skeleton";
import {
WorkspaceBody,
WorkspaceContainer,
WorkspaceHeader,
} from "@/components/workspace/workspace-container";
import {
downloadArtifactFile,
downloadArtifactMarkdownAsWord,
} from "@/core/artifacts/download";
import {
fetchArtifactLibrary,
type ArtifactLibraryItem,
type ArtifactLibraryKind,
type ArtifactLibrarySource,
} from "@/core/artifacts/library";
import { useI18n } from "@/core/i18n/hooks";
import { bootstrapResumePath, pathOfThread } from "@/core/threads/utils";
import { formatDateTime } from "@/core/utils/datetime";
import {
getFileExtensionDisplayName,
getFileIcon,
} from "@/core/utils/files";
import { desensitize } from "@/lib/desensitize";
import { cn } from "@/lib/utils";
const PAGE_SIZE = 50;
const FILTERS: Array<{ value: ArtifactLibraryKind; label: string }> = [
{ value: "all", label: "全部" },
{ value: "document", label: "文档" },
{ value: "spreadsheet", label: "表格" },
{ value: "text_data", label: "文本与数据" },
{ value: "archive", label: "压缩包" },
{ value: "other", label: "其他" },
];
const SOURCE_FILTERS: Array<{
value: ArtifactLibrarySource;
label: string;
}> = [
{ value: "all", label: "全部来源" },
{ value: "general_qa", label: "通用问答" },
{ value: "agent", label: "智能体" },
];
type DownloadFormat = "original" | "word";
function isMarkdownArtifact(name: string): boolean {
return /\.(md|markdown)$/i.test(name);
}
function artifactDownloadKey(
item: ArtifactLibraryItem,
format: DownloadFormat,
): string {
return `${item.threadId}:${item.path}:${format}`;
}
function formatBytes(bytes: number): string {
if (!Number.isFinite(bytes) || bytes <= 0) return "0 B";
const units = ["B", "KB", "MB", "GB"];
const index = Math.min(
Math.floor(Math.log(bytes) / Math.log(1024)),
units.length - 1,
);
const value = bytes / 1024 ** index;
const digits = index === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
return `${value.toFixed(digits)} ${units[index]}`;
}
function sourceConversationPath(item: ArtifactLibraryItem): string {
const basePath =
item.threadType === "bootstrap"
? bootstrapResumePath(item.threadId)
: pathOfThread({
thread_id: item.threadId,
metadata: item.agentId ? { agent_id: item.agentId } : undefined,
});
const search = new URLSearchParams({ artifact: item.path });
return `${basePath}?${search.toString()}`;
}
function FileRow({
item,
downloading,
onDownload,
}: {
item: ArtifactLibraryItem;
downloading: DownloadFormat | null;
onDownload: (item: ArtifactLibraryItem, format: DownloadFormat) => void;
}) {
const sourcePath = sourceConversationPath(item);
const displayName = desensitize(item.name);
const sourceTitle = desensitize(item.threadTitle || "新对话");
const sourceLabel = item.sourceType === "agent" ? "智能体" : "通用问答";
const sourceName =
item.sourceType === "agent"
? desensitize(item.agentName || item.agentId || "未知智能体")
: sourceLabel;
const SourceIcon = item.sourceType === "agent" ? Bot : MessagesSquare;
const busy = downloading !== null;
return (
<tr className="group border-b transition-colors hover:bg-muted/35">
<td className="min-w-0 py-3 pr-4 pl-1">
<Link
href={sourcePath}
className="flex min-w-0 items-center gap-3 rounded-md outline-none focus-visible:ring-2 focus-visible:ring-ring"
title={`打开原对话并查看 ${displayName}`}
>
<span className="flex size-10 shrink-0 items-center justify-center rounded-lg bg-muted text-foreground/75">
{getFileIcon(item.name, "size-5")}
</span>
<span className="min-w-0">
<span className="block truncate text-sm font-medium text-foreground group-hover:text-primary">
{displayName}
</span>
<span className="mt-1 block truncate text-xs text-muted-foreground lg:hidden">
{getFileExtensionDisplayName(item.name)} · {sourceName} · 来自「
{sourceTitle}」
</span>
<span className="mt-1 hidden text-xs text-muted-foreground lg:block">
{getFileExtensionDisplayName(item.name)} 文件
</span>
</span>
</Link>
</td>
<td className="hidden min-w-0 px-3 py-3 lg:table-cell">
<div className="flex min-w-0 items-center gap-2.5">
<span className="flex size-8 shrink-0 items-center justify-center rounded-md bg-muted text-muted-foreground">
<SourceIcon className="size-4" />
</span>
<span className="min-w-0">
<span className="block truncate text-sm font-medium text-foreground">
{sourceName}
</span>
<Link
href={sourcePath}
className="block truncate text-xs text-muted-foreground underline-offset-4 hover:text-foreground hover:underline"
title={`打开原对话:${sourceTitle}`}
>
{item.sourceType === "agent" ? "智能体 · " : ""}
{sourceTitle}
</Link>
</span>
</div>
</td>
<td className="whitespace-nowrap px-3 py-3 text-sm text-muted-foreground">
{formatDateTime(item.modifiedAt)}
</td>
<td className="hidden whitespace-nowrap px-3 py-3 text-sm text-muted-foreground md:table-cell">
{formatBytes(item.sizeBytes)}
</td>
<td className="py-3 pr-1 pl-3 text-right">
<div className="flex items-center justify-end gap-1">
<Button
type="button"
size="icon"
variant="ghost"
className="size-8"
disabled={busy}
onClick={() => onDownload(item, "original")}
title={`下载原文件:${displayName}`}
aria-label={`下载原文件:${displayName}`}
>
{downloading === "original" ? (
<Loader2 className="size-4 animate-spin" />
) : (
<Download className="size-4" />
)}
</Button>
{isMarkdownArtifact(item.name) && (
<Button
type="button"
size="sm"
variant="ghost"
className="h-8 gap-1 px-2 text-xs"
disabled={busy}
onClick={() => onDownload(item, "word")}
title={`下载为 Word:${displayName}`}
aria-label={`下载为 Word:${displayName}`}
>
{downloading === "word" ? (
<Loader2 className="size-4 animate-spin" />
) : (
<FileText className="size-4" />
)}
Word
</Button>
)}
<Button
asChild
size="icon"
variant="ghost"
className="size-8"
title={`打开原对话:${sourceTitle}`}
>
<Link href={sourcePath} aria-label={`打开原对话:${sourceTitle}`}>
<MessageSquareShare className="size-4" />
</Link>
</Button>
</div>
</td>
</tr>
);
}
function LoadingRows() {
return Array.from({ length: 5 }, (_, index) => (
<tr className="border-b" key={index}>
<td className="py-3 pr-4 pl-1">
<div className="flex items-center gap-3">
<Skeleton className="size-10 shrink-0" />
<div className="min-w-0 flex-1 space-y-2">
<Skeleton className="h-4 w-2/3" />
<Skeleton className="h-3 w-1/3" />
</div>
</div>
</td>
<td className="hidden px-3 lg:table-cell"><Skeleton className="h-4 w-3/4" /></td>
<td className="px-3"><Skeleton className="h-4 w-24" /></td>
<td className="hidden px-3 md:table-cell"><Skeleton className="h-4 w-16" /></td>
<td className="pr-1 pl-3"><Skeleton className="ml-auto size-8" /></td>
</tr>
));
}
export default function ArtifactLibraryPage() {
const { t } = useI18n();
const [search, setSearch] = useState("");
const [debouncedSearch, setDebouncedSearch] = useState("");
const [fileKind, setFileKind] = useState<ArtifactLibraryKind>("all");
const [sourceType, setSourceType] = useState<ArtifactLibrarySource>("all");
const [page, setPage] = useState(1);
const [downloadingKey, setDownloadingKey] = useState<string | null>(null);
const forceRefreshRef = useRef(false);
useEffect(() => {
document.title = `资料库 - ${t.pages.appName}`;
}, [t.pages.appName]);
useEffect(() => {
const timer = window.setTimeout(() => setDebouncedSearch(search.trim()), 300);
return () => window.clearTimeout(timer);
}, [search]);
useEffect(() => {
setPage(1);
}, [debouncedSearch, fileKind, sourceType]);
const offset = (page - 1) * PAGE_SIZE;
const libraryQuery = useQuery({
queryKey: ["artifact-library", debouncedSearch, fileKind, sourceType, page],
queryFn: async ({ signal }) => {
const refresh = forceRefreshRef.current;
try {
return await fetchArtifactLibrary({
query: debouncedSearch,
fileKind,
sourceType,
offset,
limit: PAGE_SIZE,
refresh,
signal,
});
} finally {
forceRefreshRef.current = false;
}
},
retry: false,
});
const total = libraryQuery.data?.total ?? 0;
const items = libraryQuery.data?.items ?? [];
const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE));
const hasPrevious = page > 1;
const hasNext = page < totalPages;
useEffect(() => {
if (page > totalPages) setPage(totalPages);
}, [page, totalPages]);
const itemKeys = useMemo(
() =>
new Set(
items.flatMap((item) => [
artifactDownloadKey(item, "original"),
artifactDownloadKey(item, "word"),
]),
),
[items],
);
useEffect(() => {
if (downloadingKey && !itemKeys.has(downloadingKey)) setDownloadingKey(null);
}, [downloadingKey, itemKeys]);
const handleDownload = async (
item: ArtifactLibraryItem,
format: DownloadFormat,
) => {
const key = artifactDownloadKey(item, format);
if (downloadingKey) return;
setDownloadingKey(key);
try {
if (format === "word") {
await downloadArtifactMarkdownAsWord({
filepath: item.path,
threadId: item.threadId,
});
toast.success("Word 文件已生成");
} else {
await downloadArtifactFile({
filepath: item.path,
threadId: item.threadId,
});
}
} catch (error) {
console.error(error);
toast.error(error instanceof Error ? error.message : "下载失败");
} finally {
setDownloadingKey(null);
}
};
const handleRefresh = () => {
forceRefreshRef.current = true;
void libraryQuery.refetch();
};
return (
<WorkspaceContainer>
<WorkspaceHeader />
<WorkspaceBody className="items-stretch">
<div className="flex min-h-0 w-full flex-1 flex-col">
<header className="shrink-0 border-b bg-card/40 px-5 py-6 sm:px-8">
<div className="mx-auto flex w-full max-w-6xl flex-col gap-5">
<div className="flex flex-col justify-between gap-4 sm:flex-row sm:items-start">
<div>
<div className="flex items-center gap-2">
<Library className="size-6 text-primary" />
<h1 className="text-2xl font-semibold tracking-tight">资料库</h1>
</div>
<p className="mt-2 text-sm text-muted-foreground">
汇总当前账号在对话中生成的文件,不包含上传文件、图片、音频和视频。
</p>
</div>
<Button
type="button"
variant="outline"
size="sm"
onClick={handleRefresh}
disabled={libraryQuery.isFetching}
>
<RefreshCw className={cn("size-4", libraryQuery.isFetching && "animate-spin")} />
刷新
</Button>
</div>
<div className="relative max-w-xl">
<Search className="pointer-events-none absolute top-1/2 left-3 size-4 -translate-y-1/2 text-muted-foreground" />
<Input
type="search"
className="h-11 pl-9"
placeholder="搜索文件名、来源对话或智能体名称"
value={search}
onChange={(event) => setSearch(event.target.value)}
/>
</div>
<div className="flex flex-wrap items-center justify-between gap-4">
<div className="flex flex-wrap items-center gap-3">
<div className="flex flex-wrap gap-1" aria-label="文件类型筛选">
{FILTERS.map((filter) => (
<Button
key={filter.value}
type="button"
size="sm"
variant={fileKind === filter.value ? "secondary" : "ghost"}
aria-pressed={fileKind === filter.value}
onClick={() => setFileKind(filter.value)}
>
{filter.label}
</Button>
))}
</div>
<Select
value={sourceType}
onValueChange={(value) =>
setSourceType(value as ArtifactLibrarySource)
}
>
<SelectTrigger
size="sm"
className="w-32"
aria-label="筛选文件来源"
>
<SelectValue />
</SelectTrigger>
<SelectContent>
{SOURCE_FILTERS.map((filter) => (
<SelectItem key={filter.value} value={filter.value}>
{filter.label}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="flex items-center gap-2 text-xs text-muted-foreground">
<ArrowDownWideNarrow className="size-4" />
最近生成
</div>
</div>
</div>
</header>
<ScrollArea className="min-h-0 flex-1">
<main className="mx-auto w-full max-w-6xl px-5 py-5 sm:px-8">
<div className="mb-2 text-xs text-muted-foreground tabular-nums">
{libraryQuery.isLoading ? "正在统计文件…" : `共 ${total} 个文件`}
</div>
<div className="overflow-x-auto">
<table className="w-full table-fixed border-collapse">
<thead>
<tr className="border-b text-left text-xs font-normal text-muted-foreground">
<th className="w-[48%] py-3 pr-4 pl-1 font-normal lg:w-[35%]">文件名</th>
<th className="hidden w-[25%] px-3 py-3 font-normal lg:table-cell">来源</th>
<th className="w-[34%] px-3 py-3 font-normal md:w-[22%] lg:w-[16%]">生成时间</th>
<th className="hidden w-[13%] px-3 py-3 font-normal lg:w-[10%] md:table-cell">大小</th>
<th className="w-[18%] py-3 pr-1 pl-3 text-right font-normal md:w-[17%] lg:w-[14%]">
<span className="sr-only">操作</span>
</th>
</tr>
</thead>
<tbody>
{libraryQuery.isLoading ? (
<LoadingRows />
) : (
items.map((item) => {
const key = `${item.threadId}:${item.path}`;
const activeDownload = downloadingKey?.startsWith(`${key}:`)
? downloadingKey.endsWith(":word")
? "word"
: "original"
: null;
return (
<FileRow
key={key}
item={item}
downloading={activeDownload}
onDownload={(target, format) =>
void handleDownload(target, format)
}
/>
);
})
)}
</tbody>
</table>
</div>
{!libraryQuery.isLoading && libraryQuery.isError && (
<div className="flex flex-col items-center gap-3 py-16 text-center">
<FileX2 className="size-9 text-muted-foreground" />
<div>
<p className="text-sm font-medium">资料库加载失败</p>
<p className="mt-1 text-xs text-muted-foreground">
{libraryQuery.error instanceof Error
? libraryQuery.error.message
: "请稍后重试"}
</p>
</div>
<Button variant="outline" size="sm" onClick={handleRefresh}>
重新加载
</Button>
</div>
)}
{!libraryQuery.isLoading && !libraryQuery.isError && items.length === 0 && (
<div className="flex flex-col items-center gap-3 py-20 text-center">
<FileX2 className="size-9 text-muted-foreground" />
<div>
<p className="text-sm font-medium">
{debouncedSearch || fileKind !== "all" || sourceType !== "all"
? "没有找到匹配的文件"
: "还没有生成文件"}
</p>
<p className="mt-1 text-xs text-muted-foreground">
对话中生成的文档、表格和数据文件会自动出现在这里。
</p>
</div>
</div>
)}
{!libraryQuery.isLoading && !libraryQuery.isError && totalPages > 1 && (
<div className="flex items-center justify-center gap-3 py-7">
<Button
variant="outline"
size="sm"
disabled={!hasPrevious}
onClick={() => setPage((current) => Math.max(1, current - 1))}
>
<ChevronLeft className="size-4" />
上一页
</Button>
<span className="text-sm text-muted-foreground tabular-nums">
{page} / {totalPages}
</span>
<Button
variant="outline"
size="sm"
disabled={!hasNext}
onClick={() => setPage((current) => current + 1)}
>
下一页
<ChevronRight className="size-4" />
</Button>
</div>
)}
</main>
</ScrollArea>
</div>
</WorkspaceBody>
</WorkspaceContainer>
);
}
@@ -0,0 +1,448 @@
import { BookOpen, Database, Download, Loader2, Plus, RefreshCw, Search, ShieldCheck, Trash2, Upload } from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { Input } from "@/components/ui/input";
import { ScrollArea } from "@/components/ui/scroll-area";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Textarea } from "@/components/ui/textarea";
import { KnowledgeProductNav } from "@/components/workspace/knowledge-product-nav";
import { MarkdownContent } from "@/components/workspace/messages/markdown-content";
import { getStoredAuth } from "@/core/auth";
import {
type AssistantKnowledgeBase,
type AssistantWikiPage,
useAssistantImportJobs,
useAssistantKnowledgeActions,
useAssistantKnowledgeBases,
useAssistantKnowledgeSources,
useAssistantWikiPages,
} from "@/strategy-components/api/assistant-knowledge";
function baseSubtitle(base: AssistantKnowledgeBase) {
if (base.is_global) return "全库沉淀总库 · 所有 WeKnora 导入数据都会同步进入这里并做实体对齐";
return base.description || "自建助手知识库 · 可上传普通知识库导出的 Wiki 向量包";
}
function EmptyInitializeCard({
isAdmin,
onInitialize,
pending,
}: {
isAdmin: boolean;
onInitialize: () => void;
pending: boolean;
}) {
return (
<div className="flex flex-1 items-center justify-center p-8">
<div className="max-w-lg rounded-xl border bg-card p-8 text-center">
<Database className="mx-auto mb-4 size-10 text-primary" />
<h1 className="text-xl font-semibold">初始化助手知识库</h1>
<p className="mt-2 text-sm text-muted-foreground">
未配置 WeKnora 时会默认展示助手知识库。首次使用需要创建“知识梳理总库”,后续从 WeKnora 导入的数据都会沉淀到这里。
</p>
{isAdmin ? (
<Button className="mt-5" disabled={pending} onClick={onInitialize}>
{pending ? <Loader2 className="mr-2 size-4 animate-spin" /> : <Database className="mr-2 size-4" />}
创建知识梳理总库
</Button>
) : (
<p className="mt-5 text-sm">请联系管理员初始化。</p>
)}
</div>
</div>
);
}
export default function AssistantKnowledgePage() {
const isAdmin = getStoredAuth()?.system_role === "admin";
const [query, setQuery] = useState("");
const [selectedBaseId, setSelectedBaseId] = useState<string | null>(null);
const [selected, setSelected] = useState<AssistantWikiPage | null>(null);
const [createOpen, setCreateOpen] = useState(false);
const [createDraft, setCreateDraft] = useState({ name: "", description: "" });
const packageInputRef = useRef<HTMLInputElement | null>(null);
const bases = useAssistantKnowledgeBases();
const actions = useAssistantKnowledgeActions();
useEffect(() => {
const values = bases.data ?? [];
if (!values.length) {
setSelectedBaseId(null);
return;
}
if (selectedBaseId && values.some((base) => base.id === selectedBaseId)) return;
setSelectedBaseId((values.find((base) => base.is_global) ?? values[0]).id);
}, [bases.data, selectedBaseId]);
useEffect(() => {
setSelected(null);
}, [selectedBaseId]);
const selectedBase = useMemo(
() => (bases.data ?? []).find((base) => base.id === selectedBaseId) ?? null,
[bases.data, selectedBaseId],
);
const pages = useAssistantWikiPages(query, selectedBaseId);
const sources = useAssistantKnowledgeSources(selectedBaseId);
const jobs = useAssistantImportJobs(selectedBaseId);
const activePage = selected ?? pages.data?.[0] ?? null;
const counts = useMemo(
() => ({
pages: pages.data?.length ?? selectedBase?.page_count ?? 0,
sources: sources.data?.length ?? selectedBase?.source_count ?? 0,
jobs: jobs.data?.length ?? 0,
}),
[jobs.data, pages.data, selectedBase?.page_count, selectedBase?.source_count, sources.data],
);
const initializeGlobal = () => {
actions.initialize.mutate("知识梳理总库", {
onSuccess: (base) => {
setSelectedBaseId(base.id);
toast.success("知识梳理总库已创建");
},
onError: (error) => toast.error(error.message),
});
};
const createBase = async () => {
const name = createDraft.name.trim();
if (!name) {
toast.error("请输入助手知识库名称");
return;
}
try {
const base = await actions.createBase.mutateAsync({
name,
description: createDraft.description.trim(),
});
setSelectedBaseId(base.id);
setCreateDraft({ name: "", description: "" });
setCreateOpen(false);
toast.success("助手知识库已创建");
} catch (error) {
toast.error(error instanceof Error ? error.message : String(error));
}
};
const uploadPackage = async (file: File | null | undefined) => {
if (!file) return;
try {
await actions.uploadWeKnoraPackage.mutateAsync({
file,
assistantBaseId: selectedBaseId,
});
toast.success("导出包已上传,后台正在异步导入");
} catch (error) {
toast.error(error instanceof Error ? error.message : String(error));
} finally {
if (packageInputRef.current) packageInputRef.current.value = "";
}
};
if (bases.isLoading) {
return (
<div className="flex size-full flex-col">
<KnowledgeProductNav />
<div className="flex flex-1 items-center justify-center">
<Loader2 className="animate-spin" />
</div>
</div>
);
}
if (!bases.data?.length) {
return (
<div className="flex size-full flex-col">
<KnowledgeProductNav />
<EmptyInitializeCard
isAdmin={isAdmin}
pending={actions.initialize.isPending}
onInitialize={initializeGlobal}
/>
</div>
);
}
return (
<div className="flex size-full min-h-0 flex-col bg-background">
<KnowledgeProductNav />
<div className="grid min-h-0 flex-1 grid-cols-[280px_1fr]">
<aside className="flex min-h-0 flex-col border-r bg-muted/20">
<div className="border-b p-4">
<div className="flex items-center justify-between">
<div>
<h2 className="text-sm font-semibold">助手知识库</h2>
<p className="mt-1 text-xs text-muted-foreground">总库置顶,自建库可上传 WeKnora 导出包</p>
</div>
{isAdmin && (
<Button size="icon" variant="outline" onClick={() => setCreateOpen(true)}>
<Plus className="size-4" />
</Button>
)}
</div>
</div>
<ScrollArea className="min-h-0 flex-1">
<div className="space-y-2 p-3">
{(bases.data ?? []).map((base) => (
<button
key={base.id}
type="button"
onClick={() => setSelectedBaseId(base.id)}
className={`w-full rounded-lg border px-3 py-3 text-left transition ${
selectedBaseId === base.id ? "border-primary bg-primary/10 text-primary" : "bg-card hover:bg-muted"
}`}
>
<div className="flex items-center gap-2">
<Database className="size-4 shrink-0" />
<span className="min-w-0 flex-1 truncate text-sm font-medium">{base.name}</span>
{base.is_global && <Badge variant="outline" className="text-[10px]">总库</Badge>}
</div>
<div className="mt-2 flex gap-3 text-[11px] text-muted-foreground">
<span>{base.page_count ?? 0} 页</span>
<span>{base.source_count ?? 0} 来源</span>
<span>{base.entity_count ?? 0} 实体</span>
</div>
</button>
))}
</div>
</ScrollArea>
</aside>
<main className="flex min-h-0 flex-col">
<header className="flex items-center gap-4 border-b px-6 py-4">
<div className="min-w-0">
<h1 className="truncate text-xl font-semibold">{selectedBase?.name}</h1>
<p className="mt-1 truncate text-xs text-muted-foreground">
{selectedBase ? baseSubtitle(selectedBase) : "请选择助手知识库"}
</p>
</div>
<Badge variant="outline">
<ShieldCheck className="mr-1 size-3" />
实体对齐
</Badge>
{isAdmin && (
<div className="ml-auto flex items-center gap-2">
<input
ref={packageInputRef}
type="file"
accept=".jsonl,.ndjson,application/x-ndjson,application/json"
className="hidden"
onChange={(event) => void uploadPackage(event.target.files?.[0])}
/>
<Button
variant="outline"
disabled={actions.uploadWeKnoraPackage.isPending || !selectedBaseId}
onClick={() => packageInputRef.current?.click()}
>
{actions.uploadWeKnoraPackage.isPending ? (
<Loader2 className="mr-2 size-4 animate-spin" />
) : (
<Upload className="mr-2 size-4" />
)}
上传 WeKnora 导出包
</Button>
</div>
)}
</header>
<Tabs defaultValue="wiki" className="flex min-h-0 flex-1 flex-col">
<div className="flex items-center gap-4 border-b px-6">
<TabsList className="my-2">
<TabsTrigger value="wiki">Wiki({counts.pages})</TabsTrigger>
<TabsTrigger value="sources">来源({counts.sources})</TabsTrigger>
<TabsTrigger value="jobs">导入记录({counts.jobs})</TabsTrigger>
</TabsList>
<div className="relative ml-auto w-72">
<Search className="absolute left-2.5 top-1/2 size-4 -translate-y-1/2 text-muted-foreground" />
<Input
value={query}
onChange={(event) => setQuery(event.target.value)}
placeholder="搜索标题、实体别名和正文"
className="pl-8"
/>
</div>
</div>
<TabsContent value="wiki" className="mt-0 grid min-h-0 flex-1 grid-cols-[300px_1fr]">
<ScrollArea className="border-r">
<div className="space-y-1 p-3">
{pages.isLoading && <Loader2 className="m-4 size-5 animate-spin" />}
{pages.data?.map((page) => (
<button
key={page.id}
type="button"
onClick={() => setSelected(page)}
className={`w-full rounded-md px-3 py-2 text-left ${
activePage?.id === page.id ? "bg-primary/10 text-primary" : "hover:bg-muted"
}`}
>
<div className="truncate text-sm font-medium">{page.title}</div>
<div className="mt-1 flex items-center gap-2 text-[11px] text-muted-foreground">
<span>{page.page_type}</span>
<span>{new Date(page.updated_at).toLocaleString()}</span>
</div>
</button>
))}
{!pages.isLoading && !pages.data?.length && (
<div className="p-4 text-xs text-muted-foreground">
暂无 Wiki 页面。请先到普通知识库详情页下载向量化数据包,再在当前助手知识库上传。
</div>
)}
</div>
</ScrollArea>
<ScrollArea>
<article className="mx-auto max-w-5xl p-8">
{activePage ? (
<>
<h1 className="mb-1 flex items-center gap-2 text-2xl font-semibold">
<BookOpen className="size-5 text-primary" />
{activePage.title}
</h1>
<p className="mb-6 text-xs text-muted-foreground">{activePage.slug}</p>
<MarkdownContent content={activePage.content_markdown} isLoading={false} rehypePlugins={[]} />
</>
) : (
<p className="text-sm text-muted-foreground">暂无 Wiki 页面。</p>
)}
</article>
</ScrollArea>
</TabsContent>
<TabsContent value="sources" className="mt-0 min-h-0 flex-1 overflow-auto p-6">
<div className="grid gap-3 md:grid-cols-2 xl:grid-cols-3">
{sources.data?.map((source) => (
<div key={source.id} className="rounded-lg border bg-card p-4">
<div className="flex items-center gap-2">
<Database className="size-4" />
<span className="truncate font-medium">{source.source_name}</span>
<Badge variant="outline" className="ml-auto">{source.source_type}</Badge>
</div>
<p className="mt-3 text-xs text-muted-foreground">业务键:{source.source_key}</p>
{isAdmin && (
<div className="mt-4 flex justify-end gap-2">
<Button
size="sm"
variant="outline"
disabled={source.source_type !== "weknora" || actions.reimportSource.isPending}
onClick={() =>
actions.reimportSource.mutate(source.id, {
onSuccess: () => toast.success("已发起重新导入"),
onError: (error) => toast.error(error.message),
})
}
>
<RefreshCw className="mr-2 size-4" />
重新导入
</Button>
<Button
size="sm"
variant="destructive"
disabled={actions.deleteSource.isPending}
onClick={() => {
if (window.confirm(`确认删除来源“${source.source_name}”的贡献?仍被其他来源引用的知识会保留。`)) {
actions.deleteSource.mutate(source.id, {
onSuccess: () => toast.success("来源贡献已删除"),
onError: (error) => toast.error(error.message),
});
}
}}
>
<Trash2 className="mr-2 size-4" />
删除来源
</Button>
</div>
)}
</div>
))}
{!sources.data?.length && <p className="text-sm text-muted-foreground">暂无来源。</p>}
</div>
</TabsContent>
<TabsContent value="jobs" className="mt-0 min-h-0 flex-1 overflow-auto p-6">
<div className="space-y-3">
{jobs.data?.map((job) => (
<div key={job.id} className="rounded-lg border bg-card p-4">
<div className="flex items-center gap-3">
<Badge>{job.status}</Badge>
<span className="text-sm">{job.trigger}</span>
<span className="text-xs text-muted-foreground">阶段:{job.phase}</span>
<span className="ml-auto text-xs">{new Date(job.created_at).toLocaleString()}</span>
{isAdmin && job.package_download_url && (
<Button asChild size="sm" variant="outline">
<a href={job.package_download_url} target="_blank" rel="noreferrer">
<Download className="mr-2 size-4" />
下载导出包
</a>
</Button>
)}
</div>
<div className="mt-2 text-xs text-muted-foreground">
{Object.entries(job.counts || {}).map(([key, value]) => `${key} ${value}`).join(" · ") || "暂无统计"}
</div>
{job.last_error_message && (
<div className="mt-2 rounded-md bg-destructive/10 px-3 py-2 text-xs text-destructive">
{job.last_error_message}
</div>
)}
</div>
))}
{!jobs.data?.length && <p className="text-sm text-muted-foreground">暂无导入记录。</p>}
</div>
</TabsContent>
</Tabs>
</main>
</div>
<Dialog open={createOpen} onOpenChange={setCreateOpen}>
<DialogContent>
<DialogHeader>
<DialogTitle>新建助手知识库</DialogTitle>
<DialogDescription>
自建助手知识库用于承接从 WeKnora 普通知识库下载的 Wiki 向量包;导入时同一份数据也会沉淀到“知识梳理总库”。
</DialogDescription>
</DialogHeader>
<div className="space-y-4">
<div className="space-y-1.5">
<label className="text-sm font-medium">名称</label>
<Input
value={createDraft.name}
onChange={(event) => setCreateDraft((prev) => ({ ...prev, name: event.target.value }))}
placeholder="例如:政策资料库"
/>
</div>
<div className="space-y-1.5">
<label className="text-sm font-medium">描述</label>
<Textarea
value={createDraft.description}
onChange={(event) => setCreateDraft((prev) => ({ ...prev, description: event.target.value }))}
placeholder="可选,用于说明这个助手知识库的范围"
rows={3}
/>
</div>
</div>
<DialogFooter>
<Button variant="outline" onClick={() => setCreateOpen(false)}>
取消
</Button>
<Button disabled={actions.createBase.isPending} onClick={() => void createBase()}>
{actions.createBase.isPending ? <Loader2 className="mr-2 size-4 animate-spin" /> : <Plus className="mr-2 size-4" />}
创建
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
</div>
);
}
+44 -9
View File
@@ -142,6 +142,7 @@ export default function ChatPage() {
// per-thread header toggle so stale localStorage can't re-enable it.
const [knowledgeRagEnabled] = useKnowledgeRagEnabled();
const [selectedKnowledgeBaseIds, setSelectedKnowledgeBaseIds] = useState<string[]>([]);
const [selectedAssistantKnowledgeBaseIds, setSelectedAssistantKnowledgeBaseIds] = useState<string[]>([]);
const [knowledgeSelectionReady, setKnowledgeSelectionReady] = useState(false);
const [knowledgeSelectionLoading, setKnowledgeSelectionLoading] = useState(false);
const knowledgeSelectionThread = useRef<string | null>(null);
@@ -150,18 +151,26 @@ export default function ChatPage() {
setSelectedKnowledgeBaseIds(ids);
setKnowledgeSelectionReady(true);
}, []);
const changeAssistantKnowledgeSelection = useCallback((ids: string[]) => {
setSelectedAssistantKnowledgeBaseIds(ids);
setKnowledgeSelectionReady(true);
}, []);
const streamContext = useMemo(
() => ({
...settings.context,
knowledge_rag_enabled: knowledgeRagEnabled,
...(knowledgeSelectionReady
? { llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds }
? {
llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds,
assistant_knowledge_base_ids: selectedAssistantKnowledgeBaseIds,
}
: {}),
}),
[
settings.context,
knowledgeRagEnabled,
knowledgeSelectionReady,
selectedAssistantKnowledgeBaseIds,
selectedKnowledgeBaseIds,
],
);
@@ -200,6 +209,7 @@ export default function ChatPage() {
setHasStartedConversation(false);
setPromptPrefixOn(false);
setSelectedKnowledgeBaseIds([]);
setSelectedAssistantKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
setKnowledgeSelectionLoading(false);
knowledgeSelectionThread.current = "__new__";
@@ -225,18 +235,29 @@ export default function ChatPage() {
if (knowledgeSelectionThread.current !== threadId) return;
const metadata = record.metadata as Record<string, unknown> | undefined;
const raw = metadata?.llmwiki_knowledge_base_ids;
const assistantRaw = metadata?.assistant_knowledge_base_ids;
const llmwikiIds = Array.isArray(raw)
? raw.map((item) => String(item).trim()).filter(Boolean)
: [];
const assistantIds = Array.isArray(assistantRaw)
? assistantRaw.map((item) => String(item).trim()).filter(Boolean)
: [];
if (Array.isArray(raw)) {
setSelectedKnowledgeBaseIds(
raw.map((item) => String(item).trim()).filter(Boolean),
);
setKnowledgeSelectionReady(true);
setSelectedKnowledgeBaseIds(llmwikiIds);
} else {
setSelectedKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
}
if (Array.isArray(assistantRaw)) {
setSelectedAssistantKnowledgeBaseIds(assistantIds);
} else {
setSelectedAssistantKnowledgeBaseIds([]);
}
setKnowledgeSelectionReady(Array.isArray(raw) || Array.isArray(assistantRaw));
})
.catch(() => {
if (knowledgeSelectionThread.current === threadId) {
setSelectedKnowledgeBaseIds([]);
setSelectedAssistantKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
}
})
@@ -259,6 +280,7 @@ export default function ChatPage() {
if (!known.some((a) => a.id === selectedAgentId)) {
setSelectedAgentId(null);
setSelectedKnowledgeBaseIds([]);
setSelectedAssistantKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
}
}, [selectedAgentId, selectorAgents, mineAgents, squareAgents, selectorLoading]);
@@ -269,7 +291,10 @@ export default function ChatPage() {
threadId: isNewThread ? undefined : threadId,
context: streamContext,
threadMetadata: knowledgeSelectionReady
? { llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds }
? {
llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds,
assistant_knowledge_base_ids: selectedAssistantKnowledgeBaseIds,
}
: undefined,
isMock,
onStart: (createdThreadId) => {
@@ -443,7 +468,10 @@ export default function ChatPage() {
metadata: {
agent_id: selectedAgentId,
...(knowledgeSelectionReady
? { llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds }
? {
llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds,
assistant_knowledge_base_ids: selectedAssistantKnowledgeBaseIds,
}
: {}),
},
});
@@ -454,7 +482,10 @@ export default function ChatPage() {
threadId: created.thread_id,
text,
...(knowledgeSelectionReady
? { knowledgeBaseIds: selectedKnowledgeBaseIds }
? {
knowledgeBaseIds: selectedKnowledgeBaseIds,
assistantKnowledgeBaseIds: selectedAssistantKnowledgeBaseIds,
}
: {}),
}),
);
@@ -516,6 +547,7 @@ export default function ChatPage() {
hideAdminPrompt,
preferencesLoading,
selectedAgentId,
selectedAssistantKnowledgeBaseIds,
selectedKnowledgeBaseIds,
knowledgeSelectionReady,
agentHandoffPending,
@@ -619,6 +651,7 @@ export default function ChatPage() {
const handleAgentChange = useCallback((agentId: string | null) => {
setSelectedAgentId(agentId);
setSelectedKnowledgeBaseIds([]);
setSelectedAssistantKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
saveLastAgentId(agentId);
}, []);
@@ -907,7 +940,9 @@ export default function ChatPage() {
source={selectedAgentId ? "agent" : "all"}
agentId={selectedAgentId}
value={selectedKnowledgeBaseIds}
assistantValue={selectedAssistantKnowledgeBaseIds}
onChange={changeKnowledgeSelection}
onAssistantChange={changeAssistantKnowledgeSelection}
autoSelectAll={isLandingState && !!selectedAgentId}
disabled={
env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" ||
@@ -3,6 +3,11 @@ import { useState } from "react";
import { Navigate, useParams, useSearchParams } from "react-router-dom";
import { Button } from "@/components/ui/button";
import {
KNOWLEDGE_NORMAL_VIEW_VALUE,
KnowledgeProductNav,
useKnowledgeBaseProductVersion,
} from "@/components/workspace/knowledge-product-nav";
import {
Dialog,
DialogContent,
@@ -70,7 +75,9 @@ export default function LlmWikiKnowledgePage({
const { noteId = "" } = useParams();
const [searchParams] = useSearchParams();
const runtime = useLlmWikiRuntime();
const productVersion = useKnowledgeBaseProductVersion();
const wikiSlug = searchParams.get("wikiSlug")?.trim() || "";
const explicitNormalView = searchParams.get("view") === KNOWLEDGE_NORMAL_VIEW_VALUE;
if (mode === "detail" && noteId && wikiSlug) {
return <Navigate to={buildWeKnoraWikiPagePath(noteId, wikiSlug)} replace />;
@@ -80,7 +87,14 @@ export default function LlmWikiKnowledgePage({
return <RuntimeLoading />;
}
if (mode === "list" && productVersion.data === "v2" && !explicitNormalView) {
return <Navigate to="/page/workspace/knowledge/assistant" replace />;
}
if (runtime.isError || runtime.data?.provider !== "weknora") {
if (mode === "list" || mode === "detail") {
return <Navigate to="/page/workspace/knowledge/assistant" replace />;
}
if (mode === "graph") {
return <KnowledgeGraphPage />;
}
@@ -95,5 +109,5 @@ export default function LlmWikiKnowledgePage({
return <WeKnoraKnowledgeDetailPage />;
}
return <WeKnoraKnowledgeWorkspace />;
return <div className="flex size-full min-h-0 flex-col"><KnowledgeProductNav /><div className="min-h-0 flex-1"><WeKnoraKnowledgeWorkspace /></div></div>;
}
@@ -30,6 +30,7 @@ import { agentRefOfThread } from '@/core/threads/utils';
import ChatsPage from './ChatsPage';
import ChatPage from './ChatPage';
import ArtifactLibraryPage from './ArtifactLibraryPage';
import IframeChatPage from './IframeChatPage';
import AgentsPage from './AgentsPage';
import AgentChatPage from './AgentChatPage';
@@ -41,6 +42,7 @@ import AdminSkillsPage from './AdminSkillsPage';
import ScheduledTaskRunDetailPage from './ScheduledTaskRunDetailPage';
import ScheduledTasksPage from './ScheduledTasksPage';
import LlmWikiKnowledgePage from './LlmWikiKnowledgePage';
import AssistantKnowledgePage from './AssistantKnowledgePage';
import WeKnoraWikiManagementPage from '@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage';
import WeKnoraWikiPageView from '@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView';
import MarkdownDocxEditorPage from './MarkdownDocxEditorPage';
@@ -95,6 +97,7 @@ function shouldShowCompactTopActions(pathname: string): boolean {
pathname === '/page/workspace/agents' ||
pathname === '/page/workspace/skills' ||
pathname === '/page/workspace/memory' ||
pathname === '/page/workspace/library' ||
pathname === '/page/workspace/knowledge' ||
pathname.startsWith('/page/workspace/knowledge/')
);
@@ -518,6 +521,20 @@ export default function WorkspaceRoutes({
path="chats/:thread_id"
element={<ChatThreadEntryRoute layoutProps={layoutProps} />}
/>
<Route
path="library"
element={
<WorkspaceLayout
{...layoutProps}
workspaceSidebarHidden={
layoutProps.workspaceSidebarHidden ||
!layoutProps.pageSidebarHidden
}
>
<ArtifactLibraryPage />
</WorkspaceLayout>
}
/>
<Route
path="chats_iframe/new"
element={<IframeChatEntryRoute layoutProps={layoutProps} hasThreadParam={false} />}
@@ -753,6 +770,14 @@ export default function WorkspaceRoutes({
</WorkspaceLayout>
}
/>
<Route
path="knowledge/assistant"
element={
<WorkspaceLayout {...layoutProps}>
<AssistantKnowledgePage />
</WorkspaceLayout>
}
/>
<Route
path="knowledge/:noteId/wiki/view"
element={
@@ -0,0 +1,324 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { apiFetch } from "@/core/api/fetch-client";
import { getBackendBaseURL } from "@/core/config";
const BASE = () => `${getBackendBaseURL()}/api/assistant-knowledge`;
export interface AssistantKnowledgeBase {
id: string;
name: string;
description?: string;
base_kind?: "global" | "custom";
is_global?: boolean;
status: string;
source_count?: number;
page_count?: number;
entity_count?: number;
relation_count?: number;
created_at: string;
updated_at: string;
}
export interface AssistantWikiPage {
id: string;
base_id: string;
slug: string;
title: string;
page_type: string;
content_markdown: string;
summary: string;
current_revision_id?: string | null;
updated_at: string;
}
export interface AssistantWikiRevision {
id: string;
page_id: string;
revision_no: number;
summary: string;
content_markdown: string;
change_type: string;
created_by?: string | null;
created_at: string;
}
export interface AssistantKnowledgeSource {
id: string;
base_id: string;
source_type: "weknora" | "skill" | "local_file";
source_name: string;
source_key: string;
status: string;
updated_at: string;
}
export interface AssistantImportJob {
id: string;
base_id: string;
source_id: string;
mirrored_global_base_id?: string;
mirrored_global_job_id?: string;
package_download_url?: string;
trigger: string;
status: string;
phase: string;
counts: Record<string, number>;
last_error_message?: string | null;
created_at: string;
completed_at?: string | null;
}
export interface LlmWikiExportPackageJob {
id: string;
mapping_id: string;
status: string;
phase: string;
counts: Record<string, number>;
package_download_url?: string;
status_url?: string;
error?: string | null;
created_at?: string | null;
updated_at?: string | null;
completed_at?: string | null;
file_exists?: boolean;
file_size_bytes?: number;
}
async function readJson<T>(response: Response): Promise<T> {
if (!response.ok) {
const data = await response.json().catch(() => null);
const detail = data && typeof data === "object" && "detail" in data
? (data as { detail: unknown }).detail
: `请求失败 (${response.status})`;
throw new Error(typeof detail === "string" ? detail : JSON.stringify(detail));
}
return (await response.json()) as T;
}
function normalizeImportJob(job: AssistantImportJob): AssistantImportJob {
const url = normalizeBackendUrl(job.package_download_url);
return url ? { ...job, package_download_url: url } : job;
}
function normalizeBackendUrl(url: string | undefined | null): string | undefined {
if (!url || /^https?:\/\//i.test(url)) return url || undefined;
return `${getBackendBaseURL()}${url.startsWith("/") ? url : `/${url}`}`;
}
function normalizeExportPackageJob(job: LlmWikiExportPackageJob): LlmWikiExportPackageJob {
return {
...job,
package_download_url: normalizeBackendUrl(job.package_download_url),
status_url: normalizeBackendUrl(job.status_url),
};
}
export async function getAssistantKnowledgeBase() {
const data = await readJson<{ base: AssistantKnowledgeBase | null }>(await apiFetch(`${BASE()}/base`));
return data.base;
}
export async function getAssistantKnowledgeBaseById(baseId: string) {
const data = await readJson<{ base: AssistantKnowledgeBase }>(
await apiFetch(`${BASE()}/bases/${encodeURIComponent(baseId)}`),
);
return data.base;
}
export async function listAssistantKnowledgeBases() {
return (await readJson<{ bases: AssistantKnowledgeBase[] }>(await apiFetch(`${BASE()}/bases`))).bases;
}
export async function initializeAssistantKnowledgeBase(name = "知识梳理总库") {
const data = await readJson<{ base: AssistantKnowledgeBase }>(
await apiFetch(`${BASE()}/base/initialize`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ name }),
}),
);
return data.base;
}
export async function createAssistantKnowledgeBase(input: { name: string; description?: string }) {
const data = await readJson<{ base: AssistantKnowledgeBase }>(
await apiFetch(`${BASE()}/bases`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
return data.base;
}
function withBaseParam(baseId?: string | null) {
const params = new URLSearchParams();
if (baseId) params.set("base_id", baseId);
return params;
}
export async function listAssistantWikiPages(q = "", baseId?: string | null) {
const params = withBaseParam(baseId);
if (q.trim()) params.set("q", q.trim());
return (await readJson<{ pages: AssistantWikiPage[] }>(await apiFetch(`${BASE()}/wiki-pages?${params}`))).pages;
}
export async function listAssistantKnowledgeSources(baseId?: string | null) {
const params = withBaseParam(baseId);
return (await readJson<{ sources: AssistantKnowledgeSource[] }>(await apiFetch(`${BASE()}/sources?${params}`))).sources;
}
export async function listAssistantImportJobs(baseId?: string | null) {
const params = withBaseParam(baseId);
return (
await readJson<{ jobs: AssistantImportJob[] }>(await apiFetch(`${BASE()}/import-jobs?${params}`))
).jobs.map(normalizeImportJob);
}
export async function listAssistantWikiRevisions(slug: string, baseId?: string | null) {
const params = withBaseParam(baseId);
return (await readJson<{ revisions: AssistantWikiRevision[] }>(
await apiFetch(`${BASE()}/wiki-pages/${encodeURIComponent(slug)}/revisions?${params}`),
)).revisions;
}
export async function rollbackAssistantWikiRevision(slug: string, revisionId: string, baseId?: string | null) {
const params = withBaseParam(baseId);
return readJson<AssistantWikiRevision>(
await apiFetch(
`${BASE()}/wiki-pages/${encodeURIComponent(slug)}/revisions/${encodeURIComponent(revisionId)}/rollback?${params}`,
{ method: "POST" },
),
);
}
export async function exportLlmWikiToAssistant(mappingId: string, assistantBaseId?: string | null) {
return normalizeImportJob(await readJson<AssistantImportJob>(
await apiFetch(
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(mappingId)}/export-to-assistant`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ assistant_base_id: assistantBaseId || null }),
},
),
));
}
export async function createLlmWikiExportPackage(mappingId: string) {
return normalizeExportPackageJob(
await readJson<LlmWikiExportPackageJob>(
await apiFetch(
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(mappingId)}/export-package`,
{ method: "POST" },
),
),
);
}
export async function listLlmWikiExportPackages(mappingId: string) {
return (
await readJson<{ jobs: LlmWikiExportPackageJob[] }>(
await apiFetch(
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(mappingId)}/export-package`,
),
)
).jobs.map(normalizeExportPackageJob);
}
export async function getLlmWikiExportPackageStatus(mappingId: string, jobId: string) {
return normalizeExportPackageJob(
await readJson<LlmWikiExportPackageJob>(
await apiFetch(
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(mappingId)}/export-package/${encodeURIComponent(jobId)}`,
),
),
);
}
export async function reimportAssistantKnowledgeSource(sourceId: string) {
return normalizeImportJob(await readJson<AssistantImportJob>(
await apiFetch(`${BASE()}/sources/${encodeURIComponent(sourceId)}/reimport`, {
method: "POST",
}),
));
}
export function getAssistantImportPackageUrl(jobId: string) {
return `${BASE()}/import-jobs/${encodeURIComponent(jobId)}/package`;
}
export async function uploadAssistantWeKnoraPackage(input: { file: File; assistantBaseId?: string | null }) {
const body = new FormData();
body.set("file", input.file);
if (input.assistantBaseId) body.set("assistant_base_id", input.assistantBaseId);
return normalizeImportJob(await readJson<AssistantImportJob>(
await apiFetch(`${BASE()}/imports/weknora-package`, {
method: "POST",
body,
}),
));
}
export async function deleteAssistantKnowledgeSource(sourceId: string) {
const response = await apiFetch(`${BASE()}/sources/${encodeURIComponent(sourceId)}`, {
method: "DELETE",
});
if (!response.ok) await readJson(response);
}
const KEY = "assistant-knowledge";
export function useAssistantKnowledgeBase(baseId?: string | null) {
return useQuery({
queryKey: [KEY, "base", baseId ?? "global"],
queryFn: () => (baseId ? getAssistantKnowledgeBaseById(baseId) : getAssistantKnowledgeBase()),
});
}
export function useAssistantKnowledgeBases() {
return useQuery({ queryKey: [KEY, "bases"], queryFn: listAssistantKnowledgeBases });
}
export function useAssistantWikiPages(query = "", baseId?: string | null) {
return useQuery({
queryKey: [KEY, "pages", baseId ?? "global", query],
queryFn: () => listAssistantWikiPages(query, baseId),
});
}
export function useAssistantKnowledgeSources(baseId?: string | null) {
return useQuery({
queryKey: [KEY, "sources", baseId ?? "global"],
queryFn: () => listAssistantKnowledgeSources(baseId),
});
}
export function useAssistantImportJobs(baseId?: string | null) {
return useQuery({
queryKey: [KEY, "jobs", baseId ?? "global"],
queryFn: () => listAssistantImportJobs(baseId),
refetchInterval: (query) =>
(query.state.data ?? []).some((job) => ["queued", "running"].includes(job.status))
? 3000
: false,
});
}
export function useAssistantKnowledgeActions() {
const client = useQueryClient();
const invalidate = () => client.invalidateQueries({ queryKey: [KEY] });
return {
initialize: useMutation({ mutationFn: initializeAssistantKnowledgeBase, onSuccess: invalidate }),
createBase: useMutation({ mutationFn: createAssistantKnowledgeBase, onSuccess: invalidate }),
uploadWeKnoraPackage: useMutation({ mutationFn: uploadAssistantWeKnoraPackage, onSuccess: invalidate }),
reimportSource: useMutation({ mutationFn: reimportAssistantKnowledgeSource, onSuccess: invalidate }),
deleteSource: useMutation({ mutationFn: deleteAssistantKnowledgeSource, onSuccess: invalidate }),
rollback: useMutation({
mutationFn: ({ slug, revisionId, baseId }: { slug: string; revisionId: string; baseId?: string | null }) =>
rollbackAssistantWikiRevision(slug, revisionId, baseId),
onSuccess: invalidate,
}),
};
}
@@ -29,7 +29,7 @@ export interface LlmWikiKnowledgeBase {
remote_status: "available" | "missing";
name: string;
description: string;
type: "document" | "faq";
type: "document" | "faq" | "wiki";
publication_status: "private" | "pending" | "published" | "rejected";
owner_user_id?: string | null;
is_owner: boolean;
@@ -37,6 +37,12 @@ export interface LlmWikiKnowledgeBase {
knowledge_count: number;
chunk_count: number;
processing_count: number;
share_count?: number;
is_processing?: boolean;
is_temporary?: boolean;
wiki_index_enabled?: boolean;
external_search_enabled?: boolean;
external_search_updated_at?: string | null;
embedding_model_id?: string | null;
vector_store_id?: string | null;
vector_store_name?: string | null;
@@ -44,19 +50,65 @@ export interface LlmWikiKnowledgeBase {
vector_store_engine_type?: string | null;
vector_store_status?: string | null;
chunking_config?: Record<string, unknown>;
image_processing_config?: Record<string, unknown>;
indexing_strategy?: Record<string, boolean>;
capabilities?: Record<string, boolean>;
extract_config?: Record<string, unknown>;
faq_config?: Record<string, unknown>;
wiki_config?: Record<string, unknown>;
question_generation_config?: Record<string, unknown>;
auto_tag_config?: Record<string, unknown>;
vlm_config?: Record<string, unknown>;
asr_config?: Record<string, unknown>;
summary_model_id?: string | null;
storage_provider?: string | null;
created_at?: string | null;
updated_at?: string | null;
published_at?: string | null;
remote_created_at?: string | null;
remote_updated_at?: string | null;
is_conversation_deposit?: boolean;
}
export interface PublicLlmWikiKnowledgeBase extends LlmWikiKnowledgeBase {
wiki: {
status: "ready" | "empty" | "unavailable";
page_count: number;
loaded_page_count: number;
stats_complete: boolean;
page_type_counts: Record<string, number>;
page_status_counts: Record<string, number>;
index_mode: "explicit" | "generated" | "none" | "unknown";
has_explicit_index: boolean;
index_slug: string | null;
default_slug: string | null;
created_at: string | null;
updated_at: string | null;
embed_route: string;
};
local_index: {
enabled: boolean;
wiki_index_enabled: boolean;
state: LlmWikiWikiIndexStatus["state"];
fully_vectorized: boolean;
retrieval_mode: "vector" | "wiki_api";
remote_page_count: number;
local_page_count: number;
ready_page_count: number;
failed_page_count: number;
vector_count: number;
last_started_at?: string | null;
last_completed_at?: string | null;
last_success_at?: string | null;
updated_at?: string | null;
};
}
export interface PublicLlmWikiKnowledgeBaseList {
knowledge_bases: PublicLlmWikiKnowledgeBase[];
total: number;
}
export interface LlmWikiDocument {
id: string;
type?: string;
@@ -742,6 +794,13 @@ export async function getPublicLlmWikiKnowledgeBase(
);
}
export async function listPublicLlmWikiKnowledgeBases(): Promise<PublicLlmWikiKnowledgeBaseList> {
return readJson(
await apiFetch(`${PUBLIC_API()}/knowledge-bases`),
"读取公开知识库列表",
);
}
export async function listPublicLlmWikiWikiPages(
knowledgeBaseId: string,
page: number,
@@ -0,0 +1,277 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { apiFetch } from "@/core/api/fetch-client";
import { getBackendBaseURL } from "@/core/config";
const BASE = () => `${getBackendBaseURL()}/api/skill-knowledge`;
export type SkillKnowledgeTargetType = "weknora" | "assistant";
export type SkillKnowledgeTargetMode =
| "wiki"
| "document"
| "faq"
| "conversation_archive"
| "readonly_sidecar"
| "assistant";
export interface SkillKnowledgeTarget {
target_type: SkillKnowledgeTargetType;
target_mode: SkillKnowledgeTargetMode;
target_id: string;
target_name: string;
remote_business_key?: string | null;
readonly?: boolean;
}
export interface SkillKnowledgeBinding {
id: string;
skill_name: string;
target_type: SkillKnowledgeTargetType;
target_mode: SkillKnowledgeTargetMode;
target_id: string;
target_name: string;
remote_business_key?: string | null;
remote_refs?: Array<{
provider: string;
remote_kind: string;
remote_id: string;
slug?: string;
}>;
status: string;
active: boolean;
auto_sync_enabled: boolean;
synced_digest?: string | null;
last_synced_at?: string | null;
last_error_message?: string | null;
}
export interface SkillKnowledgeSyncItem {
id: string;
job_id: string;
binding_id: string;
skill_name: string;
target_type: SkillKnowledgeTargetType;
target_mode: SkillKnowledgeTargetMode;
target_id: string;
status: string;
phase: string;
progress: number;
attempts: number;
last_error_message?: string | null;
}
export interface SkillKnowledgeJob {
id: string;
status: string;
total_count: number;
success_count: number;
partial_failed_count: number;
failed_count: number;
review_mode: "required" | "auto_high_confidence" | "off";
created_at: string;
completed_at?: string | null;
items?: SkillKnowledgeSyncItem[];
}
export interface SkillKnowledgeReview {
id: string;
job_id: string;
item_id: string;
binding_id: string;
object_type: "entity" | "relation" | "wiki" | "alignment";
object_key: string;
original: Record<string, unknown>;
edited?: Record<string, unknown> | null;
status: string;
reason?: string | null;
}
export interface SkillKnowledgeAutoSyncSettings {
watch_enabled: boolean;
schedule_enabled: boolean;
schedule_cron: string;
debounce_seconds: number;
write_remote_enabled: boolean;
review_mode: "required" | "auto_high_confidence" | "off";
}
async function readJson<T>(response: Response): Promise<T> {
if (!response.ok) {
const data = await response.json().catch(() => null);
const detail =
data && typeof data === "object" && "detail" in data
? JSON.stringify((data as { detail: unknown }).detail)
: `请求失败 (${response.status})`;
throw new Error(detail.replace(/^"|"$/g, ""));
}
return (await response.json()) as T;
}
export async function listSkillKnowledgeBindings(
skillNames?: string[],
): Promise<SkillKnowledgeBinding[]> {
const params = new URLSearchParams();
for (const name of skillNames ?? []) params.append("skill_names", name);
const response = await apiFetch(`${BASE()}/bindings?${params.toString()}`);
return (await readJson<{ bindings: SkillKnowledgeBinding[] }>(response)).bindings;
}
export async function createSkillKnowledgeJob(input: {
skill_names: string[];
targets: SkillKnowledgeTarget[];
force?: boolean;
review_mode?: "required" | "auto_high_confidence" | "off";
}): Promise<SkillKnowledgeJob> {
return readJson(
await apiFetch(`${BASE()}/sync-jobs`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
}
export async function listSkillKnowledgeJobs(): Promise<SkillKnowledgeJob[]> {
const response = await apiFetch(`${BASE()}/sync-jobs`);
return (await readJson<{ jobs: SkillKnowledgeJob[] }>(response)).jobs;
}
export async function getSkillKnowledgeJob(id: string): Promise<SkillKnowledgeJob> {
return readJson(await apiFetch(`${BASE()}/sync-jobs/${encodeURIComponent(id)}`));
}
export async function listSkillKnowledgeReviews(): Promise<SkillKnowledgeReview[]> {
const response = await apiFetch(`${BASE()}/reviews?status=pending`);
return (await readJson<{ reviews: SkillKnowledgeReview[] }>(response)).reviews;
}
export async function reviewSkillKnowledge(input: {
review_ids: string[];
action: "approve" | "reject";
edits?: Record<string, Record<string, unknown>>;
reason?: string;
}) {
return readJson<{ updated: number; resumed_item_ids: string[] }>(
await apiFetch(`${BASE()}/reviews/actions`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
}
export async function retrySkillKnowledgeItem(itemId: string) {
return readJson<SkillKnowledgeSyncItem>(
await apiFetch(`${BASE()}/sync-items/${encodeURIComponent(itemId)}/retry`, {
method: "POST",
}),
);
}
export async function detachSkillKnowledgeBinding(
bindingId: string,
deleteContributions: boolean,
) {
return readJson<SkillKnowledgeBinding>(
await apiFetch(`${BASE()}/bindings/${encodeURIComponent(bindingId)}/detach`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ delete_contributions: deleteContributions }),
}),
);
}
export async function resyncSkillKnowledgeBinding(bindingId: string) {
return readJson<SkillKnowledgeJob>(
await apiFetch(`${BASE()}/bindings/${encodeURIComponent(bindingId)}/resync`, {
method: "POST",
}),
);
}
export async function setSkillKnowledgeBindingAutoSync(
bindingId: string,
enabled: boolean,
) {
return readJson<SkillKnowledgeBinding>(
await apiFetch(
`${BASE()}/bindings/${encodeURIComponent(bindingId)}/${enabled ? "enable" : "disable"}-auto-sync`,
{ method: "POST" },
),
);
}
export async function getSkillKnowledgeAutoSyncSettings() {
return readJson<SkillKnowledgeAutoSyncSettings>(
await apiFetch(`${BASE()}/auto-sync-settings`),
);
}
export async function updateSkillKnowledgeAutoSyncSettings(
input: Partial<SkillKnowledgeAutoSyncSettings>,
) {
return readJson<SkillKnowledgeAutoSyncSettings>(
await apiFetch(`${BASE()}/auto-sync-settings`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
}
const KEY = "skill-knowledge";
export function useSkillKnowledgeBindings(skillNames?: string[]) {
return useQuery({
queryKey: [KEY, "bindings", ...(skillNames ?? [])],
queryFn: () => listSkillKnowledgeBindings(skillNames),
refetchOnWindowFocus: false,
});
}
export function useSkillKnowledgeJobs() {
return useQuery({
queryKey: [KEY, "jobs"],
queryFn: listSkillKnowledgeJobs,
refetchInterval: (query) =>
(query.state.data ?? []).some((job) => ["queued", "running"].includes(job.status))
? 1500
: false,
});
}
export function useSkillKnowledgeReviews() {
return useQuery({ queryKey: [KEY, "reviews"], queryFn: listSkillKnowledgeReviews });
}
export function useSkillKnowledgeAutoSyncSettings() {
return useQuery({
queryKey: [KEY, "auto-sync-settings"],
queryFn: getSkillKnowledgeAutoSyncSettings,
});
}
export function useSkillKnowledgeActions() {
const client = useQueryClient();
const invalidate = () => client.invalidateQueries({ queryKey: [KEY] });
return {
createJob: useMutation({ mutationFn: createSkillKnowledgeJob, onSuccess: invalidate }),
review: useMutation({ mutationFn: reviewSkillKnowledge, onSuccess: invalidate }),
retry: useMutation({ mutationFn: retrySkillKnowledgeItem, onSuccess: invalidate }),
resync: useMutation({ mutationFn: resyncSkillKnowledgeBinding, onSuccess: invalidate }),
setAutoSync: useMutation({
mutationFn: ({ id, enabled }: { id: string; enabled: boolean }) =>
setSkillKnowledgeBindingAutoSync(id, enabled),
onSuccess: invalidate,
}),
updateAutoSyncSettings: useMutation({
mutationFn: updateSkillKnowledgeAutoSyncSettings,
onSuccess: invalidate,
}),
detach: useMutation({
mutationFn: ({ id, deleteContributions }: { id: string; deleteContributions: boolean }) =>
detachSkillKnowledgeBinding(id, deleteContributions),
onSuccess: invalidate,
}),
};
}
@@ -1,19 +1,16 @@
import { AlertCircle, Edit3, FileText, Loader2, RefreshCw, Trash2 } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { AlertCircle, Download, Edit3, FileText, Loader2, RefreshCw, Trash2 } from "lucide-react";
import { useCallback, useEffect, useRef, useState } from "react";
import { Navigate, useNavigate, useParams, useSearchParams } from "react-router-dom";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog";
import { Input } from "@/components/ui/input";
import { Textarea } from "@/components/ui/textarea";
import { MarkdownCodeEditor } from "@/components/wiki/MarkdownCodeEditor";
import { getBackendBaseURL } from "@/core/config";
import { getStoredAuth } from "@/core/auth";
import { buildWeKnoraWikiPagePath } from "@/core/llmwiki/source-links";
import {
createLlmWikiWeKnoraEmbedSession,
@@ -23,8 +20,20 @@ import {
useLlmWikiWikiPage,
useLlmWikiRuntime,
} from "@/strategy-components/api/llmwiki";
import {
createLlmWikiExportPackage,
listLlmWikiExportPackages,
type LlmWikiExportPackageJob,
} from "@/strategy-components/api/assistant-knowledge";
const LIST_PATH = "/page/workspace/knowledge";
const RUNNING_EXPORT_STATUSES = new Set(["queued", "running"]);
const EXPORT_STATUS_LABELS: Record<string, string> = {
queued: "排队中",
running: "整理中",
completed: "已完成",
failed: "失败",
};
type DeerFlowThemePayload = {
type: "deerflow-weknora-theme";
@@ -92,6 +101,25 @@ function buildFrameUrl(framePath: string) {
return new URL(`${backendBase || backendOrigin}/${framePath.replace(/^\/+/, "")}`);
}
function formatBytes(value: number | undefined | null) {
const bytes = Number(value || 0);
if (!Number.isFinite(bytes) || bytes <= 0) return "—";
const units = ["B", "KB", "MB", "GB", "TB"];
let current = bytes;
let unitIndex = 0;
while (current >= 1024 && unitIndex < units.length - 1) {
current /= 1024;
unitIndex += 1;
}
return `${current >= 10 || unitIndex === 0 ? current.toFixed(0) : current.toFixed(1)} ${units[unitIndex]}`;
}
function formatTime(value: string | undefined | null) {
if (!value) return "—";
const time = new Date(value);
return Number.isNaN(time.getTime()) ? "—" : time.toLocaleString();
}
export function WeKnoraKnowledgeDetailPage() {
const {
knowledge_base_id: strategyKnowledgeBaseId,
@@ -104,15 +132,40 @@ export function WeKnoraKnowledgeDetailPage() {
const [frameSrc, setFrameSrc] = useState("");
const [frameVersion, setFrameVersion] = useState(0);
const [frameError, setFrameError] = useState<string | null>(null);
const [exportPackageLoading, setExportPackageLoading] = useState(false);
const [exportJobsLoading, setExportJobsLoading] = useState(false);
const [exportJobs, setExportJobs] = useState<LlmWikiExportPackageJob[]>([]);
const isAdmin = getStoredAuth()?.system_role === "admin";
const runtime = useLlmWikiRuntime();
const details = useLlmWikiKnowledgeBase(knowledgeBaseId || null, !!knowledgeBaseId);
const refreshExportJobs = useCallback(
async (silent = false) => {
if (!knowledgeBaseId || !isAdmin) return;
if (!silent) setExportJobsLoading(true);
try {
setExportJobs(await listLlmWikiExportPackages(knowledgeBaseId));
} catch (error) {
if (!silent) toast.error(error instanceof Error ? error.message : String(error));
} finally {
if (!silent) setExportJobsLoading(false);
}
},
[isAdmin, knowledgeBaseId],
);
const detailsReady = !!details.data;
const requestedTab = searchParams.get("tab");
const detailsWikiEnabled =
details.data?.type === "wiki" ||
details.data?.wiki_index_enabled === true ||
details.data?.indexing_strategy?.wiki_enabled === true ||
details.data?.capabilities?.wiki_enabled === true ||
(details.data?.wiki_config && Object.keys(details.data.wiki_config).length > 0);
const preferredTab =
requestedTab === "documents" || requestedTab === "wiki" || requestedTab === "graph"
? requestedTab
: details.data?.indexing_strategy?.wiki_enabled
: detailsWikiEnabled
? "wiki"
: "documents";
const sourceChunkId = searchParams.get("sourceChunkId") || searchParams.get("chunkId") || "";
@@ -179,6 +232,22 @@ export function WeKnoraKnowledgeDetailPage() {
};
}, [frameSrc]);
useEffect(() => {
if (!isAdmin || !knowledgeBaseId || runtime.data?.provider !== "weknora") {
setExportJobs([]);
return;
}
void refreshExportJobs();
}, [isAdmin, knowledgeBaseId, refreshExportJobs, runtime.data?.provider]);
useEffect(() => {
if (!isAdmin || !knowledgeBaseId || !exportJobs.some((job) => RUNNING_EXPORT_STATUSES.has(job.status))) {
return;
}
const timer = window.setInterval(() => void refreshExportJobs(true), 5000);
return () => window.clearInterval(timer);
}, [exportJobs, isAdmin, knowledgeBaseId, refreshExportJobs]);
if (nativeWikiPageUrl) {
return <Navigate to={nativeWikiPageUrl} replace />;
}
@@ -232,8 +301,25 @@ export function WeKnoraKnowledgeDetailPage() {
const canWrite = details.data.can_write && !details.data.is_conversation_deposit;
const canEditCurrentWikiArticle =
canWrite && !!details.data.indexing_strategy?.wiki_enabled && !!wikiSlug;
const canManageWiki = canWrite && !!details.data.indexing_strategy?.wiki_enabled;
canWrite && !!detailsWikiEnabled && !!wikiSlug;
const canManageWiki = canWrite && !!detailsWikiEnabled;
const createVectorizedPackage = () => {
if (exportPackageLoading) return;
setExportPackageLoading(true);
void (async () => {
try {
const job = await createLlmWikiExportPackage(knowledgeBaseId);
setExportJobs((current) => [job, ...current.filter((item) => item.id !== job.id)]);
toast.success("已创建向量化数据包整理记录,完成后可在下方列表下载");
void refreshExportJobs(true);
} catch (error) {
toast.error(error instanceof Error ? error.message : String(error));
} finally {
setExportPackageLoading(false);
}
})();
};
return (
<div className="flex h-full min-h-0 flex-col bg-background">
@@ -255,6 +341,17 @@ export function WeKnoraKnowledgeDetailPage() {
)}
</div>
<div className="flex shrink-0 items-center gap-2">
{isAdmin && (
<Button
size="sm"
variant="outline"
disabled={exportPackageLoading}
onClick={createVectorizedPackage}
>
{exportPackageLoading ? <Loader2 className="mr-2 size-4 animate-spin" /> : <Download className="mr-2 size-4" />}
生成向量化数据包
</Button>
)}
<Button
size="sm"
variant="outline"
@@ -266,6 +363,13 @@ export function WeKnoraKnowledgeDetailPage() {
</div>
</div>
)}
{canManageWiki && isAdmin && (
<ExportPackageJobsPanel
jobs={exportJobs}
loading={exportJobsLoading}
onRefresh={() => void refreshExportJobs()}
/>
)}
{frameSrc && (
<iframe
key={`${frameSrc}:${frameVersion}`}
@@ -293,6 +397,84 @@ export function WeKnoraKnowledgeDetailPage() {
);
}
function ExportPackageJobsPanel({
jobs,
loading,
onRefresh,
}: {
jobs: LlmWikiExportPackageJob[];
loading: boolean;
onRefresh: () => void;
}) {
return (
<div className="shrink-0 border-b bg-muted/20 px-4 py-3">
<div className="mb-2 flex items-center justify-between">
<div>
<div className="text-sm font-medium">向量化数据包记录</div>
<p className="text-xs text-muted-foreground">
每次点击都会生成一条后台整理记录;文件较大时,请等待状态变为“已完成”后再下载。
</p>
</div>
<Button size="sm" variant="ghost" onClick={onRefresh} disabled={loading}>
{loading ? <Loader2 className="mr-2 size-4 animate-spin" /> : <RefreshCw className="mr-2 size-4" />}
刷新
</Button>
</div>
<div className="max-h-44 space-y-2 overflow-y-auto pr-1">
{jobs.map((job) => {
const running = RUNNING_EXPORT_STATUSES.has(job.status);
const completed = job.status === "completed";
const canDownload = completed && !!job.package_download_url && job.file_exists !== false;
const counts = job.counts ?? {};
return (
<div key={job.id} className="flex items-center gap-3 rounded-md border bg-card px-3 py-2 text-xs">
<Badge
variant={job.status === "failed" ? "destructive" : completed ? "default" : "secondary"}
className="shrink-0"
>
{running && <Loader2 className="mr-1 inline size-3 animate-spin" />}
{EXPORT_STATUS_LABELS[job.status] || job.status}
</Badge>
<div className="min-w-0 flex-1">
<div className="flex flex-wrap items-center gap-x-3 gap-y-1">
<span className="font-medium">任务 {job.id.slice(0, 8)}</span>
<span className="text-muted-foreground">创建:{formatTime(job.created_at)}</span>
<span className="text-muted-foreground">大小:{formatBytes(job.file_size_bytes)}</span>
</div>
<div className="mt-1 flex flex-wrap gap-x-3 gap-y-1 text-muted-foreground">
<span>Wiki {counts.wiki_page ?? counts.wiki_pages ?? 0}</span>
<span>文档 {counts.document ?? counts.documents ?? 0}</span>
<span>片段 {counts.chunk ?? counts.chunks ?? 0}</span>
<span>向量 {counts.vector ?? counts.vectors ?? 0}</span>
<span>阶段:{job.phase}</span>
{job.error && <span className="text-destructive">{job.error}</span>}
</div>
</div>
{canDownload ? (
<Button asChild size="sm" variant="outline" className="shrink-0">
<a href={job.package_download_url} target="_blank" rel="noreferrer">
<Download className="mr-2 size-4" />
下载
</a>
</Button>
) : (
<Button size="sm" variant="outline" className="shrink-0" disabled>
{completed && job.file_exists === false ? "文件不存在" : "等待完成"}
</Button>
)}
</div>
);
})}
{!jobs.length && (
<div className="rounded-md border border-dashed bg-card px-3 py-6 text-center text-xs text-muted-foreground">
暂无导出记录。点击上方“生成向量化数据包”后,任务会显示在这里。
</div>
)}
</div>
</div>
);
}
function CurrentWikiArticleActions({
knowledgeBaseId,
slug,
@@ -54,6 +54,7 @@ import {
import { WikiProtectedImage } from "./WikiProtectedImage";
const LIST_PATH = "/page/workspace/knowledge";
const GENERATED_PUBLIC_INDEX_SLUG = "__index__";
function cleanText(value?: string | null) {
return (value || "").replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
@@ -144,13 +145,40 @@ export function WeKnoraWikiPageView({
!!noteId && publicEmbed,
);
const pages = publicEmbed ? publicPages : privatePages;
const pagesList = useMemo(() => pages.data?.pages || [], [pages.data?.pages]);
const indexPage = useMemo(
() => (publicEmbed ? pagesList.find(isWikiIndexPage) : undefined),
[pagesList, publicEmbed],
const remotePagesList = useMemo(() => pages.data?.pages || [], [pages.data?.pages]);
const explicitIndexPage = useMemo(
() => (publicEmbed ? remotePagesList.find(isWikiIndexPage) : undefined),
[publicEmbed, remotePagesList],
);
const generatedIndexPage = useMemo<LlmWikiWikiPage | undefined>(
() => publicEmbed && remotePagesList.length > 0 && !explicitIndexPage
? {
id: "__generated_public_index__",
slug: GENERATED_PUBLIC_INDEX_SLUG,
title: "索引",
page_type: "index",
status: "generated",
content: "# 索引\n\n知识库页面目录",
summary: "",
aliases: [],
source_refs: [],
in_links: [],
out_links: [],
version: 0,
}
: undefined,
[explicitIndexPage, publicEmbed, remotePagesList.length],
);
const indexPage = explicitIndexPage || generatedIndexPage;
const pagesList = useMemo(
() => generatedIndexPage ? [generatedIndexPage, ...remotePagesList] : remotePagesList,
[generatedIndexPage, remotePagesList],
);
const landingPage = useMemo(() => selectDefaultWikiPage(pagesList), [pagesList]);
const wikiSlug = requestedWikiSlug || (publicEmbed ? landingPage?.slug || "" : "");
const generatedIndexActive = generatedIndexPage?.slug === wikiSlug;
const generatedIndexRequested =
publicEmbed && wikiSlug === GENERATED_PUBLIC_INDEX_SLUG && !explicitIndexPage;
const privatePage = useLlmWikiWikiPage(
noteId || null,
wikiSlug || null,
@@ -159,7 +187,11 @@ export function WeKnoraWikiPageView({
const publicPage = usePublicLlmWikiWikiPage(
noteId || null,
wikiSlug || null,
!!noteId && !!wikiSlug && publicEmbed,
!!noteId &&
!!wikiSlug &&
publicEmbed &&
!generatedIndexActive &&
!generatedIndexRequested,
);
const page = publicEmbed ? publicPage : privatePage;
const publicWikiIndex = usePublicLlmWikiWikiIndex(
@@ -170,7 +202,9 @@ export function WeKnoraWikiPageView({
const [directoryQuery, setDirectoryQuery] = useState("");
const [collapsedDirectories, setCollapsedDirectories] = useState<Set<string>>(() => new Set());
const routedPage = (location.state as { wikiPage?: LlmWikiWikiPage } | null)?.wikiPage;
const pageData = page.data || (routedPage?.slug === wikiSlug ? routedPage : undefined);
const pageData = generatedIndexActive
? generatedIndexPage
: page.data || (routedPage?.slug === wikiSlug ? routedPage : undefined);
const filteredPages = useMemo(
() => filterWikiPagesWithAncestors(pagesList, directoryQuery),
[directoryQuery, pagesList],
@@ -345,6 +379,24 @@ export function WeKnoraWikiPageView({
</div>
);
}
if (publicEmbed && pages.error) {
return (
<div className="flex h-full flex-col items-center justify-center gap-4 px-6 text-center">
<div className="flex size-12 items-center justify-center rounded-full bg-destructive/10 text-destructive">
<BookOpenText className="size-6" />
</div>
<div>
<div className="font-medium text-foreground">无法读取 Wiki 目录</div>
<div className="mt-1 text-sm text-muted-foreground">
{pages.error instanceof Error ? pages.error.message : "公开 Wiki 服务暂时不可用"}
</div>
</div>
<Button onClick={() => void pages.refetch()}>
<RefreshCw className="mr-2 size-4" />重试
</Button>
</div>
);
}
return (
<div className="flex h-full flex-col items-center justify-center gap-4 text-sm text-muted-foreground">
{publicEmbed ? "该知识库暂无可展示的 Wiki 页面" : "当前链接缺少 Wiki 页面标识"}
+54 -6
View File
@@ -379,7 +379,7 @@ FastAPI application on port 8001 with health checks at `GET|HEAD /health`, `/api
| **Uploads** (`/api/threads/{id}/uploads`) | `POST /` - upload files (auto-converts PDF/PPT/Excel/Word); `GET /list` - list; `DELETE /{filename}` - delete |
| **Threads** (`/api/threads/{id}`) | `DELETE /` - remove DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
| **Threads search** (`POST /threads/search`, LangGraph-compat) | Conversation listing. **System threads are excluded by default**: rows whose `metadata.system` is truthy (scheduler runs, roundtable workers) never crowd the user's list; a caller whose `metadata` filter explicitly references `thread_type`/`system` (e.g. studio notebooks) opts out of the exclusion. New `query` field = case-insensitive title (`display_name`) substring search, used by the chats page's server-side pagination. `POST /threads/count` takes the same request body (limit/offset ignored) and returns `{total}` under identical filtering/exclusion semantics ? powers the chats page's ??/??? display without fetching rows (`ThreadMetaStore.count`). `ThreadMetaStore.search` gained `exclude_system`/`query` params; JSON-blob filters batch-scan until the requested `limit`/`offset` window is full (so pagination over a heavily-filtered set never under-fills), with a `thread_id` tie-breaker on the `updated_at DESC` ordering for stable OFFSET paging. Tests: `tests/test_thread_meta_search.py` |
| **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types |
| **Artifacts** (`/api/threads/{id}/artifacts`, `/api/artifact-library`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types. `GET /api/artifact-library` scans every current-user normal thread's authoritative `outputs/` directory (including files omitted from `present_files`), excludes uploads and image/audio/video media, joins the owning conversation title/agent route metadata, and exposes search/filter/pagination with a bounded 15-second per-user cache plus `refresh=true` bypass. Frontend route `/page/workspace/library` links each item back to the source chat with `?artifact=<path>` so the chat opens the existing artifact panel directly. |
| **Word export** (`POST /api/writing/export/docx`) | Shared Markdown → DOCX generator used by conversation, artifact, scheduled-task, Deep Research, and AI-writing download surfaces. `app/gateway/word_export.py` builds OOXML with no Python conversion dependency: A4 portrait, 3.7/3.5/2.8/2.6 cm margins, real multilevel heading numbering (`一、` / `(一)` / `1.`), two-character first-line indent on heading levels 1–3, formal body/table styles, page numbers beginning at 1 with odd pages left-aligned and even pages right-aligned, and embedded `app/gateway/assets/fonts/方正小标宋简体.ttf`. Before numbering it removes complete manual heading prefixes (including `2.1` / `2.1.1`) and drops Markdown horizontal rules from Word output. The ordinary-Q&A Lead Agent can emit the matching Markdown heading forms and avoid unordered bullet lists when the admin enables `prompt_prefix.ordinary_qa_markdown_format_enabled` in `system_settings.json`; the switch defaults off, while writing setup, writing mode, notebook, scheduled runs, and SOUL-driven custom agents keep their independent prompts. The font loader validates its internal family, SHA-256, and OpenType embedding permission (`OS/2.fsType`) before export. Tests: `tests/test_word_export.py`, `tests/test_es_query_routing_prompt.py`. |
| **Deep Research** (`/api/deep-research`) | Durable, user-owned research sessions/jobs with replayable SSE events and hidden-thread artifacts. `basic`/`quick` use the compact runner; `detailed` creates bounded subtopics, searches/writes sections concurrently, and assembles a report; `deep` recursively searches follow-up questions under a deterministic breadth/depth query budget. `multi_agent` is a LangGraph 1.x editor/researcher/writer/reviewer graph: its initial plan emits `awaiting_input`, the recorder stores the interrupt and releases the lease, and `POST /jobs/{id}/resume` rebuilds the graph from the persisted approved/revised plan. A bounded `custom_outline` is frozen into the session; headings/list entries deterministically supply the detailed and multi-agent plan, and the shared writer receives it only as a guarded content constraint. Completed reports support owner-scoped follow-up (`GET /sessions/{id}/messages`, `POST /sessions/{id}/chat`): context is only the current report, selected sources and recent messages; source-id citations are persisted, while `allow_new_research=true` is explicitly rejected. Owners may change that evidence set only after the job stops via `PATCH /sessions/{id}/sources/{source_id}`; this applies to future follow-ups rather than rewriting a completed report. Optional report illustrations are gated by `config.yaml -> deep_research.image_generation`: the Harness `OpenAICompatibleImageProvider` requires base64 output, the shared runner step writes only bounded `research-image-01..04.(png|jpg|webp)` artifacts and Markdown uses private `deep-research://` links. `GET /sessions/{id}/artifacts/{name}` checks session ownership before serving an image; an unavailable provider yields a warning but never fails the text report. Runners live in `packages/harness/deerflow/agents/deep_research/runners/`, may never import `app.*`, and use only the `AdapterBundle`. |
| **Gateway auth** | Owner-scoped Deep Research routes await the platform's asynchronous `get_current_user` resolver before they read or write persistence, ensuring each session and job operation receives a concrete user ID. Deep Research artifacts are always resolved from a whitelisted filename via `/mnt/user-data/outputs/<name>` before mapping to a host path, preserving the sandbox guard on native Windows and Linux hosts. |
@@ -1611,8 +1611,10 @@ same DOM value, so a fetched-and-revoked object URL is not acceptable.
Third-party pages embed published Wiki content through the chrome-free route
`/#/embed/knowledge/{mapping_id}` (optional exact-page `wikiSlug`). It defaults
to a recognized Wiki index page (title `索引`/`index`, page type `index`, or an
`index`/`home`/`readme` slug), promotes that page to the first public-directory
entry, then falls back to an overview/root page when no index exists. It retains
`index`/`home`/`readme` slug) and promotes that page to the first public-directory
entry. When upstream pages exist without an explicit index shell, the frontend
injects a virtual `__index__` page and builds its catalogue from the page list;
an all-empty dynamic index response must also fall back to that page list. It retains
directory navigation and supports the shared public-embed `theme` / `bg` /
`text` appearance inputs. WeKnora dynamically renders index page-type counts,
nested category headings, page summaries, and `data-slug` links even though the
@@ -1628,9 +1630,14 @@ the loaded page title, slug, or alias and rewrites them to exact embedded
`wikiSlug` routes. Preserve WeKnora's `wiki-content-link` interaction: primary
brand color, medium weight, dashed underline becoming solid on hover, focus
outline, and in-reader SPA navigation rather than a page reload. `GET
/api/public/llmwiki/knowledge-bases` is the
anonymous discovery endpoint and returns only each published ordinary mapping's
`id` and `name` (plus the collection total). All anonymous reader calls use that
/api/public/llmwiki/knowledge-bases` is the anonymous discovery endpoint. It
validates published ordinary mappings against WeKnora, drops stale mappings,
and returns safe comprehensive metadata from the WeKnora list/detail/page APIs:
document/chunk/processing/share counts, non-secret configuration and timestamps,
Wiki page type/status counts and explicit/generated index entry, plus aggregate
local-vector state/counts/timestamps. Never expose remote WeKnora ids,
tenant/creator identity, storage credentials, API credentials, raw vectors, or
per-page indexing errors. All anonymous reader calls use that
public namespace and the `/knowledge-bases/{id}` Wiki-page children. Those
endpoints must resolve with a synthetic anonymous user and `is_admin=False`,
explicitly require `publication_status=published`, and hard-deny the system
@@ -1709,3 +1716,44 @@ before MarkItDown extraction, avoiding concurrent profile locks. The image
build and `OFFLINE_INSTALL.sh --office-check` both run
`scripts/office_runtime_selfcheck.py`; that smoke test creates and re-parses
minimal DOCX/XLSX/PPTX files and does not need the application database.
## Skill/assistant knowledge invariants
Skill distillation is implemented in `deerflow.skill_knowledge` and
`deerflow.persistence.skill_knowledge`; native assistant bases are in
`deerflow.assistant_knowledge` and
`deerflow.persistence.assistant_knowledge`. Gateway composition, remote adapters,
the lease dispatcher, and APIs remain in `app.gateway` so the harness-to-app
import firewall is preserved.
Never execute a skill or read/upload/summarize its code and script files.
Knowledge fingerprints exclude those files and Markdown code fences. All
resyncs must stage and validate the knowledge digest before activation; the old
snapshot remains authoritative on failure. A binding may have only one active
item (`active_dedupe_key` plus atomic lease claims). Skill distillation targets
only writable WeKnora Wiki knowledge bases in this product version; reject direct
assistant-base targets, non-Wiki/FAQ/conversation/readonly targets, and direct
Neo4j writes. Detach/delete operates on source contributions, not canonical rows,
and shared entities/relations are retired only when their final contribution is
gone.
Assistant knowledge has one global full-library base (`知识梳理总库`) and any
number of admin-created custom bases. Ordinary users have read/search access and
administrators own all mutations. Assistant bases only accept WeKnora export
packages or imports triggered from a WeKnora mapping. Do not add direct skill
imports or arbitrary local-file imports to assistant knowledge. Importing into a
custom assistant base must also mirror the package into the global base for
cross-source entity alignment.
## openGauss / GaussDB persistence invariant
`database.backend=gauss` is an application-ORM backend, parallel to `mysql`.
It normalizes `database.gauss_url` to `opengauss+asyncpg://` and requires the
`gauss` optional dependency (`asyncpg` plus `opengauss-sqlalchemy`). Never route
a Gauss URL through SQLAlchemy's generic PostgreSQL dialect: the wire protocol
may connect, but dialect initialization and schema DDL can fail against the
Gauss server/version behavior. Keep the LangGraph checkpointer on the separately
configured SQLite backend; the PostgreSQL checkpointer schema is not part of the
Gauss compatibility contract. Treat both `postgresql` and `opengauss` dialect
names as timezone-aware PostgreSQL-family engines in portable types and
dialect-specific session settings.
+2 -2
View File
@@ -13,8 +13,8 @@ FROM python:3.12-slim-bookworm AS builder
ARG NODE_MAJOR=22
ARG APT_MIRROR
ARG UV_INDEX_URL
# Optional extras to install (e.g. "postgres" for PostgreSQL support)
# Usage: docker build --build-arg UV_EXTRAS=postgres ...
# Optional extras to install (e.g. "postgres" or "gauss")
# Usage: docker build --build-arg UV_EXTRAS=gauss ...
ARG UV_EXTRAS
# Optionally override apt mirror for restricted networks (e.g. APT_MIRROR=mirrors.aliyun.com)
+68 -4
View File
@@ -158,6 +158,7 @@ FastAPI application providing REST endpoints for frontend integration:
| `POST /api/threads/{id}/uploads` | Upload files (auto-converts PDF/PPT/Excel/Word to Markdown, rejects directory paths) |
| `GET /api/threads/{id}/uploads/list` | List uploaded files |
| `DELETE /api/threads/{id}` | Delete DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
| `GET /api/artifact-library` | List the current user's generated conversation files across normal threads, with filename/conversation search, file-kind filtering, pagination, and an optional cache-bypassing refresh. The filesystem `outputs/` directories remain authoritative; uploads and image/audio/video files are excluded. |
| `GET /api/threads/{id}/artifacts/{path}` | Serve generated artifacts |
| `GET/POST /api/deep-research/*` | Durable Deep Research sessions and jobs. `quick`/`basic` run the compact report flow; `detailed` plans subtopics and writes independent sections; `deep` recursively investigates bounded follow-up branches. All modes reuse DeerFlow's configured material provider, LLM factory, hidden-thread artifacts, and SSE event log. Deep Research session/job/event/source/message writes serialise their rows in the writer transaction, rather than doing a post-commit ORM refresh or readback; this prevents MySQL read/write-splitting replica lag from failing “start writing”, dispatcher claim, report persistence, or follow-up history. Whole-document artifact/report rewrites likewise create their reversible version snapshots from writer-local ORM data and never refresh after commit, so replica lag cannot restore an otherwise successful rewrite. A report is written to the session and confirmed before its Job becomes `completed`; an unconfirmed write is retried and never exposed as a false completion. Retrying an already-created queued job re-nudges the dispatcher, recovering an interrupted HTTP response without creating a second report. If progress-event persistence, durable replay, or live fan-out is briefly unavailable, the worker/stream logs the degradation and continues or reconnects rather than failing the whole job. Whole-report rewrite accepts an optional `reportOutline` Markdown template; the durable job persists it and applies it as the chapter structure constraint. Optional report illustrations use a separately configured OpenAI-compatible image endpoint and are served only through the owning session's authenticated artifact route. |
| `/api/enterprise-research/*` | Temporarily disabled. Its router and persistence initialization are not registered, so these endpoints return 404 and no enterprise-research tables are created or migrated during startup. |
@@ -988,8 +989,10 @@ report-structure column for databases stamped by either old variant.
Published knowledge bases also have a login-free, chrome-free reader at
`/#/embed/knowledge/{mapping_id}` for direct third-party iframe use. With no
`wikiSlug`, it selects the Wiki **索引** page first (recognized by title, page
type, or `index`/`home`/`readme` slug), pins that page to the top of the public
directory, and otherwise falls back to an overview/root page;
type, or `index`/`home`/`readme` slug) and pins that page to the top of the
public directory. If WeKnora has pages but no explicit index page, the reader
creates a virtual `__index__` landing page from the public directory; empty
dynamic-index groups also fall back to those page summaries;
`?wikiSlug=...` deep-links one page. The reader supports
`theme=light|dark`, `bg`, and `text` embed appearance parameters. Index-article
catalogue sections (page-type counts, nested categories, page summaries) are
@@ -1005,8 +1008,14 @@ only from the read-only `/api/public/llmwiki/knowledge-bases/{id}/...` namespace
which rechecks `publication_status=published` on every request and denies the
system conversation-deposit knowledge base. Private or unpublished bases are
never exposed by this route. Anonymous consumers can discover the available
public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each
entry deliberately contains only the DeerFlow mapping `id` and display `name`.
public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`. The
endpoint validates each published mapping against WeKnora, omits stale remote
mappings, and returns safe comprehensive metadata: description/type,
document/chunk/processing/share counts, non-secret configuration and
timestamps, Wiki page/type/status counts plus explicit/generated index mode and
default embed route, and aggregate local-vector state/counts/timestamps. It
never returns the WeKnora id, tenant/creator identity, storage credentials, API
credentials, raw vectors, or per-page indexing errors.
Protected `resource://` and supported object-storage images are fetched through
the current mapping instead of being exposed as upstream URLs. Authenticated
readers use `GET /api/llmwiki/knowledge-bases/{id}/files?file_path=...`; public
@@ -1077,3 +1086,58 @@ uploads are parsed directly; legacy DOC/XLS/PPT and ODT/ODS/ODP files are
normalized through an isolated LibreOffice profile before Markdown extraction.
Run `./OFFLINE_INSTALL.sh --office-check` from the offline bundle to generate
and re-parse Word, Excel, and PowerPoint samples without requiring a database.
## Skill knowledge distillation and assistant knowledge
The Gateway now exposes an administrator-only skill distillation pipeline at
`/api/skill-knowledge`. Each request can send up to 200 skills to one writable
WeKnora Wiki knowledge base; multi-base distillation is done by repeated
operations so the local binding remains `skill_name + target_id`. The scanner
never reads or uploads code/script files, strips Markdown code fences, blocks
credential-like files and content, safely inspects ZIP members, and converts
supported Office/PDF/spreadsheet files off the event loop. Jobs use database
claims and renewable leases, stage snapshots before activation, retain the
prior active version on failure, support per-item retry, and keep per-binding
contribution records for safe detach/delete. Direct assistant-base targets,
non-Wiki/FAQ/conversation/readonly targets, and direct Neo4j writes are rejected
for this product version.
`/api/assistant-knowledge` manages DeerFlow-native assistant knowledge bases.
There is one global full-library base named `知识梳理总库`, plus admin-created
custom assistant bases. Reads and search are available to authenticated users;
global initialization, custom-base creation, WeKnora export/import, source
reimport, and source deletion are admin-only. Assistant bases only accept
WeKnora export packages or imports triggered from a WeKnora mapping; direct
skill import and arbitrary local-file import return 410. Importing into a custom
assistant base also mirrors the same package into the global base, where
canonical entities and relations are aligned across all imported sources.
The database schema is introduced by Alembic revision `20260901_01`.
## openGauss / GaussDB application database
The application ORM can use a PG-compatible openGauss/GaussDB database through
the official `opengauss-sqlalchemy` async dialect. Install the optional driver
set before enabling it:
```bash
uv sync --extra gauss
```
Configure the business database independently from the LangGraph checkpointer:
```yaml
database:
backend: gauss
gauss_url: $GAUSS_DATABASE_URL
checkpointer:
type: sqlite
connection_string: .deer-flow/data/deerflow.db
```
`GAUSS_DATABASE_URL` should normally be
`opengauss+asyncpg://user:password@host:26000/deerflow`. Plain
`opengauss://`, `gaussdb://`, and PostgreSQL-style URLs are accepted and
normalized to the openGauss dialect. Do not configure the LangGraph
PostgreSQL checkpointer against GaussDB: its migrations are PostgreSQL-specific.
For a production image, build with `--build-arg UV_EXTRAS=gauss`.
@@ -32,6 +32,7 @@ from app.gateway.routers import (
agents,
ai_writing,
artifacts,
assistant_knowledge,
assistants_compat,
auth,
browser_context,
@@ -87,6 +88,7 @@ from app.gateway.routers import (
sensitive_words,
sentiment_agent,
skill_addresses,
skill_knowledge,
skills,
suggestions,
system_settings,
@@ -1580,6 +1582,10 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Skills API is mounted at /api/skills
app.include_router(skills.router)
# Admin skill distillation matrix + global assistant knowledge base.
app.include_router(skill_knowledge.router)
app.include_router(assistant_knowledge.router)
app.include_router(assistant_knowledge.export_router)
# Artifacts API is mounted at /api/threads/{thread_id}/artifacts
app.include_router(artifacts.router)
@@ -331,6 +331,33 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.llmwiki_store = make_llmwiki_store(sf)
register_llmwiki_store(app.state.llmwiki_store)
# Skill distillation and the single global assistant knowledge base.
# Both repositories live in ``deerflow.*``; only this composition layer
# wires them to Gateway executors and remote adapters.
from deerflow.persistence.assistant_knowledge import make_assistant_knowledge_store
from deerflow.persistence.skill_knowledge import make_skill_knowledge_store
app.state.assistant_knowledge_store = make_assistant_knowledge_store(sf)
app.state.skill_knowledge_store = make_skill_knowledge_store(sf)
if app.state.skill_knowledge_store is not None and app.state.assistant_knowledge_store is not None:
from app.gateway.skill_knowledge_job_dispatcher import SkillKnowledgeJobDispatcher
from app.gateway.skill_knowledge_job_executor import SkillKnowledgeJobExecutor
app.state.skill_knowledge_executor = SkillKnowledgeJobExecutor(app)
app.state.skill_knowledge_dispatcher = SkillKnowledgeJobDispatcher(
app.state.skill_knowledge_store,
app.state.skill_knowledge_executor,
)
app.state.skill_knowledge_dispatcher.start()
from app.gateway.skill_knowledge_auto_sync import SkillKnowledgeAutoSyncService
app.state.skill_knowledge_auto_sync = SkillKnowledgeAutoSyncService(app)
app.state.skill_knowledge_auto_sync.start()
else:
app.state.skill_knowledge_executor = None
app.state.skill_knowledge_dispatcher = None
app.state.skill_knowledge_auto_sync = None
# DeerFlow-local Wiki mirror/vector index. The store is always wired so
# migrations and status inspection work while feature flags are off;
# network/search services only exist when explicitly enabled.
@@ -351,13 +378,21 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
resolved_llmwiki = get_resolved_llmwiki_runtime(config)
weknora_client = build_weknora_client(resolved_llmwiki) if resolved_llmwiki.weknora_enabled else None
if local_index_config.enabled:
embedding_config = local_index_config.embedding
embedding_configured = bool(
embedding_config.enabled
and embedding_config.base_url
and embedding_config.model.strip()
)
if embedding_configured:
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(embedding_config)
if local_index_config.enabled and app.state.llmwiki_embedding is not None:
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(local_index_config.embedding)
app.state.llmwiki_vector_cache = WikiVectorCache(
app.state.llmwiki_index_store,
max_knowledge_bases=local_index_config.cache_max_knowledge_bases,
@@ -649,6 +684,12 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.workflows.runtime.sql_runner import dispose_engines
await dispose_engines()
skill_knowledge_auto_sync = getattr(app.state, "skill_knowledge_auto_sync", None)
if skill_knowledge_auto_sync is not None:
await skill_knowledge_auto_sync.close()
skill_knowledge_dispatcher = getattr(app.state, "skill_knowledge_dispatcher", None)
if skill_knowledge_dispatcher is not None:
await skill_knowledge_dispatcher.close()
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
if queue is not None:
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
@@ -40,6 +40,7 @@ _MAX_SOURCE_CHARS = 1400
@dataclass(frozen=True)
class LlmWikiRagContext:
knowledge_base_ids: list[str]
assistant_knowledge_base_ids: list[str]
source_count: int
prompt: str
sources: list[dict[str, Any]]
@@ -50,6 +51,10 @@ def _store(request: Request) -> LlmWikiStore | None:
return getattr(request.app.state, "llmwiki_store", None)
def _assistant_store(request: Request) -> Any | None:
return getattr(request.app.state, "assistant_knowledge_store", None)
async def _actor(request: Request) -> tuple[str, bool]:
user = await get_optional_user_from_request(request)
if user is None:
@@ -76,6 +81,13 @@ def _body_selected_ids(body: Any) -> list[str]:
return []
def _body_selected_assistant_ids(body: Any) -> list[str]:
for source in (getattr(body, "context", None), getattr(body, "metadata", None)):
if isinstance(source, Mapping) and "assistant_knowledge_base_ids" in source:
return _as_id_list(source.get("assistant_knowledge_base_ids"))
return []
def _assistant_agent_id(body: Any) -> str | None:
assistant_id = validate_agent_id(getattr(body, "assistant_id", None))
if assistant_id:
@@ -106,8 +118,17 @@ def resolve_requested_llmwiki_knowledge_base_ids(body: Any) -> list[str]:
return _body_selected_ids(body)
def resolve_requested_assistant_knowledge_base_ids(body: Any) -> list[str]:
"""Return assistant knowledge bases explicitly selected for the run."""
return _body_selected_assistant_ids(body)
def llmwiki_selection_requested(body: Any) -> bool:
return bool(resolve_requested_llmwiki_knowledge_base_ids(body))
return bool(
resolve_requested_llmwiki_knowledge_base_ids(body)
or resolve_requested_assistant_knowledge_base_ids(body)
)
def _latest_human_query(graph_input: dict[str, Any]) -> str:
@@ -191,18 +212,50 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
}
def _source_from_assistant_result(result: dict[str, Any], index: int) -> dict[str, Any]:
title = str(result.get("title") or f"助手知识库片段 {index}")
matched_section = result.get("matched_section") if isinstance(result.get("matched_section"), Mapping) else {}
content = str(
matched_section.get("content")
or result.get("content_markdown")
or result.get("content")
or result.get("summary")
or ""
)
snippet = str(result.get("summary") or content)
raw = {
**result,
"source": "助手知识库",
"url": "/page/workspace/knowledge/assistant",
}
return {
"index": index,
"id": result.get("id") or result.get("slug") or f"assistant:{index}",
"title": title,
"type": "assistant_knowledge",
"snippet": _truncate(snippet, 360),
"content": content,
"source": "助手知识库",
"score": float(result.get("score") or 0),
"url": "/page/workspace/knowledge/assistant",
"skillName": "assistant_knowledge_search",
"mode": "citation",
"raw": raw,
}
def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
if not sources:
return "当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
return "当前用户选择了知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
lines = [
"当前用户选择了 LLMWiki 知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
"当前用户选择了知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
"请优先依据这些资料回答;如果资料不足,请明确说明不足之处,不要编造。",
"引用规则:正文关键结论必须使用 [1]、[2] 这样的编号引用下方资料;只能引用存在的编号;不要在正文末尾重复输出完整参考文献列表,前端会展示引用卡片。",
"",
f"用户问题:{query}",
"",
f'<llmwiki_wiki_reference_batch source_count="{len(sources)}">',
f'<knowledge_reference_batch source_count="{len(sources)}">',
]
for source in sources:
meta = " / ".join(
@@ -213,12 +266,12 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
)
if item
)
lines.append(f"[{source['index']}] {source.get('title') or 'LLMWiki 片段'}")
lines.append(f"[{source['index']}] {source.get('title') or '知识库片段'}")
if meta:
lines.append(f"来源:{meta}")
lines.append(_truncate(str(source.get("content") or source.get("snippet") or ""), _MAX_SOURCE_CHARS))
lines.append("")
lines.append("</llmwiki_wiki_reference_batch>")
lines.append("</knowledge_reference_batch>")
return "\n".join(lines).strip()
@@ -229,52 +282,99 @@ async def build_llmwiki_rag_context(
graph_input: dict[str, Any],
) -> LlmWikiRagContext:
selected_ids = resolve_requested_llmwiki_knowledge_base_ids(body)
if not selected_ids:
return LlmWikiRagContext([], 0, "", [], "")
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
local_config = request.app.state.config.llmwiki.local_wiki_index
if not (local_config.enabled and local_config.internal_search_enabled) and not runtime.weknora_enabled:
return LlmWikiRagContext(selected_ids, 0, "", [], "")
store = _store(request)
if store is None:
return LlmWikiRagContext(selected_ids, 0, "", [], "")
assistant_selected_ids = resolve_requested_assistant_knowledge_base_ids(body)
if not selected_ids and not assistant_selected_ids:
return LlmWikiRagContext([], [], 0, "", [], "")
query = _latest_human_query(graph_input)
if not query:
return LlmWikiRagContext(selected_ids, 0, "", [], "")
return LlmWikiRagContext(selected_ids, assistant_selected_ids, 0, "", [], "")
user_id, is_admin = await _actor(request)
rows: list[dict[str, Any]] = []
for mapping_id in selected_ids:
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if row is None:
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
rows.append(row)
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
if search_service is None:
return LlmWikiRagContext(
selected_ids,
0,
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
[],
query,
)
try:
payload = await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
except WikiIndexError as exc:
logger.warning("Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s", sanitize_log_param(user_id), len(rows), exc.code)
return LlmWikiRagContext(selected_ids, 0, f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。", [], query)
mapping_by_id = {str(row["id"]): row for row in rows}
sources: list[dict[str, Any]] = []
for result in payload.get("results") or []:
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
if mapping is None:
continue
sources.append(_source_from_result(result, mapping, len(sources) + 1))
if len(sources) >= _MAX_SOURCES:
break
return LlmWikiRagContext(selected_ids, len(sources), _build_prompt(query, sources), sources, query)
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
local_config = request.app.state.config.llmwiki.local_wiki_index
user_id, is_admin = await _actor(request)
if selected_ids:
llmwiki_available = (local_config.enabled and local_config.internal_search_enabled) or runtime.weknora_enabled
store = _store(request)
if not llmwiki_available or store is None:
if not assistant_selected_ids:
return LlmWikiRagContext(selected_ids, assistant_selected_ids, 0, "", [], query)
else:
rows: list[dict[str, Any]] = []
for mapping_id in selected_ids:
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if row is None:
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
rows.append(row)
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
if search_service is None:
if not assistant_selected_ids:
return LlmWikiRagContext(
selected_ids,
assistant_selected_ids,
0,
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
[],
query,
)
try:
payload = (
await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
if search_service is not None
else {"results": []}
)
except WikiIndexError as exc:
logger.warning(
"Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s",
sanitize_log_param(user_id),
len(rows),
exc.code,
)
if not assistant_selected_ids:
return LlmWikiRagContext(
selected_ids,
assistant_selected_ids,
0,
f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。",
[],
query,
)
payload = {"results": []}
mapping_by_id = {str(row["id"]): row for row in rows}
for result in payload.get("results") or []:
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
if mapping is None:
continue
sources.append(_source_from_result(result, mapping, len(sources) + 1))
if len(sources) >= _MAX_SOURCES:
break
if assistant_selected_ids and len(sources) < _MAX_SOURCES:
assistant_store = _assistant_store(request)
if assistant_store is not None:
rows = await assistant_store.search_pages(
base_ids=assistant_selected_ids,
query=query,
limit=_MAX_SOURCES - len(sources),
include_chunks=True,
embedding_client=getattr(request.app.state, "llmwiki_embedding", None),
)
for row in rows:
sources.append(_source_from_assistant_result(row, len(sources) + 1))
if len(sources) >= _MAX_SOURCES:
break
return LlmWikiRagContext(
selected_ids,
assistant_selected_ids,
len(sources),
_build_prompt(query, sources),
sources,
query,
)
def attach_llmwiki_rag_to_graph_input(
@@ -294,6 +394,7 @@ def attach_llmwiki_rag_to_graph_input(
existing = str(extra.get("prompt_prefix") or "").strip()
extra["prompt_prefix"] = f"{existing}\n\n{rag.prompt}".strip() if existing else rag.prompt
extra["llmwiki_knowledge_base_ids"] = rag.knowledge_base_ids
extra["assistant_knowledge_base_ids"] = rag.assistant_knowledge_base_ids
extra["llmwiki_reference_query"] = rag.query
if rag.sources:
extra["llmwiki_reference_sources"] = rag.sources
@@ -461,7 +461,7 @@ async def _set_analytics_read_timeout(session) -> None:
- MariaDB: ``max_statement_time`` (seconds)
- MySQL <= 5.6: no per-statement timeout variable — skip it entirely
(5.6 raises "Unknown system variable" otherwise)
- PostgreSQL: ``statement_timeout`` (milliseconds)
- PostgreSQL/openGauss: ``statement_timeout`` (milliseconds)
"""
conn = await session.connection()
dialect = conn.dialect
@@ -474,7 +474,7 @@ async def _set_analytics_read_timeout(session) -> None:
await _try_set_session_value(session, f"SET SESSION max_execution_time = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
# else: MySQL <= 5.6 has no statement-timeout variable — skip silently.
await _try_set_session_value(session, f"SET SESSION innodb_lock_wait_timeout = {_LEADERBOARD_LOCK_WAIT_SECONDS}")
elif name == "postgresql":
elif name in {"postgresql", "opengauss"}:
await _try_set_session_value(session, f"SET LOCAL statement_timeout = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
@@ -1,11 +1,15 @@
import asyncio
import logging
import mimetypes
import time
import zipfile
from datetime import datetime, timezone
from collections import OrderedDict
from datetime import UTC, datetime
from pathlib import Path
from typing import Literal
from urllib.parse import quote
from fastapi import APIRouter, HTTPException, Request
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import FileResponse, PlainTextResponse, Response
from pydantic import BaseModel
@@ -13,6 +17,7 @@ from app.gateway.authz import require_permission
from app.gateway.path_utils import aresolve_thread_virtual_path
from deerflow.config.paths import get_paths
from deerflow.runtime.thread_paths import aresolve_path_user_id
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
@@ -38,6 +43,85 @@ EDITABLE_TEXT_SUFFIXES = {
_VIRTUAL_USER_DATA_PREFIX = "mnt/user-data"
ArtifactLibraryKind = Literal["all", "document", "spreadsheet", "text_data", "archive", "other"]
ArtifactLibrarySource = Literal["all", "general_qa", "agent"]
_MEDIA_MIME_PREFIXES = ("image/", "audio/", "video/")
_MEDIA_SUFFIXES = {
".aac",
".aiff",
".ape",
".avi",
".bmp",
".flac",
".gif",
".heic",
".ico",
".jpeg",
".jpg",
".m4a",
".m4v",
".mkv",
".mov",
".mp3",
".mp4",
".ogg",
".png",
".svg",
".tiff",
".wav",
".webm",
".webp",
".wma",
}
_DOCUMENT_SUFFIXES = {
".doc",
".docx",
".epub",
".odt",
".pdf",
".ppt",
".pptx",
".rtf",
}
_SPREADSHEET_SUFFIXES = {".csv", ".ods", ".tsv", ".xls", ".xlsx"}
_TEXT_DATA_SUFFIXES = {
".bash",
".c",
".conf",
".cpp",
".css",
".go",
".h",
".html",
".ini",
".java",
".js",
".json",
".jsonl",
".log",
".markdown",
".md",
".mmd",
".py",
".rs",
".sh",
".sql",
".toml",
".ts",
".tsx",
".txt",
".xml",
".yaml",
".yml",
}
_ARCHIVE_SUFFIXES = {".7z", ".gz", ".rar", ".skill", ".tar", ".tgz", ".zip"}
_ARTIFACT_LIBRARY_CACHE_TTL_SECONDS = 15.0
_ARTIFACT_LIBRARY_CACHE_MAX_USERS = 128
_ARTIFACT_LIBRARY_SCAN_CONCURRENCY = 8
_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE = 500
_artifact_library_cache: OrderedDict[str, tuple[float, list[dict]]] = OrderedDict()
def coerce_artifact_virtual_path(path: str) -> str:
"""Normalize an artifacts URL path to a sandbox virtual path.
@@ -62,6 +146,28 @@ class ArtifactUpdateRequest(BaseModel):
content: str
class ArtifactLibraryItem(BaseModel):
thread_id: str
thread_title: str
agent_id: str | None = None
agent_name: str | None = None
thread_type: str | None = None
source_type: Literal["general_qa", "agent"]
path: str
name: str
size_bytes: int
mime_type: str | None = None
modified_at: str
file_kind: str
class ArtifactLibraryResponse(BaseModel):
total: int
offset: int
limit: int
items: list[ArtifactLibraryItem]
def _build_content_disposition(disposition_type: str, filename: str) -> str:
"""Build an RFC 5987 encoded Content-Disposition header value."""
return f"{disposition_type}; filename*=UTF-8''{quote(filename)}"
@@ -139,16 +245,22 @@ def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
resolved.relative_to(outputs_root)
except (OSError, ValueError):
continue
relative = resolved.relative_to(outputs_root).as_posix()
mime_type, _ = mimetypes.guess_type(resolved.name)
stat = resolved.stat()
try:
relative = resolved.relative_to(outputs_root).as_posix()
mime_type, _ = mimetypes.guess_type(resolved.name)
stat = resolved.stat()
except OSError:
# A run may atomically replace or remove a file while this directory
# is being scanned. Treat that transient entry as absent instead of
# failing the complete thread/library listing.
continue
files.append(
{
"path": f"mnt/user-data/outputs/{relative}",
"name": resolved.name,
"size_bytes": stat.st_size,
"mime_type": mime_type,
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(),
}
)
@@ -156,6 +268,190 @@ def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
return files
def artifact_library_kind(name: str) -> str:
"""Classify one non-media artifact for the library filters."""
suffix = Path(name).suffix.lower()
if suffix in _DOCUMENT_SUFFIXES:
return "document"
if suffix in _SPREADSHEET_SUFFIXES:
return "spreadsheet"
if suffix in _TEXT_DATA_SUFFIXES:
return "text_data"
if suffix in _ARCHIVE_SUFFIXES:
return "archive"
return "other"
def is_artifact_library_file(file: dict) -> bool:
"""Return whether a generated file belongs in the document-only library."""
mime_type = str(file.get("mime_type") or "").lower()
if mime_type.startswith(_MEDIA_MIME_PREFIXES):
return False
return Path(str(file.get("name") or file.get("path") or "")).suffix.lower() not in _MEDIA_SUFFIXES
async def _search_artifact_library_threads(thread_store) -> list[dict]:
"""Load every normal conversation owned by the current request user."""
rows: list[dict] = []
seen: set[str] = set()
offset = 0
while True:
batch = await thread_store.search(
limit=_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE,
offset=offset,
exclude_system=True,
)
if not batch:
break
added = 0
for row in batch:
thread_id = str(row.get("thread_id") or "")
if not thread_id or thread_id in seen:
continue
seen.add(thread_id)
rows.append(row)
added += 1
if len(batch) < _ARTIFACT_LIBRARY_THREAD_BATCH_SIZE or added == 0:
break
offset += len(batch)
return rows
async def _scan_artifact_library(thread_store) -> list[dict]:
"""Scan generated outputs for all current-user conversation threads."""
rows = await _search_artifact_library_threads(thread_store)
paths = get_paths()
semaphore = asyncio.Semaphore(_ARTIFACT_LIBRARY_SCAN_CONCURRENCY)
async def scan_thread(row: dict) -> list[dict]:
thread_id = str(row.get("thread_id") or "")
if not thread_id:
return []
async with semaphore:
try:
path_user_id = await aresolve_path_user_id(thread_id)
outputs_dir = paths.sandbox_outputs_dir(thread_id, user_id=path_user_id)
files = await asyncio.to_thread(collect_outputs_listing, outputs_dir)
except Exception:
logger.warning("Failed to scan artifacts for library thread %s", thread_id, exc_info=True)
return []
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
agent_id = metadata.get("agent_id")
agent_name = metadata.get("agent_name")
thread_type = metadata.get("thread_type")
normalized_agent_id = agent_id.strip() if isinstance(agent_id, str) else ""
normalized_agent_name = agent_name.strip() if isinstance(agent_name, str) else ""
title = str(row.get("display_name") or "新对话")
items: list[dict] = []
for file in files:
if not is_artifact_library_file(file):
continue
items.append(
{
"thread_id": thread_id,
"thread_title": title,
"agent_id": normalized_agent_id or None,
"agent_name": normalized_agent_name or None,
"thread_type": thread_type if isinstance(thread_type, str) and thread_type else None,
"source_type": "agent" if normalized_agent_id else "general_qa",
**file,
"file_kind": artifact_library_kind(str(file.get("name") or "")),
}
)
return items
nested = await asyncio.gather(*(scan_thread(row) for row in rows))
items = [item for group in nested for item in group]
items.sort(
key=lambda item: (
str(item.get("modified_at") or ""),
str(item.get("thread_id") or ""),
str(item.get("path") or ""),
),
reverse=True,
)
return items
async def _cached_artifact_library(thread_store, *, refresh: bool) -> list[dict]:
user_id = str(get_effective_user_id() or "default")
now = time.monotonic()
cached = _artifact_library_cache.get(user_id)
if not refresh and cached and cached[0] > now:
_artifact_library_cache.move_to_end(user_id)
return cached[1]
items = await _scan_artifact_library(thread_store)
_artifact_library_cache[user_id] = (now + _ARTIFACT_LIBRARY_CACHE_TTL_SECONDS, items)
_artifact_library_cache.move_to_end(user_id)
while len(_artifact_library_cache) > _ARTIFACT_LIBRARY_CACHE_MAX_USERS:
_artifact_library_cache.popitem(last=False)
return items
@router.get(
"/artifact-library",
response_model=ArtifactLibraryResponse,
summary="List Generated Files Across Conversations",
description="Aggregate non-media files generated in the current user's normal conversations.",
)
@require_permission("threads", "read")
async def list_artifact_library(
request: Request,
query: str | None = Query(default=None, max_length=200),
file_kind: ArtifactLibraryKind = Query(default="all"),
source_type: ArtifactLibrarySource = Query(default="all"),
offset: int = Query(default=0, ge=0),
limit: int = Query(default=50, ge=1, le=200),
refresh: bool = Query(default=False),
) -> ArtifactLibraryResponse:
"""Return a flat, newest-first library of generated conversation files.
The filesystem remains the source of truth: every owned, non-system thread's
``outputs`` directory is scanned, including files written without an
explicit ``present_files`` call. Uploads and image/audio/video outputs are
intentionally outside this collection.
"""
from app.gateway.deps import get_agent_store, get_thread_store
items = await _cached_artifact_library(get_thread_store(request), refresh=refresh)
agent_ids = list(dict.fromkeys(str(item.get("agent_id") or "") for item in items if item.get("agent_id")))
agent_names: dict[str, str] = {}
if agent_ids:
try:
agents = await get_agent_store(request).list_by_ids(
agent_ids,
str(get_effective_user_id() or "default"),
)
agent_names = {str(agent.get("id") or ""): str(agent.get("name") or "").strip() for agent in agents if agent.get("id") and agent.get("name")}
except Exception:
logger.warning("Failed to resolve artifact library agent names", exc_info=True)
items_with_sources = [
{
**item,
"agent_name": (agent_names.get(str(item.get("agent_id") or "")) or item.get("agent_name") or item.get("agent_id")),
}
for item in items
]
normalized_query = (query or "").strip().casefold()
filtered = [
item
for item in items_with_sources
if (file_kind == "all" or item["file_kind"] == file_kind)
and (source_type == "all" or item["source_type"] == source_type)
and (not normalized_query or normalized_query in str(item.get("name") or "").casefold() or normalized_query in str(item.get("thread_title") or "").casefold() or normalized_query in str(item.get("agent_name") or "").casefold())
]
page = filtered[offset : offset + limit]
return ArtifactLibraryResponse(
total=len(filtered),
offset=offset,
limit=limit,
items=[ArtifactLibraryItem.model_validate(item) for item in page],
)
@router.get(
"/threads/{thread_id}/artifacts",
summary="List Artifact Files",
File diff suppressed because it is too large. Load diff
@@ -55,6 +55,7 @@ from deerflow.config.agents_config import load_agent_config
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
from deerflow.config.system_settings import load_system_settings, save_system_settings
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.persistence.llmwiki import LlmWikiStore
@@ -496,15 +497,7 @@ def _should_follow_weknora_binary_redirect(path: str) -> bool:
"""Keep protected preview/image bytes inside the DeerFlow proxy."""
segments = _path_segments(path)
return (
len(segments) == 5
and segments[:3] == ["api", "v1", "knowledge"]
and segments[4] == "preview"
) or (
len(segments) == 5
and segments[:3] == ["api", "v1", "knowledge-bases"]
and segments[4] == "files"
)
return (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge"] and segments[4] == "preview") or (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge-bases"] and segments[4] == "files")
def _forward_request_headers(request: Request) -> dict[str, str]:
@@ -1833,6 +1826,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
"knowledge_count": int(source.get("knowledge_count") or 0),
"chunk_count": int(source.get("chunk_count") or 0),
"processing_count": int(source.get("processing_count") or 0),
"share_count": int(source.get("share_count") or 0),
"is_processing": bool(source.get("is_processing") or int(source.get("processing_count") or 0) > 0),
"is_temporary": bool(source.get("is_temporary", False)),
"embedding_model_id": source.get("embedding_model_id"),
"vector_store_id": source.get("vector_store_id"),
"vector_store_name": source.get("vector_store_name"),
@@ -1840,16 +1836,23 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
"vector_store_engine_type": source.get("vector_store_engine_type"),
"vector_store_status": source.get("vector_store_status"),
"chunking_config": source.get("chunking_config") if isinstance(source.get("chunking_config"), dict) else {},
"image_processing_config": (source.get("image_processing_config") if isinstance(source.get("image_processing_config"), dict) else {}),
"indexing_strategy": source.get("indexing_strategy") if isinstance(source.get("indexing_strategy"), dict) else {},
"capabilities": source.get("capabilities") if isinstance(source.get("capabilities"), dict) else {},
"extract_config": source.get("extract_config") if isinstance(source.get("extract_config"), dict) else {},
"faq_config": source.get("faq_config") if isinstance(source.get("faq_config"), dict) else {},
"wiki_config": source.get("wiki_config") if isinstance(source.get("wiki_config"), dict) else {},
"question_generation_config": (source.get("question_generation_config") if isinstance(source.get("question_generation_config"), dict) else {}),
"auto_tag_config": source.get("auto_tag_config") if isinstance(source.get("auto_tag_config"), dict) else {},
"vlm_config": source.get("vlm_config") if isinstance(source.get("vlm_config"), dict) else {},
"asr_config": source.get("asr_config") if isinstance(source.get("asr_config"), dict) else {},
"summary_model_id": source.get("summary_model_id"),
"storage_provider": (source.get("storage_provider_config", {}).get("provider") if isinstance(source.get("storage_provider_config"), dict) else None),
"created_at": row.get("created_at"),
"updated_at": row.get("updated_at"),
"published_at": row.get("published_at"),
"remote_created_at": source.get("created_at"),
"remote_updated_at": source.get("updated_at"),
}
@@ -1877,11 +1880,7 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
write=False,
is_admin=False,
)
if (
row is None
or row.get("publication_status") != "published"
or _is_conversation_deposit_mapping(row)
):
if row is None or row.get("publication_status") != "published" or _is_conversation_deposit_mapping(row):
raise HTTPException(status_code=404, detail="Published knowledge base not found")
return row
@@ -2537,24 +2536,260 @@ async def test_llmwiki_connection(request: Request, body: LlmWikiConnectionTest)
return {"success": True, "provider": "weknora", "health": health}
_PUBLIC_WIKI_SUMMARY_CONCURRENCY = 8
_PUBLIC_WIKI_PAGE_SIZE = 500
_PUBLIC_WIKI_PAGE_LIMIT = 100
_GENERATED_PUBLIC_WIKI_INDEX_SLUG = "__index__"
def _is_public_wiki_index_page(page: dict[str, Any]) -> bool:
page_type = str(page.get("page_type") or "").strip().lower()
title = str(page.get("title") or "").strip().lower()
slug_name = str(page.get("slug") or "").strip("/").rsplit("/", 1)[-1].lower()
return page_type == "index" or title in {"索引", "index"} or slug_name in {"index", "home", "readme"}
def _public_wiki_default_page(pages: list[dict[str, Any]]) -> dict[str, Any] | None:
return (
next((page for page in pages if _is_public_wiki_index_page(page)), None)
or next((page for page in pages if str(page.get("page_type") or "").lower() == "overview"), None)
or next(
(page for page in pages if not str(page.get("parent_slug") or "").strip() and int(page.get("depth") or 0) == 0),
None,
)
or (pages[0] if pages else None)
)
def _count_public_wiki_values(pages: list[dict[str, Any]], key: str, fallback: str) -> dict[str, int]:
counts: dict[str, int] = {}
for page in pages:
value = str(page.get(key) or fallback).strip().lower() or fallback
counts[value] = counts.get(value, 0) + 1
return counts
def _public_wiki_time_bound(pages: list[dict[str, Any]], key: str, *, latest: bool) -> str | None:
values = [str(page.get(key) or "").strip() for page in pages]
values = [value for value in values if value]
if not values:
return None
return max(values) if latest else min(values)
async def _all_public_wiki_page_views(
client: WeKnoraClient,
remote_id: str,
) -> tuple[list[dict[str, Any]], int, bool]:
pages_by_slug: dict[str, dict[str, Any]] = {}
remote_total = 0
complete = False
for page_number in range(1, _PUBLIC_WIKI_PAGE_LIMIT + 1):
result = await client.list_wiki_pages(
remote_id,
page=page_number,
page_size=_PUBLIC_WIKI_PAGE_SIZE,
query="",
)
raw_pages = result.get("pages") if isinstance(result, dict) else []
page_items = raw_pages if isinstance(raw_pages, list) else []
for page_item in page_items:
if not isinstance(page_item, dict):
continue
page = _wiki_page_view(page_item)
if page is None:
continue
slug = str(page.get("slug") or "").strip()
pages_by_slug[slug or f"__page_{len(pages_by_slug)}"] = page
reported_total = int(result.get("total") or 0)
remote_total = max(remote_total, reported_total, len(pages_by_slug))
total_pages = max(1, int(result.get("total_pages") or 1))
if not page_items or page_number >= total_pages or (reported_total > 0 and len(pages_by_slug) >= remote_total):
complete = True
break
return list(pages_by_slug.values()), max(remote_total, len(pages_by_slug)), complete
def _public_wiki_summary(
mapping_id: str,
pages: list[dict[str, Any]],
*,
total: int,
complete: bool,
) -> dict[str, Any]:
explicit_index = next((page for page in pages if _is_public_wiki_index_page(page)), None)
default_page = _public_wiki_default_page(pages)
if explicit_index is not None:
index_mode = "explicit"
index_slug: str | None = str(explicit_index.get("slug") or "") or None
elif pages:
index_mode = "generated"
index_slug = _GENERATED_PUBLIC_WIKI_INDEX_SLUG
else:
index_mode = "none"
index_slug = None
default_slug = index_slug or (str(default_page.get("slug") or "") if default_page else None)
route = f"/embed/knowledge/{quote(mapping_id, safe='')}"
if default_slug:
route = f"{route}?{urlencode({'wikiSlug': default_slug})}"
return {
"status": "ready" if pages else "empty",
"page_count": total,
"loaded_page_count": len(pages),
"stats_complete": complete,
"page_type_counts": _count_public_wiki_values(pages, "page_type", "page"),
"page_status_counts": _count_public_wiki_values(pages, "status", "unknown"),
"index_mode": index_mode,
"has_explicit_index": explicit_index is not None,
"index_slug": index_slug,
"default_slug": default_slug,
"created_at": _public_wiki_time_bound(pages, "created_at", latest=False),
"updated_at": _public_wiki_time_bound(pages, "updated_at", latest=True),
"embed_route": route,
}
def _unavailable_public_wiki_summary(mapping_id: str) -> dict[str, Any]:
return {
"status": "unavailable",
"page_count": 0,
"loaded_page_count": 0,
"stats_complete": False,
"page_type_counts": {},
"page_status_counts": {},
"index_mode": "unknown",
"has_explicit_index": False,
"index_slug": None,
"default_slug": None,
"created_at": None,
"updated_at": None,
"embed_route": f"/embed/knowledge/{quote(mapping_id, safe='')}",
}
def _public_local_index_summary(
request: Request,
mapping: dict[str, Any],
state: dict[str, Any] | None,
) -> dict[str, Any]:
app_state = request.app.state
local_config = getattr(
getattr(getattr(app_state, "config", None), "llmwiki", None),
"local_wiki_index",
None,
)
enabled = bool(getattr(local_config, "enabled", False))
current = dict(state or {})
state_name = str(current.get("state") or "not_synced")
if state_name == "idle" and int(current.get("index_revision") or 0) == 0 and not current.get("last_completed_at") and not current.get("last_success_at"):
state_name = "not_synced"
embedding = getattr(app_state, "llmwiki_embedding", None)
fingerprint = embedding.fingerprint if embedding is not None else None
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(
current,
fingerprint,
)
return {
"enabled": enabled,
"wiki_index_enabled": bool(mapping.get("wiki_index_enabled", True)),
"state": state_name,
"fully_vectorized": fully_vectorized,
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
"remote_page_count": int(current.get("remote_page_count") or 0),
"local_page_count": int(current.get("local_page_count") or 0),
"ready_page_count": int(current.get("ready_page_count") or 0),
"failed_page_count": int(current.get("failed_page_count") or 0),
"vector_count": int(current.get("vector_count") or 0),
"last_started_at": current.get("last_started_at"),
"last_completed_at": current.get("last_completed_at"),
"last_success_at": current.get("last_success_at"),
"updated_at": current.get("updated_at"),
}
@router.get("/api/public/llmwiki/knowledge-bases")
async def list_public_knowledge_bases(request: Request) -> dict[str, Any]:
"""List the minimal identity of every knowledge base published as a public Wiki."""
"""List live, published Wiki libraries with safe metadata and index health."""
rows = await _store(request).list_visible(
"__public_wiki__",
scope="public",
is_admin=False,
)
knowledge_bases = [
{
"id": str(row["id"]),
"name": str(row.get("name") or ""),
}
for row in rows
if row.get("publication_status") == "published"
and not _is_conversation_deposit_mapping(row)
rows = [
row
for row in await _store(request).list_visible(
"__public_wiki__",
scope="public",
is_admin=False,
)
if row.get("publication_status") == "published" and not _is_conversation_deposit_mapping(row)
]
client = _client_or_503()
try:
remote_items = await client.list_knowledge_bases()
except WeKnoraError as exc:
_raise_upstream(exc)
remote_by_id = {str(item.get("id")): item for item in remote_items if isinstance(item, dict) and item.get("id")}
# A successful upstream list is authoritative: stale DeerFlow mappings are
# omitted so discovery never advertises a Wiki that its detail route cannot open.
rows = [row for row in rows if str(row.get("weknora_id") or "") in remote_by_id]
mapping_ids = [str(row["id"]) for row in rows]
index_states: dict[str, dict[str, Any]] = {}
index_store = getattr(request.app.state, "llmwiki_index_store", None)
if index_store is not None and mapping_ids:
try:
index_states = {str(state.get("knowledge_base_mapping_id") or ""): state for state in await index_store.list_index_status(mapping_ids) if isinstance(state, dict)}
except Exception:
logger.warning("Failed to read public Wiki local-index summaries", exc_info=True)
semaphore = asyncio.Semaphore(_PUBLIC_WIKI_SUMMARY_CONCURRENCY)
async def build_summary(row: dict[str, Any]) -> dict[str, Any]:
mapping_id = str(row["id"])
remote_id = str(row["weknora_id"])
remote = remote_by_id[remote_id]
async with semaphore:
try:
remote = {**remote, **await client.get_knowledge_base(remote_id)}
except WeKnoraError as exc:
if exc.status_code == 404:
return {}
logger.warning(
"Failed to load public Wiki detail metadata: mapping_id=%s",
mapping_id,
exc_info=True,
)
try:
pages, page_total, complete = await _all_public_wiki_page_views(client, remote_id)
wiki = _public_wiki_summary(
mapping_id,
pages,
total=page_total,
complete=complete,
)
except WeKnoraError as exc:
logger.warning(
"Failed to load public Wiki page summary: mapping_id=%s",
mapping_id,
exc_info=True,
)
wiki = (
_public_wiki_summary(mapping_id, [], total=0, complete=True)
if exc.status_code == 404
else _unavailable_public_wiki_summary(mapping_id)
)
return {
**_mapping_view(
row,
remote,
actor_user_id="__public_wiki__",
is_admin=False,
),
"wiki": wiki,
"local_index": _public_local_index_summary(
request,
row,
index_states.get(mapping_id),
),
}
knowledge_bases = [summary for summary in await asyncio.gather(*(build_summary(row) for row in rows)) if summary]
return {"knowledge_bases": knowledge_bases, "total": len(knowledge_bases)}
@@ -2596,13 +2831,7 @@ async def list_public_wiki_pages(
except WeKnoraError as exc:
_raise_upstream(exc)
raw_pages = result.get("pages") if isinstance(result, dict) else []
pages = [
view
for page_item in (raw_pages if isinstance(raw_pages, list) else [])
if isinstance(page_item, dict)
for view in [_wiki_page_view(page_item)]
if view is not None
]
pages = [view for page_item in (raw_pages if isinstance(raw_pages, list) else []) if isinstance(page_item, dict) for view in [_wiki_page_view(page_item)] if view is not None]
return {**result, "pages": pages}
@@ -2650,11 +2879,7 @@ def _public_wiki_index_item(value: Any) -> dict[str, Any] | None:
"slug": slug,
"title": str(value.get("title") or slug),
"summary": str(value.get("summary") or ""),
"category_path": (
[str(item) for item in value.get("category_path", []) if str(item).strip()]
if isinstance(value.get("category_path"), list)
else []
),
"category_path": ([str(item) for item in value.get("category_path", []) if str(item).strip()] if isinstance(value.get("category_path"), list) else []),
"wiki_path": str(value.get("wiki_path") or ""),
"depth": int(value.get("depth") or 0),
}
@@ -2680,20 +2905,12 @@ async def _all_public_wiki_index(client: WeKnoraClient, remote_id: str) -> dict[
version = max(version, int(payload.get("version") or 0))
raw_groups = payload.get("groups") if isinstance(payload.get("groups"), list) else []
group = next(
(
value
for value in raw_groups
if isinstance(value, dict) and value.get("type") == page_type
),
(value for value in raw_groups if isinstance(value, dict) and value.get("type") == page_type),
{},
)
total = max(total, int(group.get("total") or 0))
raw_items = group.get("items") if isinstance(group.get("items"), list) else []
items.extend(
item
for value in raw_items
if (item := _public_wiki_index_item(value)) is not None
)
items.extend(item for value in raw_items if (item := _public_wiki_index_item(value)) is not None)
next_cursor = str(group.get("next_cursor") or "")
if not next_cursor or next_cursor in seen_cursors:
break
@@ -3201,12 +3418,7 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
except WikiIndexError as exc:
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
mapping_by_id = {str(row["id"]): row for row in rows}
result["results"] = [
_sanitize_conversation_deposit_value(item)
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
else item
for item in result.get("results") or []
]
result["results"] = [_sanitize_conversation_deposit_value(item) if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {})) else item for item in result.get("results") or []]
return result
@@ -0,0 +1,328 @@
"""Administrator APIs for skill knowledge bindings and matrix jobs."""
from __future__ import annotations
import asyncio
from typing import Any, Literal
from croniter import croniter
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field, field_validator
from app.gateway.deps import get_optional_user_from_request
from app.gateway.skill_knowledge_job_executor import _skill_directory
from deerflow.config.system_settings import (
SkillKnowledgeAutoSyncSettings,
load_system_settings,
save_system_settings,
)
from deerflow.integrations.weknora.runtime import (
build_weknora_client,
get_resolved_llmwiki_runtime,
)
from deerflow.skill_knowledge import scan_skill
router = APIRouter(prefix="/api/skill-knowledge", tags=["skill-knowledge"])
async def _require_admin(request: Request) -> str | None:
user = await get_optional_user_from_request(request)
if user is None:
return None
if getattr(user, "system_role", None) != "admin":
raise HTTPException(status_code=403, detail="技能知识归纳仅管理员可操作")
return str(user.id)
def _store(request: Request):
value = getattr(request.app.state, "skill_knowledge_store", None)
if value is None:
raise HTTPException(status_code=503, detail="技能知识归纳存储不可用")
return value
class SyncTarget(BaseModel):
target_type: Literal["weknora", "assistant"]
target_mode: Literal["wiki", "document", "faq", "conversation_archive", "readonly_sidecar", "assistant"]
target_id: str = Field(min_length=1, max_length=128)
target_name: str = Field(default="", max_length=255)
remote_business_key: str | None = Field(default=None, max_length=512)
readonly: bool = False
class CreateSyncJobRequest(BaseModel):
skill_names: list[str] = Field(min_length=1, max_length=200)
targets: list[SyncTarget] = Field(min_length=1, max_length=50)
force: bool = False
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
@field_validator("skill_names")
@classmethod
def validate_names(cls, values: list[str]) -> list[str]:
result = list(dict.fromkeys(value.strip() for value in values if value.strip()))
if not result:
raise ValueError("At least one skill is required")
return result
class ReviewRequest(BaseModel):
review_ids: list[str] = Field(min_length=1, max_length=500)
action: Literal["approve", "reject"]
edits: dict[str, dict[str, Any]] = Field(default_factory=dict)
reason: str | None = Field(default=None, max_length=1024)
class DetachRequest(BaseModel):
delete_contributions: bool
class AutoSyncSettingsUpdate(BaseModel):
watch_enabled: bool | None = None
schedule_enabled: bool | None = None
schedule_cron: str | None = Field(default=None, min_length=5, max_length=128)
debounce_seconds: int | None = Field(default=None, ge=10, le=86_400)
write_remote_enabled: bool | None = None
review_mode: Literal["required", "auto_high_confidence", "off"] | None = None
@field_validator("schedule_cron")
@classmethod
def validate_cron(cls, value: str | None) -> str | None:
if value is not None and not croniter.is_valid(value):
raise ValueError("无效的 cron 表达式")
return value
@router.get("/bindings")
async def list_bindings(
request: Request,
skill_names: list[str] | None = Query(default=None),
target_id: str | None = Query(default=None),
include_detached: bool = Query(default=False),
) -> dict[str, Any]:
await _require_admin(request)
rows = await _store(request).list_bindings(skill_names=skill_names, target_id=target_id, include_detached=include_detached)
return {"bindings": rows}
@router.get("/source-status")
async def source_status(request: Request, skill_names: list[str] = Query(...)) -> dict[str, Any]:
await _require_admin(request)
bindings = await _store(request).list_bindings(skill_names=skill_names)
by_skill: dict[str, dict[str, Any]] = {}
for name in list(dict.fromkeys(skill_names)):
try:
root = _skill_directory(request.app.state.config, name)
result = await asyncio.to_thread(scan_skill, name, root)
by_skill[name] = {"source_digest": result.source_digest, "counts": result.counts, "blocked_files": result.blocked_files}
except Exception as exc: # noqa: BLE001
by_skill[name] = {"error": str(exc)}
for binding in bindings:
status = by_skill.get(binding["skill_name"])
if status and status.get("source_digest") and status["source_digest"] != binding.get("synced_digest"):
binding["computed_status"] = "stale"
return {"skills": by_skill, "bindings": bindings}
@router.get("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
async def get_auto_sync_settings(request: Request) -> SkillKnowledgeAutoSyncSettings:
await _require_admin(request)
return load_system_settings().skill_knowledge_auto_sync
@router.put("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
async def update_auto_sync_settings(request: Request, body: AutoSyncSettingsUpdate) -> SkillKnowledgeAutoSyncSettings:
await _require_admin(request)
current = load_system_settings()
updated = current.skill_knowledge_auto_sync.model_copy(update=body.model_dump(exclude_unset=True))
saved = save_system_settings(current.model_copy(update={"skill_knowledge_auto_sync": updated}))
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
if service is not None:
service.nudge()
return saved.skill_knowledge_auto_sync
async def _set_binding_auto_sync(request: Request, binding_id: str, *, enabled: bool) -> dict[str, Any]:
await _require_admin(request)
row = await _store(request).set_binding_auto_sync(binding_id, enabled=enabled)
if row is None:
raise HTTPException(status_code=404, detail="绑定不存在")
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
if service is not None:
service.nudge()
return row
@router.post("/bindings/{binding_id}/enable-auto-sync")
async def enable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
return await _set_binding_auto_sync(request, binding_id, enabled=True)
@router.post("/bindings/{binding_id}/disable-auto-sync")
async def disable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
return await _set_binding_auto_sync(request, binding_id, enabled=False)
@router.post("/sync-jobs", status_code=202)
async def create_sync_job(request: Request, body: CreateSyncJobRequest) -> dict[str, Any]:
actor = await _require_admin(request)
skill_store = request.app.state.skill_store
missing = [name for name in body.skill_names if await skill_store.get_any(name) is None]
if missing:
raise HTTPException(status_code=404, detail=f"技能不存在:{', '.join(missing[:10])}")
targets: list[dict[str, Any]] = []
for target in body.targets:
data = target.model_dump()
if target.target_type != "weknora":
raise HTTPException(status_code=422, detail="不支持技能直接归纳到助手知识库,请先归纳到 WeKnora 普通知识库")
if target.target_mode != "wiki":
raise HTTPException(status_code=422, detail="技能归纳目标仅支持 WeKnora Wiki 知识库")
mapping = await request.app.state.llmwiki_store.get_authorized(target.target_id, actor or "system", write=True, is_admin=True)
if mapping is None:
raise HTTPException(status_code=404, detail=f"普通知识库不存在:{target.target_id}")
data["target_name"] = target.target_name or str(mapping.get("name") or target.target_id)
data["remote_business_key"] = target.remote_business_key or f"weknora:kb:{mapping['weknora_id']}"
targets.append(data)
job = await _store(request).create_job(
skill_names=body.skill_names,
targets=targets,
created_by=actor,
force=body.force,
review_mode=body.review_mode,
)
dispatcher = getattr(request.app.state, "skill_knowledge_dispatcher", None)
if dispatcher is not None:
dispatcher.nudge()
return job
@router.get("/sync-jobs")
async def list_sync_jobs(request: Request, limit: int = Query(default=50, ge=1, le=200)) -> dict[str, Any]:
await _require_admin(request)
return {"jobs": await _store(request).list_jobs(limit=limit)}
@router.get("/sync-jobs/{job_id}")
async def get_sync_job(request: Request, job_id: str) -> dict[str, Any]:
await _require_admin(request)
row = await _store(request).get_job(job_id)
if row is None:
raise HTTPException(status_code=404, detail="归纳任务不存在")
return row
@router.post("/sync-jobs/{job_id}/cancel")
async def cancel_sync_job(request: Request, job_id: str) -> dict[str, bool]:
await _require_admin(request)
if not await _store(request).cancel_job(job_id):
raise HTTPException(status_code=404, detail="归纳任务不存在")
return {"canceled": True}
@router.post("/sync-items/{item_id}/retry", status_code=202)
async def retry_sync_item(request: Request, item_id: str) -> dict[str, Any]:
await _require_admin(request)
row = await _store(request).requeue_item(item_id)
if row is None:
raise HTTPException(status_code=404, detail="归纳任务项不存在")
request.app.state.skill_knowledge_dispatcher.nudge()
return row
@router.post("/bindings/{binding_id}/resync", status_code=202)
async def resync_binding(request: Request, binding_id: str) -> dict[str, Any]:
actor = await _require_admin(request)
binding = await _store(request).get_binding(binding_id)
if binding is None or not binding.get("active"):
raise HTTPException(status_code=404, detail="绑定不存在")
target = {
"target_type": binding["target_type"],
"target_mode": binding["target_mode"],
"target_id": binding["target_id"],
"target_name": binding["target_name"],
"remote_business_key": binding.get("remote_business_key"),
}
job = await _store(request).create_job(
skill_names=[binding["skill_name"]],
targets=[target],
created_by=actor,
force=True,
review_mode="auto_high_confidence",
)
request.app.state.skill_knowledge_dispatcher.nudge()
return job
@router.post("/bindings/{binding_id}/detach")
async def detach_binding(request: Request, binding_id: str, body: DetachRequest) -> dict[str, Any]:
await _require_admin(request)
store = _store(request)
binding = await store.get_binding(binding_id)
if binding is None:
raise HTTPException(status_code=404, detail="绑定不存在")
if await store.has_active_item(binding_id):
raise HTTPException(status_code=409, detail="该绑定仍有运行中的归纳项,请先取消任务")
cleanup_failed = False
if body.delete_contributions:
try:
if binding["target_type"] == "assistant":
sources = await request.app.state.assistant_knowledge_store.list_sources()
for source in sources:
if source.get("source_type") == "skill" and source.get("source_key") == binding["skill_name"]:
await request.app.state.assistant_knowledge_store.delete_source(source["id"])
elif binding["target_type"] == "weknora":
mapping = await request.app.state.llmwiki_store.get_authorized(
str(binding["target_id"]),
"system",
write=True,
is_admin=True,
)
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
if mapping is None or not runtime.weknora_enabled:
raise RuntimeError("远端普通知识库当前不可写")
client = build_weknora_client(runtime)
remote_kb_id = str(mapping["weknora_id"])
for artifact in await store.list_artifacts(binding_id):
if artifact["kind"] == "wiki_page" and artifact.get("slug"):
await client.delete_wiki_page(remote_kb_id, artifact["slug"])
elif artifact.get("remote_id") and artifact["kind"] in {
"raw_source",
"summary_document",
"faq_projection",
"conversation_archive",
"graph_projection",
}:
await client.delete_document(str(artifact["remote_id"]))
except Exception: # noqa: BLE001
cleanup_failed = True
row = await store.detach_binding(
binding_id,
delete_contributions=body.delete_contributions,
cleanup_failed=cleanup_failed,
)
if row is None:
raise HTTPException(status_code=404, detail="绑定不存在")
return row
@router.get("/reviews")
async def list_reviews(
request: Request,
job_id: str | None = Query(default=None),
item_id: str | None = Query(default=None),
status: str | None = Query(default="pending"),
) -> dict[str, Any]:
await _require_admin(request)
return {"reviews": await _store(request).list_reviews(job_id=job_id, item_id=item_id, status=status)}
@router.post("/reviews/actions")
async def review_actions(request: Request, body: ReviewRequest) -> dict[str, Any]:
actor = await _require_admin(request)
item_ids = await _store(request).review(body.review_ids, action=body.action, actor=actor, edits=body.edits, reason=body.reason)
resumed: list[str] = []
executor = request.app.state.skill_knowledge_executor
for item_id in item_ids:
if await _store(request).pending_review_count(item_id) == 0:
asyncio.create_task(executor.resume_reviewed_item(item_id))
resumed.append(item_id)
return {"updated": len(body.review_ids), "resumed_item_ids": resumed}
@@ -48,6 +48,14 @@ class PromptPrefixResponse(BaseModel):
)
class KnowledgeBaseProductVersionResponse(BaseModel):
version: Literal["v1", "v2"] = "v1"
class KnowledgeBaseProductVersionUpdateRequest(BaseModel):
version: Literal["v1", "v2"]
class PromptPrefixUpdateRequest(BaseModel):
enabled: bool | None = None
switch_label: str | None = None
@@ -175,6 +183,21 @@ async def update_prompt_prefix(body: PromptPrefixUpdateRequest, request: Request
return _to_response(saved.prompt_prefix)
@router.get("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
async def get_knowledge_base_product_version() -> KnowledgeBaseProductVersionResponse:
return KnowledgeBaseProductVersionResponse(version=load_system_settings().knowledge_base_product_version)
@router.put("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
async def update_knowledge_base_product_version(
body: KnowledgeBaseProductVersionUpdateRequest,
request: Request,
) -> KnowledgeBaseProductVersionResponse:
await _require_admin(request)
saved = save_system_settings(load_system_settings().model_copy(update={"knowledge_base_product_version": body.version}))
return KnowledgeBaseProductVersionResponse(version=saved.knowledge_base_product_version)
@router.get("/plugin-management", response_model=PluginManagementResponse)
async def get_plugin_management() -> PluginManagementResponse:
"""Return the iframe plugin domain -> agent mappings."""
@@ -405,6 +405,7 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
"reasoning_effort",
"rag_mode_enabled",
"llmwiki_knowledge_base_ids",
"assistant_knowledge_base_ids",
"writing_mode",
"writing_artifact_path",
"canvas_artifact_title",
@@ -1116,7 +1117,7 @@ async def start_run(
if prefix_settings.enabled and prefix_settings.prompt_prefix.strip():
graph_input = _prepend_prompt_prefix(graph_input, prefix_settings.prompt_prefix.strip())
llmwiki_rag = (
LlmWikiRagContext([], 0, "", [], "")
LlmWikiRagContext([], [], 0, "", [], "")
if fixed_answer is not None
else await build_llmwiki_rag_context(
request=request,
@@ -1128,12 +1129,14 @@ async def start_run(
graph_input = attach_llmwiki_rag_to_graph_input(graph_input, llmwiki_rag)
effective_rag_mode = True
run_metadata["llmwiki_knowledge_base_ids"] = llmwiki_rag.knowledge_base_ids
run_metadata["assistant_knowledge_base_ids"] = llmwiki_rag.assistant_knowledge_base_ids
run_metadata["llmwiki_rag_source_count"] = llmwiki_rag.source_count
try:
await run_ctx.thread_store.update_metadata(
thread_id,
{
"llmwiki_knowledge_base_ids": llmwiki_rag.knowledge_base_ids,
"assistant_knowledge_base_ids": llmwiki_rag.assistant_knowledge_base_ids,
"rag_mode_enabled": True,
"rag_mode": True,
},
@@ -1153,6 +1156,8 @@ async def start_run(
body_context["rag_mode_enabled"] = True
if "llmwiki_knowledge_base_ids" not in body_context and llmwiki_rag.knowledge_base_ids:
body_context["llmwiki_knowledge_base_ids"] = llmwiki_rag.knowledge_base_ids
if "assistant_knowledge_base_ids" not in body_context and llmwiki_rag.assistant_knowledge_base_ids:
body_context["assistant_knowledge_base_ids"] = llmwiki_rag.assistant_knowledge_base_ids
if not llmwiki_rag.knowledge_base_ids:
existing_excluded = list(body_context.get("excluded_tools") or getattr(body, "excluded_tools", None) or [])
if "llmwiki_search" not in existing_excluded:
@@ -0,0 +1,155 @@
"""Debounced file watcher and cron scheduler for opted-in knowledge bindings."""
from __future__ import annotations
import asyncio
import logging
from datetime import UTC, datetime
from typing import Any
from croniter import croniter
from app.gateway.skill_knowledge_job_executor import _skill_directory
from deerflow.config.system_settings import load_system_settings
from deerflow.skill_knowledge import scan_skill
logger = logging.getLogger(__name__)
class SkillKnowledgeAutoSyncService:
def __init__(self, app: Any, *, poll_seconds: float = 10.0) -> None:
self.app = app
self.store = app.state.skill_knowledge_store
self.poll_seconds = poll_seconds
self._task: asyncio.Task | None = None
self._wake = asyncio.Event()
self._observed: dict[str, tuple[str, datetime]] = {}
self._schedule_expression = ""
self._next_schedule: datetime | None = None
def start(self) -> None:
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._loop())
def nudge(self) -> None:
self._wake.set()
async def close(self) -> None:
if self._task is None:
return
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
async def _queue_skill(
self,
skill_name: str,
bindings: list[dict[str, Any]],
*,
trigger: str,
review_mode: str,
) -> None:
targets = [
{
"target_type": binding["target_type"],
"target_mode": binding["target_mode"],
"target_id": binding["target_id"],
"target_name": binding["target_name"],
"remote_business_key": binding.get("remote_business_key"),
}
for binding in bindings
]
job = await self.store.create_job(
skill_names=[skill_name],
targets=targets,
created_by=None,
trigger=trigger,
review_mode=review_mode,
)
if job["total_count"]:
self.app.state.skill_knowledge_dispatcher.nudge()
async def _watch(
self,
bindings_by_skill: dict[str, list[dict[str, Any]]],
*,
debounce_seconds: int,
write_remote_enabled: bool,
review_mode: str,
) -> None:
now = datetime.now(UTC)
for skill_name, bindings in bindings_by_skill.items():
try:
root = _skill_directory(self.app.state.config, skill_name)
scan = await asyncio.to_thread(scan_skill, skill_name, root)
except Exception:
logger.exception("Failed to fingerprint skill %s", skill_name)
continue
synced = {str(binding.get("synced_digest") or "") for binding in bindings}
if synced == {scan.source_digest}:
self._observed.pop(skill_name, None)
continue
await self.store.mark_skill_stale(skill_name)
previous = self._observed.get(skill_name)
if previous is None or previous[0] != scan.source_digest:
self._observed[skill_name] = (scan.source_digest, now)
continue
if (now - previous[1]).total_seconds() < debounce_seconds:
continue
if write_remote_enabled:
current = await self.store.list_bindings(skill_names=[skill_name])
stale = [binding for binding in current if binding.get("auto_sync_enabled") and binding.get("status") == "stale"]
if stale:
await self._queue_skill(
skill_name,
stale,
trigger="watcher",
review_mode=review_mode,
)
self._observed.pop(skill_name, None)
def _schedule_due(self, expression: str, now: datetime) -> bool:
if expression != self._schedule_expression or self._next_schedule is None:
self._schedule_expression = expression
self._next_schedule = croniter(expression, now).get_next(datetime)
return False
if now < self._next_schedule:
return False
self._next_schedule = croniter(expression, now).get_next(datetime)
return True
async def _tick(self) -> None:
settings = (await asyncio.to_thread(load_system_settings)).skill_knowledge_auto_sync
bindings = [row for row in await self.store.list_bindings() if row.get("auto_sync_enabled")]
by_skill: dict[str, list[dict[str, Any]]] = {}
for binding in bindings:
by_skill.setdefault(str(binding["skill_name"]), []).append(binding)
if settings.watch_enabled:
await self._watch(
by_skill,
debounce_seconds=settings.debounce_seconds,
write_remote_enabled=settings.write_remote_enabled,
review_mode=settings.review_mode,
)
if settings.schedule_enabled and settings.write_remote_enabled and by_skill and self._schedule_due(settings.schedule_cron, datetime.now(UTC)):
for skill_name, skill_bindings in by_skill.items():
await self._queue_skill(
skill_name,
skill_bindings,
trigger="scheduled",
review_mode=settings.review_mode,
)
async def _loop(self) -> None:
while True:
try:
await self._tick()
except Exception:
logger.exception("Skill knowledge automatic sync tick failed")
try:
await asyncio.wait_for(self._wake.wait(), timeout=self.poll_seconds)
except TimeoutError:
pass
self._wake.clear()
@@ -0,0 +1,96 @@
"""Small persistent-queue dispatcher for skill knowledge items."""
from __future__ import annotations
import asyncio
import logging
import socket
from typing import Any
from uuid import uuid4
logger = logging.getLogger(__name__)
class SkillKnowledgeJobDispatcher:
def __init__(
self,
store: Any,
executor: Any,
*,
interval_seconds: float = 2.0,
concurrency: int = 3,
lease_seconds: int = 900,
) -> None:
self.store = store
self.executor = executor
self.interval_seconds = interval_seconds
self.lease_seconds = lease_seconds
self.owner = f"{socket.gethostname()}:{uuid4()}"
self.semaphore = asyncio.Semaphore(concurrency)
self._wake = asyncio.Event()
self._task: asyncio.Task | None = None
self._running: set[str] = set()
def start(self) -> None:
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._loop())
def nudge(self) -> None:
self._wake.set()
async def close(self) -> None:
if self._task is not None:
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
async def _run(self, item: dict) -> None:
item_id = str(item["id"])
async with self.semaphore:
heartbeat = asyncio.create_task(self._heartbeat(item_id))
try:
await self.executor.run_item(item)
finally:
heartbeat.cancel()
try:
await heartbeat
except asyncio.CancelledError:
pass
self._running.discard(item_id)
self._wake.set()
async def _heartbeat(self, item_id: str) -> None:
while True:
await asyncio.sleep(min(30, max(5, self.lease_seconds // 3)))
if not await self.store.renew_lease(
item_id,
owner=self.owner,
lease_seconds=self.lease_seconds,
):
return
async def _loop(self) -> None:
while True:
try:
for item in await self.store.list_queued_items(limit=20):
item_id = str(item["id"])
if item_id in self._running:
continue
claimed = await self.store.claim_item(
item_id,
owner=self.owner,
lease_seconds=self.lease_seconds,
)
if claimed is None:
continue
self._running.add(item_id)
asyncio.create_task(self._run(claimed))
except Exception:
logger.exception("Skill knowledge dispatcher tick failed")
try:
await asyncio.wait_for(self._wake.wait(), timeout=self.interval_seconds)
except TimeoutError:
pass
self._wake.clear()
@@ -0,0 +1,403 @@
"""Background executor for one skill x target matrix item."""
from __future__ import annotations
import asyncio
import json
import logging
from pathlib import Path
from typing import Any
import httpx
from deerflow.assistant_knowledge import export_package_from_skill_ir
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.skill_knowledge import build_skill_ir, enrich_converted_documents, scan_skill
from deerflow.skill_knowledge.graph_writer import parameterized_statements, plan_from_ir
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.types import SkillCategory
logger = logging.getLogger(__name__)
def _skill_directory(app_config: Any, skill_name: str) -> Path:
storage = get_or_new_skill_storage(app_config=app_config)
root = storage.get_skills_root_path().resolve()
for category in SkillCategory:
direct = root / category.value / skill_name
if (direct / "SKILL.md").is_file():
return direct
for category in SkillCategory:
category_root = root / category.value
if not category_root.is_dir():
continue
for skill_md in category_root.rglob("SKILL.md"):
if skill_md.is_symlink():
continue
parsed = parse_skill_file(skill_md, category=category, relative_path=skill_md.parent.relative_to(category_root))
if parsed and parsed.name == skill_name:
return skill_md.parent
raise FileNotFoundError(f"Skill '{skill_name}' not found under running skill roots")
def _graph_projection(ir: dict[str, Any]) -> str:
rows = []
entities = {str(item.get("local_id")): item for item in ir.get("entities", [])}
for relation in ir.get("relations", []):
if float(relation.get("confidence") or 0) < 0.75 or not relation.get("evidence"):
continue
source = entities.get(str(relation.get("source_local_id")), {})
target = entities.get(str(relation.get("target_local_id")), {})
rows.append(
json.dumps(
{
"source": source.get("name"),
"source_type": source.get("type"),
"predicate": relation.get("predicate"),
"target": target.get("name"),
"target_type": target.get("type"),
"description": relation.get("description"),
"confidence": relation.get("confidence"),
"evidence": relation.get("evidence"),
},
ensure_ascii=False,
)
)
return "\n".join(rows)
class SkillKnowledgeJobExecutor:
def __init__(self, app: Any) -> None:
self.app = app
self.store = app.state.skill_knowledge_store
async def _cancel_if_requested(self, item: dict[str, Any]) -> bool:
job = await self.store.get_job(str(item["job_id"]))
if not (job or {}).get("cancel_requested"):
return False
await self.store.update_item(
str(item["id"]),
status="canceled",
phase="completed",
progress=100,
)
return True
async def _assert_source_unchanged(self, item: dict[str, Any], expected_digest: str) -> None:
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
current = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
current = await enrich_converted_documents(current)
if current.source_digest == expected_digest:
return
await self.store.mark_skill_stale(str(item["skill_name"]))
raise RuntimeError("技能知识文件在归纳期间发生变化,旧版本保持激活,请重试")
async def _write_direct_graph(
self,
item: dict[str, Any],
snapshot_id: str,
ir: dict[str, Any],
) -> dict[str, Any] | None:
config = self.app.state.config.llmwiki.weknora.graph
if not config.direct_write_enabled:
return None
if not config.neo4j_http_url or not config.username or not config.password:
raise RuntimeError("Neo4j direct writer 已开启但后端连接配置不完整")
plan = plan_from_ir(
ir,
target_type=str(item["target_type"]),
target_id=str(item["target_id"]),
binding_id=str(item["binding_id"]),
snapshot_id=snapshot_id,
job_id=str(item["job_id"]),
)
statements = parameterized_statements(plan)
if not statements:
return {"nodes": 0, "edges": 0, "statement_count": 0}
endpoint = f"{config.neo4j_http_url.rstrip('/')}/db/{config.database}/tx/commit"
async with httpx.AsyncClient(
auth=(config.username, config.password),
timeout=config.timeout_seconds,
) as client:
response = await client.post(endpoint, json={"statements": statements})
response.raise_for_status()
payload = response.json()
errors = payload.get("errors") or []
if errors:
raise RuntimeError(f"Neo4j transaction failed: {str(errors[0].get('code') or 'unknown')}")
return {
"nodes": len(plan.nodes),
"edges": len(plan.edges),
"statement_count": len(statements),
"scope": plan.scope,
}
async def run_item(self, item: dict[str, Any]) -> None:
item_id = str(item["id"])
try:
await self.store.update_item(item_id, status="running", phase="scanning", progress=5)
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
scan = await enrich_converted_documents(scan)
if await self._cancel_if_requested(item):
return
await self.store.update_item(item_id, phase="extracting", progress=30)
skill_record = await self.app.state.skill_store.get_any(str(item["skill_name"]))
ir = await asyncio.to_thread(
build_skill_ir,
scan,
display_name=(skill_record or {}).get("name_zh"),
description=str((skill_record or {}).get("description") or ""),
)
snapshot = await self.store.get_or_create_snapshot(
skill_name=str(item["skill_name"]),
source_digest=scan.source_digest,
extractor_version=str(ir["extractor_version"]),
prompt_hash=str(ir["prompt_hash"]),
model_name=str(ir["model_name"]),
manifest=scan.manifest,
ir=ir,
counts=scan.counts,
)
snapshot_id = str(snapshot["id"])
await self.store.update_item(item_id, phase="awaiting_review", progress=45, snapshot_id=snapshot_id)
job = await self.store.get_job(str(item["job_id"]))
review_mode = str((job or {}).get("review_mode") or "auto_high_confidence")
review_objects: list[tuple[str, dict]] = []
if review_mode == "required":
review_objects = [("entity", obj) for obj in ir.get("entities", [])] + [("relation", obj) for obj in ir.get("relations", [])]
elif review_mode == "auto_high_confidence":
review_objects = [(kind, obj) for kind, rows in (("entity", ir.get("entities", [])), ("relation", ir.get("relations", []))) for obj in rows if float(obj.get("confidence") or 0) < 0.9]
if review_objects:
existing = await self.store.list_reviews(item_id=item_id, status=None)
if not existing:
await self.store.create_reviews(
job_id=str(item["job_id"]),
item_id=item_id,
binding_id=str(item["binding_id"]),
snapshot_id=snapshot_id,
objects=review_objects,
)
await self.store.update_item(item_id, status="awaiting_review", phase="awaiting_review", progress=50)
return
await self._project_and_activate(item, snapshot_id, ir, scan)
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001
logger.exception("Skill knowledge item failed: item=%s", item_id)
await self.store.update_item(
item_id,
status="failed",
phase="completed",
progress=100,
error_code=type(exc).__name__,
error_message=str(exc)[:2000],
)
async def resume_reviewed_item(self, item_id: str) -> None:
if await self.store.pending_review_count(item_id):
return
jobs = await self.store.list_jobs(limit=100)
item = None
for job in jobs:
detail = await self.store.get_job(str(job["id"]))
item = next((row for row in (detail or {}).get("items", []) if row["id"] == item_id), None)
if item:
break
if not item or not item.get("snapshot_id"):
return
snapshot = await self.store.get_snapshot(str(item["snapshot_id"]))
ir = await self.store.effective_ir(str(item["snapshot_id"]), item_id)
if not snapshot or not ir:
return
try:
await self.store.update_item(item_id, status="running", phase="merging", progress=55)
await self._project_and_activate(item, str(item["snapshot_id"]), ir, None)
except Exception as exc: # noqa: BLE001
logger.exception("Reviewed skill knowledge item failed: item=%s", item_id)
await self.store.update_item(
item_id,
status="failed",
phase="completed",
progress=100,
error_code=type(exc).__name__,
error_message=str(exc)[:2000],
)
async def _project_and_activate(self, item: dict[str, Any], snapshot_id: str, ir: dict[str, Any], scan: Any | None) -> None:
if await self._cancel_if_requested(item):
return
expected_digest = str(ir.get("skill", {}).get("source_digest") or "")
if scan is None:
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
scan = await enrich_converted_documents(scan)
await self._assert_source_unchanged(item, expected_digest)
await self.store.update_item(str(item["id"]), phase="writing_wiki", progress=65)
target_type = str(item["target_type"])
artifacts: list[dict[str, Any]] = []
binding_status = "synced"
page = ir["wiki"]["skill_page"]
if target_type == "assistant":
package = export_package_from_skill_ir(ir)
await self.app.state.assistant_knowledge_store.import_package(
package=package,
source_type="skill",
source_key=str(item["skill_name"]),
source_name=str(ir["skill"].get("display_name") or item["skill_name"]),
trigger="skill_direct",
created_by=(await self.store.get_job(str(item["job_id"]))).get("created_by"),
skill_name=str(item["skill_name"]),
)
artifacts.append({"kind": "assistant_wiki_revision", "slug": page["slug"], "content": page["content"], "business_key": f"assistant:wiki:{page['slug']}"})
elif str(item["target_mode"]) == "readonly_sidecar":
artifacts.append({"kind": "sidecar_wiki", "slug": page["slug"], "content": page["content"], "business_key": f"sidecar:{item['target_id']}:{page['slug']}"})
artifacts.append({"kind": "sidecar_graph", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "content": _graph_projection(ir)})
binding_status = "pending_remote_write"
else:
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
if mapping is None:
raise ValueError("Target WeKnora mapping does not exist")
runtime = get_resolved_llmwiki_runtime(self.app.state.config)
if not runtime.weknora_enabled:
raise RuntimeError("普通知识库服务未配置")
client = build_weknora_client(runtime)
remote_kb_id = str(mapping["weknora_id"])
if scan is not None:
for file in scan.files:
source_path = Path(scan.source_root) / file.path
if file.text:
remote = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/source/{file.path}", content=file.text)
else:
remote = await client.upload_document(
remote_kb_id,
filename=f"{item['skill_name']}-{source_path.name}",
content=await asyncio.to_thread(source_path.read_bytes),
)
remote_id = str(remote.get("id") or remote.get("knowledge_id") or "") or None
artifacts.append(
{
"kind": "raw_source",
"slug": file.path,
"remote_id": remote_id,
"business_key": f"weknora:document:{remote_id}" if remote_id else None,
"remote_refs": [{"provider": "weknora", "remote_kind": "document", "remote_id": remote_id}] if remote_id else [],
}
)
target_mode = str(item["target_mode"])
if target_mode == "wiki":
remote_page: dict[str, Any]
try:
await client.get_wiki_page(remote_kb_id, page["slug"])
except WeKnoraError as exc:
if exc.status_code != 404:
raise
remote_page = await client.create_wiki_page(
remote_kb_id,
{
"slug": page["slug"],
"title": page["title"],
"content": page["content"],
"summary": str(ir["skill"].get("description") or ""),
"page_type": "skill",
},
)
else:
remote_page = await client.update_wiki_page(
remote_kb_id,
page["slug"],
{
"title": page["title"],
"content": page["content"],
"summary": str(ir["skill"].get("description") or ""),
},
)
remote_page_id = str(remote_page.get("id") or remote_page.get("page_id") or page["slug"])
artifacts.append(
{
"kind": "wiki_page",
"slug": page["slug"],
"content": page["content"],
"remote_id": remote_page_id,
"business_key": f"weknora:wiki_page:{remote_page_id}",
"remote_refs": [
{
"provider": "weknora",
"remote_kind": "wiki_page",
"remote_id": remote_page_id,
"slug": page["slug"],
}
],
}
)
else:
document_content = page["content"]
artifact_kind = "summary_document"
if target_mode == "faq":
artifact_kind = "faq_projection"
document_content = f"问题:{page['title']} 能解决什么问题,如何使用?\n\n回答:\n{page['content']}"
elif target_mode == "conversation_archive":
artifact_kind = "conversation_archive"
document_content = f"# 技能归纳会话归档:{page['title']}\n\n以下内容由技能知识归纳任务生成。\n\n{page['content']}"
remote_document = await client.create_manual_document(
remote_kb_id,
title=f"skills/{item['skill_name']}/{target_mode}.md",
content=document_content,
)
remote_document_id = str(remote_document.get("id") or remote_document.get("knowledge_id") or "") or None
artifacts.append(
{
"kind": artifact_kind,
"slug": f"skills/{item['skill_name']}/{target_mode}.md",
"content": document_content,
"remote_id": remote_document_id,
"business_key": (f"weknora:document:{remote_document_id}" if remote_document_id else None),
"remote_refs": (
[
{
"provider": "weknora",
"remote_kind": "document",
"remote_id": remote_document_id,
}
]
if remote_document_id
else []
),
}
)
projection = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/graph-projection.jsonl", content=_graph_projection(ir))
projection_id = str(projection.get("id") or projection.get("knowledge_id") or "") or None
artifacts.append({"kind": "graph_projection", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "remote_id": projection_id})
try:
graph_audit = await self._write_direct_graph(item, snapshot_id, ir)
if graph_audit is not None:
artifacts.append(
{
"kind": "neo4j_direct_write_audit",
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
"content": json.dumps(graph_audit, ensure_ascii=False),
"metadata": graph_audit,
}
)
except Exception as exc: # noqa: BLE001
logger.exception("Optional Neo4j direct graph write failed: item=%s", item["id"])
binding_status = "partial_failed"
artifacts.append(
{
"kind": "neo4j_direct_write_audit",
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
"content": json.dumps(
{"status": "failed", "error": type(exc).__name__},
ensure_ascii=False,
),
"metadata": {
"status": "failed",
"error_type": type(exc).__name__,
},
}
)
await self._assert_source_unchanged(item, expected_digest)
await self.store.update_item(str(item["id"]), phase="activating", progress=90)
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status=binding_status)
+18 -5
View File
@@ -448,7 +448,11 @@ auth_login:
database:
backend: sqlite
sqlite_dir: .deer-flow/data
pool_size: 20 # 基础连接池(postgres/mysql 生效;sqlite 忽略)
# 高斯/OpenGauss 业务库示例(启用前先执行 uv sync --extra gauss):
# backend: gauss
# gauss_url: $GAUSS_DATABASE_URL # opengauss+asyncpg://user:pass@host:26000/deerflow
# checkpointer 请继续使用下方 sqlite;不要把 PostgreSQL 专用 checkpointer 迁移直接用于高斯。
pool_size: 20 # 基础连接池(postgres/gauss/mysql 生效;sqlite 忽略)
max_overflow: 20 # 高峰期可临时超出 pool_size 的额外连接数,真实上限 = pool_size + max_overflow
pool_timeout: 30 # 池满时等待空闲连接的秒数,超时报错
pool_recycle: 1800 # 连接回收周期(秒);防火墙/pgbouncer/MySQL wait_timeout 静默断连兜底。多 worker 部署:每 worker 上限 = pool_size + max_overflow,总数 × worker 数 < DB max_connections
@@ -575,9 +579,18 @@ llmwiki:
web_base_url: "http://119.254.86.251:8088"
admin_email: "lqq@qq.com"
admin_password: "1qaz@WSX"
# 本地 Wiki 向量化:只需把 enabled 改为 true,并填写下面 3 个 embedding 值。
# 默认只在页面点击“向量化”时执行;auto_sync=true 才会后台自动增量同步,
# 且服务启动时只启动定时器,不会立刻全量向量化。
# 高风险能力,默认关闭。连接参数仅允许后端配置;生产环境请使用环境变量。
graph:
direct_write_enabled: false
neo4j_http_url: ""
username: ""
password: ""
database: "neo4j"
timeout_seconds: 30
# 本地 Wiki 向量化:用于普通知识库“下载向量化数据包”和助手知识库向量检索。
# 已有相同内容哈希 + 相同 embedding fingerprint 的 Wiki 向量会直接复用,不会重复编码。
# 如需导出时自动补建向量,把 enabled 改为 true,并填写下面 embedding 服务地址。
# auto_sync=true 才会后台自动增量同步;服务启动时只启动定时器,不会立刻全量向量化。
local_wiki_index:
enabled: false
# 免登录 POST /api/knowledge/vector-search 返回的 Wiki 一键跳转链接前缀。
@@ -585,7 +598,7 @@ llmwiki:
auto_sync: false
database_batch_size: 50
embedding:
model: bge-m3
model: bge-embedding-m3
base_url: ""
api_key: ""
@@ -0,0 +1,5 @@
"""Assistant knowledge package helpers."""
from deerflow.assistant_knowledge.importer import export_package_from_skill_ir
__all__ = ["export_package_from_skill_ir"]
@@ -0,0 +1,22 @@
"""Normalize heterogeneous sources into the assistant import-package shape."""
from __future__ import annotations
from typing import Any
def export_package_from_skill_ir(ir: dict[str, Any]) -> dict[str, Any]:
skill = ir.get("skill") or {}
return {
"source": {
"type": "skill",
"skill_name": skill.get("name"),
"source_digest": skill.get("source_digest"),
},
"wiki_pages": [ir.get("wiki", {}).get("skill_page")],
"documents": ir.get("files") or [],
"chunks": [],
"entities": ir.get("entities") or [],
"relations": ir.get("relations") or [],
"graph": {"nodes": ir.get("entities") or [], "edges": ir.get("relations") or []},
}
@@ -0,0 +1,109 @@
"""File-backed assistant knowledge import/export packages.
The assistant knowledge base is fed by WeKnora-exported data. Keep that
boundary explicit: a WeKnora export is first materialized as a durable JSONL
package, then imported asynchronously from that package. JSONL keeps large
exports append-friendly and avoids treating the HTTP request body as the job
state.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import Any
PACKAGE_VERSION = 1
PACKAGE_RECORD_TYPES = {
"source",
"wiki_page",
"document",
"chunk",
"vector",
"entity",
"relation",
"graph",
}
def package_path(root: Path, job_id: str) -> Path:
safe = "".join(ch for ch in str(job_id) if ch.isalnum() or ch in {"-", "_"})
return root / f"{safe}.jsonl"
def write_jsonl_record(handle: Any, record_type: str, payload: Mapping[str, Any]) -> None:
handle.write(json.dumps({"record_type": record_type, "payload": dict(payload)}, ensure_ascii=False))
handle.write("\n")
def write_package(path: Path, records: Iterable[tuple[str, Mapping[str, Any]]]) -> dict[str, int]:
path.parent.mkdir(parents=True, exist_ok=True)
counts: dict[str, int] = {key: 0 for key in PACKAGE_RECORD_TYPES}
tmp_path = path.with_suffix(path.suffix + ".tmp")
with tmp_path.open("w", encoding="utf-8", newline="\n") as handle:
write_jsonl_record(
handle,
"source",
{
"package_version": PACKAGE_VERSION,
},
)
counts["source"] += 1
for record_type, payload in records:
if record_type not in PACKAGE_RECORD_TYPES:
continue
write_jsonl_record(handle, record_type, payload)
counts[record_type] = counts.get(record_type, 0) + 1
tmp_path.replace(path)
return counts
def read_package(path: Path) -> dict[str, Any]:
package: dict[str, Any] = {
"source": {},
"wiki_pages": [],
"documents": [],
"chunks": [],
"vectors": [],
"entities": [],
"relations": [],
"graph": {"nodes": [], "edges": []},
}
with path.open("r", encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line:
continue
record = json.loads(line)
if not isinstance(record, dict):
continue
record_type = str(record.get("record_type") or "")
payload = record.get("payload")
if not isinstance(payload, dict):
continue
if record_type == "source":
package["source"] = {**package.get("source", {}), **payload}
elif record_type == "wiki_page":
package["wiki_pages"].append(payload)
elif record_type == "document":
package["documents"].append(payload)
elif record_type == "chunk":
package["chunks"].append(payload)
elif record_type == "vector":
package["vectors"].append(payload)
elif record_type == "entity":
package["entities"].append(payload)
elif record_type == "relation":
package["relations"].append(payload)
elif record_type == "graph":
nodes = payload.get("nodes")
edges = payload.get("edges")
if isinstance(nodes, list):
package["graph"]["nodes"].extend(item for item in nodes if isinstance(item, dict))
if isinstance(edges, list):
package["graph"]["edges"].extend(item for item in edges if isinstance(item, dict))
for key, value in payload.items():
if key not in {"nodes", "edges"}:
package["graph"][key] = value
return package
@@ -14,6 +14,11 @@ busy timeout rather than failing immediately.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
Gauss mode: application ORM data uses the official openGauss SQLAlchemy
dialect over asyncpg. LangGraph's PostgreSQL checkpointer is intentionally not
reused for GaussDB; configure the legacy ``checkpointer:`` section separately
(normally sqlite).
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
not have a built-in MySQL backend in this project, so deployments that choose
MySQL for business tables should configure the legacy ``checkpointer:``
@@ -43,9 +48,15 @@ from pydantic import BaseModel, Field
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
backend: Literal["memory", "sqlite", "postgres", "gauss", "mysql"] = Field(
default="memory",
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
description=(
"Storage backend for application data. 'memory' for development "
"(no persistence across restarts), 'sqlite' for single-node "
"deployment, 'postgres' for production multi-node deployment, "
"'gauss' for openGauss/GaussDB business tables, and 'mysql' for "
"MySQL-backed business tables."
),
)
sqlite_dir: str = Field(
default=".deer-flow/data",
@@ -60,6 +71,16 @@ class DatabaseConfig(BaseModel):
"(the +asyncpg driver suffix is added automatically where needed)."
),
)
gauss_url: str = Field(
default="",
description=(
"openGauss/GaussDB connection URL for the application ORM. "
"Use $GAUSS_DATABASE_URL in config.yaml to reference .env. "
"Example: opengauss+asyncpg://user:pass@host:26000/deerflow. "
"Plain opengauss:// or PostgreSQL-style URLs are normalized to the "
"official asynchronous openGauss SQLAlchemy dialect."
),
)
mysql_url: str = Field(
default="",
description=(
@@ -75,26 +96,26 @@ class DatabaseConfig(BaseModel):
)
pool_size: int = Field(
default=5,
description="Connection pool size for the app ORM engine (postgres/mysql only).",
description="Connection pool size for the app ORM engine (postgres/gauss/mysql only).",
)
max_overflow: int = Field(
default=10,
description=(
"Extra connections allowed beyond pool_size under load (postgres/mysql only). "
"Extra connections allowed beyond pool_size under load (postgres/gauss/mysql only). "
"The real connection ceiling is pool_size + max_overflow. SQLAlchemy default is 10."
),
)
pool_timeout: int = Field(
default=30,
description=(
"Seconds a caller waits for a free pooled connection before erroring (postgres/mysql only). "
"Seconds a caller waits for a free pooled connection before erroring (postgres/gauss/mysql only). "
"SQLAlchemy default is 30."
),
)
pool_recycle: int = Field(
default=1800,
description=(
"Recycle pooled connections after this many seconds (postgres/mysql only). "
"Recycle pooled connections after this many seconds (postgres/gauss/mysql only). "
"Prevents stale-connection errors when a firewall, pgbouncer, or MySQL "
"wait_timeout silently drops idle connections. pool_pre_ping catches most "
"cases, but pool_recycle is the belt-and-suspenders safety net. "
@@ -138,6 +159,32 @@ class DatabaseConfig(BaseModel):
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "gauss":
url = self.gauss_url.strip()
if not url:
raise ValueError("database.gauss_url is required for backend='gauss'")
# GaussDB speaks the PostgreSQL wire protocol, but its server
# version string and DDL capabilities are not identical to
# PostgreSQL. Always select the official openGauss dialect instead
# of silently sending the URL through SQLAlchemy's generic PG
# dialect (the latter commonly connects and then fails while
# SQLAlchemy initializes/creates the schema).
for prefix in (
"opengauss://",
"gaussdb://",
"postgres://",
"postgresql://",
"postgresql+asyncpg://",
):
if url.startswith(prefix):
return "opengauss+asyncpg://" + url[len(prefix) :]
if url.startswith("opengauss+asyncpg://"):
return url
raise ValueError(
"database.gauss_url must use opengauss://, "
"opengauss+asyncpg://, or a PostgreSQL-style URL"
)
if self.backend == "mysql":
url = self.mysql_url
if not url:
@@ -44,6 +44,7 @@ class WeKnoraConnectionConfig(BaseModel):
repr=False,
description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
)
graph: WeKnoraDirectGraphConfig = Field(default_factory=lambda: WeKnoraDirectGraphConfig())
model_config = ConfigDict(extra="ignore")
@field_validator("api_base_url", "web_base_url")
@@ -52,10 +53,27 @@ class WeKnoraConnectionConfig(BaseModel):
return normalize_service_url(value)
class WeKnoraDirectGraphConfig(BaseModel):
"""Backend-only, opt-in Neo4j transactional HTTP writer settings."""
direct_write_enabled: bool = False
neo4j_http_url: str = ""
username: str = ""
password: str = Field(default="", repr=False)
database: str = Field(default="neo4j", pattern=r"^[A-Za-z0-9_.-]+$")
timeout_seconds: float = Field(default=30, gt=0, le=300)
model_config = ConfigDict(extra="ignore")
@field_validator("neo4j_http_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
class LocalWikiEmbeddingConfig(BaseModel):
enabled: bool = True
provider: str = "openai"
model: str = "bge-m3"
model: str = "bge-embedding-m3"
base_url: str = ""
api_key: str = Field(default="", repr=False)
dimensions: int | None = Field(
@@ -160,6 +160,18 @@ class PromptPrefixSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class SkillKnowledgeAutoSyncSettings(BaseModel):
"""Watcher/scheduler policy for bindings that explicitly opt in."""
watch_enabled: bool = True
schedule_enabled: bool = True
schedule_cron: str = Field(default="0 */6 * * *", min_length=5, max_length=128)
debounce_seconds: int = Field(default=120, ge=10, le=86_400)
write_remote_enabled: bool = True
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
model_config = ConfigDict(extra="ignore")
class AppearanceRouteParam(BaseModel):
"""Runtime query-param descriptor for top-bar shortcut buttons.
@@ -509,6 +521,11 @@ class SystemSettings(BaseModel):
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
skill_knowledge_auto_sync: SkillKnowledgeAutoSyncSettings = Field(default_factory=SkillKnowledgeAutoSyncSettings)
knowledge_base_product_version: Literal["v1", "v2"] = Field(
default="v1",
description="v1 opens the ordinary WeKnora experience; v2 opens the DeerFlow assistant knowledge base.",
)
model_config = ConfigDict(extra="ignore")
@@ -494,7 +494,7 @@ class WeKnoraClient:
await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/graph",
params={"mode": "overview", "limit": max(1, min(limit, 500))},
params={"mode": "overview", "limit": max(1, min(limit, 10000))},
)
)
if not isinstance(value, dict):
@@ -1,7 +1,37 @@
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from __future__ import annotations
__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from deerflow.integrations.weknora.local_index.retrieval import (
WikiRetrievalService,
is_fully_vectorized,
)
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
__all__ = [
"WikiIndexError",
"WikiRetrievalService",
"WikiVectorSearchService",
"WikiSyncService",
"is_fully_vectorized",
]
def __getattr__(name: str) -> Any:
if name in {"WikiRetrievalService", "is_fully_vectorized"}:
from deerflow.integrations.weknora.local_index import retrieval
return getattr(retrieval, name)
if name in {"WikiIndexError", "WikiVectorSearchService"}:
from deerflow.integrations.weknora.local_index import search
return getattr(search, name)
if name == "WikiSyncService":
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
return WikiSyncService
raise AttributeError(name)
@@ -0,0 +1,4 @@
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
@@ -0,0 +1,174 @@
"""ORM model for DeerFlow-native assistant knowledge bases."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Index, Integer, LargeBinary, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class AssistantKnowledgeBaseRow(Base):
__tablename__ = "assistant_knowledge_base"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
description: Mapped[str | None] = mapped_column(String(2048), nullable=True)
# ``global`` is the full-library aggregation base. ``custom`` bases are
# user-created assistant knowledge bases fed from WeKnora export packages.
# Nullable keeps startup column reconciliation safe for existing databases.
base_kind: Mapped[str | None] = mapped_column(String(24), nullable=True, index=True)
status: Mapped[str] = mapped_column(String(64), default="active")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeSourceRow(Base):
__tablename__ = "assistant_knowledge_sources"
__table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_type: Mapped[str] = mapped_column(String(32), index=True)
source_key: Mapped[str] = mapped_column(String(255))
knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_name: Mapped[str] = mapped_column(String(255), default="")
last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeImportJobRow(Base):
__tablename__ = "assistant_knowledge_import_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="exporting")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeWikiPageRow(Base):
__tablename__ = "assistant_knowledge_wiki_pages"
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
slug: Mapped[str] = mapped_column(String(512), index=True)
title: Mapped[str] = mapped_column(String(512), index=True)
page_type: Mapped[str] = mapped_column(String(32), index=True)
current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeWikiRevisionRow(Base):
__tablename__ = "assistant_knowledge_wiki_revisions"
__table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
revision_no: Mapped[int] = mapped_column(Integer)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
change_type: Mapped[str] = mapped_column(String(32), default="import")
status: Mapped[str] = mapped_column(String(24), default="published")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeVectorRow(Base):
__tablename__ = "assistant_knowledge_vectors"
__table_args__ = (
Index("ix_assistant_knowledge_vector_base_fingerprint", "base_id", "embedding_fingerprint"),
Index("ix_assistant_knowledge_vector_page", "page_id"),
Index("ix_assistant_knowledge_vector_source", "source_id"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_vector_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
section_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_model: Mapped[str | None] = mapped_column(String(255), nullable=True)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class AssistantKnowledgeEntityRow(Base):
__tablename__ = "assistant_knowledge_entities"
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512), index=True)
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeEntityContributionRow(Base):
__tablename__ = "assistant_knowledge_entity_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class AssistantKnowledgeRelationRow(Base):
__tablename__ = "assistant_knowledge_relations"
__table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeRelationContributionRow(Base):
__tablename__ = "assistant_knowledge_relation_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -176,7 +176,7 @@ async def _auto_create_postgres_db(url: str) -> None:
try:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL database: %s", db_name)
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
finally:
await maint_engine.dispose()
@@ -230,10 +230,10 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
added column are created as well. Table-level ``UniqueConstraint`` objects
that are declared on the model but missing from the table are also created
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
emit too, and works on SQLite/MySQL/PostgreSQL alike) — without this a newly
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
on an existing database, weakening dedupe.
Works on sqlite/mysql/postgres — the column DDL is compiled through the
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
``BeijingDateTime``) resolve to the right backend type. Everything is
per-column best-effort: a failure is logged and startup continues.
@@ -355,13 +355,13 @@ async def init_engine(
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", "postgres", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
echo: Echo SQL to log.
pool_size: Base connection pool size (postgres/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
pool_recycle: Recycle connections after N seconds (postgres/mysql). Prevents stale
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
@@ -379,11 +379,21 @@ async def init_engine(
except RuntimeError:
_engine_loop = None
if backend == "postgres":
if backend in {"postgres", "gauss"}:
try:
import asyncpg # noqa: F401
except ImportError:
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
@@ -423,7 +433,7 @@ async def init_engine(
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend == "postgres":
elif backend in {"postgres", "gauss"}:
_engine = create_async_engine(
url,
echo=echo,
@@ -468,7 +478,7 @@ async def init_engine(
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend == "postgres" and "does not exist" in str(exc):
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database
@@ -511,7 +521,7 @@ async def init_engine(
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
logger.info("Persistence engine initialized: backend=%s", backend)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
async def init_engine_from_config(config) -> None:
@@ -3,6 +3,7 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from typing import Any
@@ -35,6 +36,23 @@ class LlmWikiIndexStore(ABC):
@abstractmethod
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
async def iter_vector_snapshot_rows(
self,
mapping_id: str,
fingerprint: str,
*,
batch_size: int = 1000,
) -> AsyncIterator[dict[str, Any]]:
"""Iterate vector rows for streaming export.
Implementations with a database backend should override this method so
very large vector packages do not need to be materialized in memory.
"""
snapshot = await self.load_vector_snapshot(mapping_id, fingerprint)
for row in snapshot.get("rows") or []:
if isinstance(row, dict):
yield row
@abstractmethod
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
@@ -2,6 +2,7 @@
from __future__ import annotations
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
@@ -181,6 +182,41 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
async def iter_vector_snapshot_rows(
self,
mapping_id: str,
fingerprint: str,
*,
batch_size: int = 1000,
) -> AsyncIterator[dict[str, Any]]:
batch = max(1, min(int(batch_size), 5000))
offset = 0
while True:
stmt = (
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
.where(
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
LlmWikiWikiPageRow.index_status == "ready",
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
.limit(batch)
.offset(offset)
)
async with self._sf() as session:
pairs = list((await session.execute(stmt)).all())
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
if not rows:
break
for row in rows:
yield row
if len(rows) < batch:
break
offset += len(rows)
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
@@ -0,0 +1,73 @@
"""Add skill distillation and assistant knowledge tables.
Revision ID: 20260901_01
Revises: 20260825_01
Create Date: 2026-09-01
"""
from __future__ import annotations
from alembic import op
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
revision = "20260901_01"
down_revision = "20260825_01"
branch_labels = None
depends_on = None
TABLES = [
SkillKnowledgeBindingRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncJobRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeArtifactRow,
SkillKnowledgeReviewRow,
SkillKnowledgeEntityRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeRelationContributionRow,
AssistantKnowledgeBaseRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeRelationContributionRow,
]
def upgrade() -> None:
bind = op.get_bind()
for model in TABLES:
model.__table__.create(bind, checkfirst=True)
def downgrade() -> None:
bind = op.get_bind()
for model in reversed(TABLES):
model.__table__.drop(bind, checkfirst=True)
@@ -0,0 +1,25 @@
"""Add assistant knowledge vector table.
Revision ID: 20260902_01
Revises: 20260901_01
Create Date: 2026-09-02
"""
from __future__ import annotations
from alembic import op
from deerflow.persistence.assistant_knowledge.model import AssistantKnowledgeVectorRow
revision = "20260902_01"
down_revision = "20260901_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
AssistantKnowledgeVectorRow.__table__.create(op.get_bind(), checkfirst=True)
def downgrade() -> None:
AssistantKnowledgeVectorRow.__table__.drop(op.get_bind(), checkfirst=True)
@@ -35,6 +35,18 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeVectorRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
@@ -98,6 +110,18 @@ from deerflow.persistence.scheduled_tasks.model import (
SchedulerThreadRow,
)
from deerflow.persistence.sensitive_words.model import SensitiveWordRow
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.task_buttons.model import TaskButtonRow
@@ -119,6 +143,15 @@ __all__ = [
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
"AssistantKnowledgeBaseRow",
"AssistantKnowledgeSourceRow",
"AssistantKnowledgeImportJobRow",
"AssistantKnowledgeWikiPageRow",
"AssistantKnowledgeWikiRevisionRow",
"AssistantKnowledgeEntityRow",
"AssistantKnowledgeEntityContributionRow",
"AssistantKnowledgeRelationRow",
"AssistantKnowledgeRelationContributionRow",
"CompactMenuOverrideRow",
"ConcurrencySampleRow",
"DashboardSessionRow",
@@ -185,6 +218,16 @@ __all__ = [
"SkillRow",
"SkillDisplayAdapterRow",
"SkillFavoriteRow",
"SkillKnowledgeArtifactRow",
"SkillKnowledgeBindingRow",
"SkillKnowledgeEntityContributionRow",
"SkillKnowledgeEntityRow",
"SkillKnowledgeRelationContributionRow",
"SkillKnowledgeRelationRow",
"SkillKnowledgeReviewRow",
"SkillKnowledgeSnapshotRow",
"SkillKnowledgeSyncItemRow",
"SkillKnowledgeSyncJobRow",
"TagAssignmentRow",
"TagRow",
"TaskButtonRow",
@@ -0,0 +1,15 @@
from deerflow.persistence.skill_knowledge.model import ( # noqa: F401
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skill_knowledge.store import SkillKnowledgeRepository, make_skill_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("SkillKnowledge") or name == "make_skill_knowledge_store"]
@@ -0,0 +1,203 @@
"""ORM rows for skill-to-knowledge distillation and review."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Float, Integer, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class SkillKnowledgeBindingRow(Base):
__tablename__ = "skill_knowledge_bindings"
__table_args__ = (UniqueConstraint("skill_name", "target_type", "target_id", name="uq_skill_knowledge_binding"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
skill_name: Mapped[str] = mapped_column(String(255), index=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_mode: Mapped[str] = mapped_column(String(32), default="wiki")
target_id: Mapped[str] = mapped_column(String(128), index=True)
target_name: Mapped[str] = mapped_column(String(255), default="")
remote_business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(48), default="pending", index=True)
active: Mapped[bool] = mapped_column(Boolean, default=True, server_default=true(), index=True)
auto_sync_enabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false(), index=True)
current_snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
synced_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
last_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
last_item_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
last_synced_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeSnapshotRow(Base):
__tablename__ = "skill_knowledge_snapshots"
__table_args__ = (UniqueConstraint("skill_name", "source_digest", "extractor_version", "prompt_hash", "model_name", name="uq_skill_knowledge_snapshot"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
skill_name: Mapped[str] = mapped_column(String(255), index=True)
source_digest: Mapped[str] = mapped_column(String(80), index=True)
extractor_version: Mapped[str] = mapped_column(String(64))
prompt_hash: Mapped[str] = mapped_column(String(80))
model_name: Mapped[str] = mapped_column(String(128))
manifest_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
ir_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
status: Mapped[str] = mapped_column(String(24), default="ready")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class SkillKnowledgeSyncJobRow(Base):
__tablename__ = "skill_knowledge_sync_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
trigger: Mapped[str] = mapped_column(String(32), default="manual")
target_count: Mapped[int] = mapped_column(Integer, default=0)
targets_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
total_count: Mapped[int] = mapped_column(Integer, default=0)
success_count: Mapped[int] = mapped_column(Integer, default=0)
partial_failed_count: Mapped[int] = mapped_column(Integer, default=0)
failed_count: Mapped[int] = mapped_column(Integer, default=0)
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
review_mode: Mapped[str] = mapped_column(String(32), default="auto_high_confidence")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class SkillKnowledgeSyncItemRow(Base):
__tablename__ = "skill_knowledge_sync_items"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
job_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
target_type: Mapped[str] = mapped_column(String(32))
target_mode: Mapped[str] = mapped_column(String(32))
target_id: Mapped[str] = mapped_column(String(128))
skill_name: Mapped[str] = mapped_column(String(255), index=True)
snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="scanning")
progress: Mapped[int] = mapped_column(Integer, default=0)
attempts: Mapped[int] = mapped_column(Integer, default=0)
active_dedupe_key: Mapped[str | None] = mapped_column(String(36), nullable=True, unique=True)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class SkillKnowledgeArtifactRow(Base):
__tablename__ = "skill_knowledge_artifacts"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
kind: Mapped[str] = mapped_column(String(48), index=True)
slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
content_text: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
remote_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="staged", index=True)
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeReviewRow(Base):
__tablename__ = "skill_knowledge_reviews"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
job_id: Mapped[str] = mapped_column(String(36), index=True)
item_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
object_type: Mapped[str] = mapped_column(String(32), index=True)
object_key: Mapped[str] = mapped_column(String(128))
original_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
edited_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="pending", index=True)
reason: Mapped[str | None] = mapped_column(String(1024), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class SkillKnowledgeEntityRow(Base):
__tablename__ = "skill_knowledge_entities"
__table_args__ = (UniqueConstraint("target_type", "target_id", "entity_type", "normalized_name", name="uq_skill_knowledge_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_id: Mapped[str] = mapped_column(String(128), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512))
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeEntityContributionRow(Base):
__tablename__ = "skill_knowledge_entity_contributions"
__table_args__ = (UniqueConstraint("entity_id", "binding_id", "snapshot_id", "local_entity_id", name="uq_skill_entity_contribution"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
local_entity_id: Mapped[str] = mapped_column(String(128))
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class SkillKnowledgeRelationRow(Base):
__tablename__ = "skill_knowledge_relations"
__table_args__ = (UniqueConstraint("target_type", "target_id", "source_entity_id", "predicate", "target_entity_id", name="uq_skill_knowledge_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_id: Mapped[str] = mapped_column(String(128), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeRelationContributionRow(Base):
__tablename__ = "skill_knowledge_relation_contributions"
__table_args__ = (UniqueConstraint("relation_id", "binding_id", "snapshot_id", "local_relation_id", name="uq_skill_relation_contribution"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
local_relation_id: Mapped[str] = mapped_column(String(128))
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -0,0 +1,758 @@
"""SQL repository for skill knowledge jobs, snapshots and contributions."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row: Any) -> dict[str, Any]:
list_json_fields = {
"aliases_json",
"evidence_json",
"manifest_json",
"refs_json",
"targets_json",
}
data = row.to_dict()
for key, value in tuple(data.items()):
if isinstance(value, datetime):
data[key] = value.isoformat()
elif key.endswith("_json"):
data[key[:-5]] = data.pop(key) or ([] if key in list_json_fields else {})
return data
class SkillKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def _retire_target_orphans(self, session: AsyncSession, *, target_type: str, target_id: str) -> None:
entities = (
await session.execute(
select(SkillKnowledgeEntityRow).where(
SkillKnowledgeEntityRow.target_type == target_type,
SkillKnowledgeEntityRow.target_id == target_id,
)
)
).scalars()
for entity in entities:
contribution = (await session.execute(select(SkillKnowledgeEntityContributionRow.id).where(SkillKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
entity.status = "active" if contribution else "deleted"
relations = (
await session.execute(
select(SkillKnowledgeRelationRow).where(
SkillKnowledgeRelationRow.target_type == target_type,
SkillKnowledgeRelationRow.target_id == target_id,
)
)
).scalars()
for relation in relations:
contribution = (await session.execute(select(SkillKnowledgeRelationContributionRow.id).where(SkillKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
relation.status = "active" if contribution else "deleted"
async def create_job(
self,
*,
skill_names: list[str],
targets: list[dict[str, Any]],
created_by: str | None,
force: bool = False,
trigger: str = "manual",
review_mode: str = "auto_high_confidence",
) -> dict[str, Any]:
names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
normalized_targets: list[dict[str, Any]] = []
seen_targets: set[tuple[str, str]] = set()
for target in targets:
target_type = str(target.get("target_type") or "").strip()
target_id = str(target.get("target_id") or "").strip()
if not target_type or not target_id or (target_type, target_id) in seen_targets:
continue
seen_targets.add((target_type, target_id))
normalized_targets.append(
{
"target_type": target_type,
"target_mode": str(target.get("target_mode") or ("wiki" if target_type == "weknora" else "assistant")),
"target_id": target_id,
"target_name": str(target.get("target_name") or target_id),
"remote_business_key": target.get("remote_business_key"),
"readonly": bool(target.get("readonly", False)),
}
)
job_id = str(uuid4())
now = _now()
async with self._sf() as session:
job = SkillKnowledgeSyncJobRow(
id=job_id,
trigger=trigger,
target_count=len(normalized_targets),
targets_json=normalized_targets,
status="queued",
total_count=0,
review_mode=review_mode,
created_by=created_by,
created_at=now,
)
session.add(job)
for skill_name in names:
for target in normalized_targets:
stmt = select(SkillKnowledgeBindingRow).where(
SkillKnowledgeBindingRow.skill_name == skill_name,
SkillKnowledgeBindingRow.target_type == target["target_type"],
SkillKnowledgeBindingRow.target_id == target["target_id"],
)
binding = (await session.execute(stmt)).scalar_one_or_none()
if binding is None:
binding = SkillKnowledgeBindingRow(
id=str(uuid4()),
skill_name=skill_name,
target_type=target["target_type"],
target_mode=target["target_mode"],
target_id=target["target_id"],
target_name=target["target_name"],
remote_business_key=target.get("remote_business_key"),
status="queued",
created_by=created_by,
created_at=now,
updated_at=now,
)
session.add(binding)
else:
active_item = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == binding.id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
if active_item is not None:
continue
binding.target_mode = target["target_mode"]
binding.target_name = target["target_name"]
binding.remote_business_key = target.get("remote_business_key")
binding.status = "queued"
binding.active = True
binding.updated_at = now
item = SkillKnowledgeSyncItemRow(
id=str(uuid4()),
job_id=job_id,
binding_id=binding.id,
target_type=target["target_type"],
target_mode=target["target_mode"],
target_id=target["target_id"],
skill_name=skill_name,
status="queued",
phase="scanning",
progress=0,
active_dedupe_key=binding.id,
created_at=now,
updated_at=now,
)
session.add(item)
binding.last_job_id = job_id
binding.last_item_id = item.id
job.total_count += 1
if job.total_count == 0:
job.status = "completed"
job.completed_at = now
await session.flush()
result = _dict(job)
result["force"] = force
await session.commit()
return result
async def list_bindings(self, *, skill_names: list[str] | None = None, target_id: str | None = None, include_detached: bool = False) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeBindingRow)
if skill_names:
stmt = stmt.where(SkillKnowledgeBindingRow.skill_name.in_(skill_names))
if target_id:
stmt = stmt.where(SkillKnowledgeBindingRow.target_id == target_id)
if not include_detached:
stmt = stmt.where(SkillKnowledgeBindingRow.active.is_(True))
stmt = stmt.order_by(SkillKnowledgeBindingRow.updated_at.desc())
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_binding(self, binding_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeBindingRow, binding_id)
return _dict(row) if row else None
async def list_artifacts(self, binding_id: str, *, active_only: bool = True) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeArtifactRow).where(SkillKnowledgeArtifactRow.binding_id == binding_id)
if active_only:
stmt = stmt.where(SkillKnowledgeArtifactRow.status == "active")
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def has_active_item(self, binding_id: str) -> bool:
async with self._sf() as session:
value = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == binding_id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
return value is not None
async def set_binding_auto_sync(self, binding_id: str, *, enabled: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeBindingRow, binding_id)
if row is None or not row.active:
return None
row.auto_sync_enabled = enabled
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
return data
async def get_job(self, job_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return None
items = (await session.execute(select(SkillKnowledgeSyncItemRow).where(SkillKnowledgeSyncItemRow.job_id == job_id).order_by(SkillKnowledgeSyncItemRow.created_at))).scalars()
result = _dict(job)
result["items"] = [_dict(item) for item in items]
return result
async def list_jobs(self, *, limit: int = 50) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeSyncJobRow).order_by(SkillKnowledgeSyncJobRow.created_at.desc()).limit(limit)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def list_queued_items(self, *, limit: int = 20) -> list[dict[str, Any]]:
now = _now()
stmt = (
select(SkillKnowledgeSyncItemRow)
.where(
or_(
SkillKnowledgeSyncItemRow.status == "queued",
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
)
)
.order_by(SkillKnowledgeSyncItemRow.created_at)
.limit(limit)
)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def claim_item(
self,
item_id: str,
*,
owner: str,
lease_seconds: int = 900,
) -> dict[str, Any] | None:
"""Atomically claim a queued item, or reclaim one whose worker lease expired."""
now = _now()
stmt = (
update(SkillKnowledgeSyncItemRow)
.where(
SkillKnowledgeSyncItemRow.id == item_id,
or_(
SkillKnowledgeSyncItemRow.status == "queued",
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
),
)
.values(
status="running",
started_at=func.coalesce(SkillKnowledgeSyncItemRow.started_at, now),
attempts=SkillKnowledgeSyncItemRow.attempts + 1,
lease_owner=owner,
lease_until=now + timedelta(seconds=lease_seconds),
updated_at=now,
)
)
async with self._sf() as session:
result = await session.execute(stmt)
if result.rowcount != 1:
await session.rollback()
return None
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def renew_lease(self, item_id: str, *, owner: str, lease_seconds: int = 900) -> bool:
now = _now()
async with self._sf() as session:
result = await session.execute(
update(SkillKnowledgeSyncItemRow)
.where(
SkillKnowledgeSyncItemRow.id == item_id,
SkillKnowledgeSyncItemRow.status == "running",
SkillKnowledgeSyncItemRow.lease_owner == owner,
)
.values(
lease_until=now + timedelta(seconds=lease_seconds),
updated_at=now,
)
)
await session.commit()
return result.rowcount == 1
async def update_item(
self,
item_id: str,
*,
status: str | None = None,
phase: str | None = None,
progress: int | None = None,
snapshot_id: str | None = None,
error_code: str | None = None,
error_message: str | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
if row is None:
return None
if status is not None:
row.status = status
if status == "running" and row.started_at is None:
row.started_at = _now()
row.attempts += 1
if status in {"completed", "partial_failed", "failed", "canceled"}:
row.completed_at = _now()
row.active_dedupe_key = None
elif status == "queued":
row.completed_at = None
if status != "running":
row.lease_owner = None
row.lease_until = None
if phase is not None:
row.phase = phase
if progress is not None:
row.progress = max(0, min(100, progress))
if snapshot_id is not None:
row.snapshot_id = snapshot_id
row.last_error_code = error_code
row.last_error_message = error_message
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def refresh_job(self, job_id: str) -> None:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return
rows = (await session.execute(select(SkillKnowledgeSyncItemRow.status, func.count()).where(SkillKnowledgeSyncItemRow.job_id == job_id).group_by(SkillKnowledgeSyncItemRow.status))).all()
counts = {str(status): int(count) for status, count in rows}
job.success_count = counts.get("completed", 0)
job.partial_failed_count = counts.get("partial_failed", 0)
job.failed_count = counts.get("failed", 0)
active = sum(counts.get(key, 0) for key in ("queued", "running", "awaiting_review"))
if counts.get("running", 0) or counts.get("awaiting_review", 0):
job.status = "running"
job.started_at = job.started_at or _now()
elif active == 0 and sum(counts.values()) >= job.total_count:
if job.failed_count and job.success_count == 0 and job.partial_failed_count == 0:
job.status = "failed"
elif job.failed_count or job.partial_failed_count:
job.status = "partial_failed"
else:
job.status = "completed"
job.completed_at = _now()
await session.commit()
async def cancel_job(self, job_id: str) -> bool:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return False
job.cancel_requested = True
await session.execute(
SkillKnowledgeSyncItemRow.__table__.update()
.where(
SkillKnowledgeSyncItemRow.job_id == job_id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "awaiting_review"]),
)
.values(
status="canceled",
phase="completed",
completed_at=_now(),
progress=100,
active_dedupe_key=None,
lease_owner=None,
lease_until=None,
)
)
job.status = "canceled"
job.completed_at = _now()
await session.commit()
return True
async def requeue_item(self, item_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
if row is None:
return None
active_item = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == row.binding_id,
SkillKnowledgeSyncItemRow.id != row.id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
if active_item is not None:
return None
row.status = "queued"
row.phase = "scanning"
row.progress = 0
row.completed_at = None
row.active_dedupe_key = row.binding_id
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def get_or_create_snapshot(self, *, skill_name: str, source_digest: str, extractor_version: str, prompt_hash: str, model_name: str, manifest: list, ir: dict, counts: dict) -> dict[str, Any]:
stmt = select(SkillKnowledgeSnapshotRow).where(
SkillKnowledgeSnapshotRow.skill_name == skill_name,
SkillKnowledgeSnapshotRow.source_digest == source_digest,
SkillKnowledgeSnapshotRow.extractor_version == extractor_version,
SkillKnowledgeSnapshotRow.prompt_hash == prompt_hash,
SkillKnowledgeSnapshotRow.model_name == model_name,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = SkillKnowledgeSnapshotRow(
id=str(uuid4()),
skill_name=skill_name,
source_digest=source_digest,
extractor_version=extractor_version,
prompt_hash=prompt_hash,
model_name=model_name,
manifest_json=manifest,
ir_json=ir,
counts_json=counts,
status="ready",
created_at=_now(),
)
session.add(row)
await session.flush()
data = _dict(row)
await session.commit()
return data
async def get_snapshot(self, snapshot_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSnapshotRow, snapshot_id)
return _dict(row) if row else None
async def create_reviews(self, *, job_id: str, item_id: str, binding_id: str, snapshot_id: str, objects: list[tuple[str, dict]]) -> int:
rows = [
SkillKnowledgeReviewRow(
id=str(uuid4()),
job_id=job_id,
item_id=item_id,
binding_id=binding_id,
snapshot_id=snapshot_id,
object_type=kind,
object_key=str(obj.get("local_id") or obj.get("slug") or uuid4()),
original_json=obj,
status="pending",
created_at=_now(),
)
for kind, obj in objects
]
async with self._sf() as session:
session.add_all(rows)
await session.commit()
return len(rows)
async def list_reviews(self, *, job_id: str | None = None, item_id: str | None = None, status: str | None = "pending") -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeReviewRow)
if job_id:
stmt = stmt.where(SkillKnowledgeReviewRow.job_id == job_id)
if item_id:
stmt = stmt.where(SkillKnowledgeReviewRow.item_id == item_id)
if status:
stmt = stmt.where(SkillKnowledgeReviewRow.status == status)
stmt = stmt.order_by(SkillKnowledgeReviewRow.created_at)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def review(self, review_ids: list[str], *, action: str, actor: str | None, edits: dict[str, dict] | None = None, reason: str | None = None) -> list[str]:
final_status = "approved" if action == "approve" else "rejected"
affected_items: set[str] = set()
async with self._sf() as session:
rows = (await session.execute(select(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.id.in_(review_ids)))).scalars()
for row in rows:
row.status = final_status
row.edited_json = (edits or {}).get(row.id)
row.reason = reason
row.reviewed_by = actor
row.reviewed_at = _now()
affected_items.add(row.item_id)
await session.commit()
return sorted(affected_items)
async def pending_review_count(self, item_id: str) -> int:
stmt = select(func.count()).select_from(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.item_id == item_id, SkillKnowledgeReviewRow.status == "pending")
async with self._sf() as session:
return int((await session.execute(stmt)).scalar_one())
async def effective_ir(self, snapshot_id: str, item_id: str) -> dict[str, Any] | None:
snapshot = await self.get_snapshot(snapshot_id)
if snapshot is None:
return None
ir = dict(snapshot.get("ir") or {})
reviews = await self.list_reviews(item_id=item_id, status=None)
if not reviews:
return ir
by_key = {(row["object_type"], row["object_key"]): row for row in reviews}
for key in ("entities", "relations"):
kind = key[:-1]
accepted = []
for obj in ir.get(key, []):
review = by_key.get((kind, str(obj.get("local_id"))))
if review and review.get("status") == "rejected":
continue
accepted.append((review or {}).get("edited") or obj)
ir[key] = accepted
return ir
async def stage_and_activate(self, *, item_id: str, snapshot_id: str, ir: dict[str, Any], artifacts: list[dict[str, Any]], status: str = "synced") -> dict[str, Any]:
now = _now()
async with self._sf() as session:
item = await session.get(SkillKnowledgeSyncItemRow, item_id)
if item is None:
raise KeyError(item_id)
binding = await session.get(SkillKnowledgeBindingRow, item.binding_id)
if binding is None:
raise KeyError(item.binding_id)
if not binding.active:
raise RuntimeError("Binding was detached before staged activation")
await session.execute(
SkillKnowledgeArtifactRow.__table__.update()
.where(
SkillKnowledgeArtifactRow.binding_id == binding.id,
SkillKnowledgeArtifactRow.status == "active",
)
.values(status="retired", updated_at=now)
)
for artifact in artifacts:
session.add(
SkillKnowledgeArtifactRow(
id=str(uuid4()),
binding_id=binding.id,
snapshot_id=snapshot_id,
kind=str(artifact.get("kind") or "wiki_page"),
slug=artifact.get("slug"),
content_text=artifact.get("content"),
remote_id=artifact.get("remote_id"),
business_key=artifact.get("business_key"),
remote_refs_json=artifact.get("remote_refs") or [],
status="active",
metadata_json=artifact.get("metadata") or {},
created_at=now,
updated_at=now,
)
)
# Retire contributions from the old active version only after all staged artifacts are ready.
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding.id))
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding.id))
entity_ids: dict[str, str] = {}
for entity in ir.get("entities", []):
normalized = normalize_name(str(entity.get("name") or ""))
if not normalized:
continue
entity_type = str(entity.get("type") or "concept")
stmt = select(SkillKnowledgeEntityRow).where(
SkillKnowledgeEntityRow.target_type == binding.target_type,
SkillKnowledgeEntityRow.target_id == binding.target_id,
SkillKnowledgeEntityRow.entity_type == entity_type,
SkillKnowledgeEntityRow.normalized_name == normalized,
)
canonical = (await session.execute(stmt)).scalar_one_or_none()
if canonical is None:
canonical = SkillKnowledgeEntityRow(
id=str(uuid4()),
target_type=binding.target_type,
target_id=binding.target_id,
entity_type=entity_type,
normalized_name=normalized,
display_name=str(entity.get("name") or normalized),
aliases_json=entity.get("aliases") or [],
attributes_json=entity.get("attributes") or {},
status="active",
created_at=now,
updated_at=now,
)
session.add(canonical)
await session.flush()
else:
canonical.status = "active"
entity_ids[str(entity.get("local_id"))] = canonical.id
session.add(
SkillKnowledgeEntityContributionRow(
id=str(uuid4()),
entity_id=canonical.id,
binding_id=binding.id,
snapshot_id=snapshot_id,
local_entity_id=str(entity.get("local_id")),
evidence_json=entity.get("evidence") or [],
attributes_json=entity.get("attributes") or {},
confidence=float(entity.get("confidence") or 0),
)
)
for relation in ir.get("relations", []):
source_id = entity_ids.get(str(relation.get("source_local_id")))
target_id = entity_ids.get(str(relation.get("target_local_id")))
confidence = float(relation.get("confidence") or 0)
if not source_id or not target_id or confidence < 0.75 or not relation.get("evidence"):
continue
predicate = normalize_predicate(str(relation.get("predicate") or "related_to"))
stmt = select(SkillKnowledgeRelationRow).where(
SkillKnowledgeRelationRow.target_type == binding.target_type,
SkillKnowledgeRelationRow.target_id == binding.target_id,
SkillKnowledgeRelationRow.source_entity_id == source_id,
SkillKnowledgeRelationRow.predicate == predicate,
SkillKnowledgeRelationRow.target_entity_id == target_id,
)
canonical = (await session.execute(stmt)).scalar_one_or_none()
if canonical is None:
canonical = SkillKnowledgeRelationRow(
id=str(uuid4()),
target_type=binding.target_type,
target_id=binding.target_id,
source_entity_id=source_id,
predicate=predicate,
target_entity_id=target_id,
description=str(relation.get("description") or ""),
status="active",
created_at=now,
updated_at=now,
)
session.add(canonical)
await session.flush()
else:
canonical.status = "active"
session.add(
SkillKnowledgeRelationContributionRow(
id=str(uuid4()),
relation_id=canonical.id,
binding_id=binding.id,
snapshot_id=snapshot_id,
local_relation_id=str(relation.get("local_id")),
evidence_json=relation.get("evidence") or [],
confidence=confidence,
)
)
await self._retire_target_orphans(
session,
target_type=binding.target_type,
target_id=binding.target_id,
)
binding.current_snapshot_id = snapshot_id
binding.source_digest = str(ir.get("skill", {}).get("source_digest") or "")
binding.synced_digest = binding.source_digest
binding.status = status
binding.last_synced_at = now
binding.last_error_code = None
binding.last_error_message = None
binding.updated_at = now
item.snapshot_id = snapshot_id
item.status = "completed" if status in {"synced", "pending_remote_write"} else status
item.phase = "completed"
item.progress = 100
item.completed_at = now
item.active_dedupe_key = None
item.lease_owner = None
item.lease_until = None
item.updated_at = now
await session.flush()
result = _dict(binding)
await session.commit()
await self.refresh_job(item.job_id)
return result
async def detach_binding(
self,
binding_id: str,
*,
delete_contributions: bool,
cleanup_failed: bool = False,
) -> dict[str, Any] | None:
async with self._sf() as session:
binding = await session.get(SkillKnowledgeBindingRow, binding_id)
if binding is None:
return None
binding.active = False
binding.status = "detached_delete_pending_cleanup" if delete_contributions and cleanup_failed else "detached_deleted" if delete_contributions else "detached_keep"
binding.updated_at = _now()
if delete_contributions:
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding_id))
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding_id))
await session.execute(SkillKnowledgeArtifactRow.__table__.update().where(SkillKnowledgeArtifactRow.binding_id == binding_id).values(status="deleted", updated_at=_now()))
await self._retire_target_orphans(
session,
target_type=binding.target_type,
target_id=binding.target_id,
)
await session.flush()
data = _dict(binding)
await session.commit()
return data
async def mark_skill_stale(self, skill_name: str) -> int:
async with self._sf() as session:
result = await session.execute(
SkillKnowledgeBindingRow.__table__.update()
.where(
SkillKnowledgeBindingRow.skill_name == skill_name,
SkillKnowledgeBindingRow.active.is_(True),
SkillKnowledgeBindingRow.status.in_(["synced", "partial_failed", "pending_remote_write"]),
)
.values(status="stale", updated_at=_now())
)
await session.commit()
return int(result.rowcount or 0)
def make_skill_knowledge_store(session_factory):
return SkillKnowledgeRepository(session_factory) if session_factory is not None else None
@@ -3,7 +3,7 @@
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta, timezone
from typing import Any
from sqlalchemy import JSON, DateTime, Text
@@ -14,6 +14,12 @@ from sqlalchemy.types import TypeDecorator
# fixed offset is exact and avoids depending on the `tzdata` package (which the
# stdlib `zoneinfo` needs on Windows / minimal containers).
BEIJING_TZ = timezone(timedelta(hours=8))
_POSTGRES_FAMILY_DIALECTS = frozenset({"postgresql", "opengauss"})
def _is_postgres_family(dialect) -> bool:
"""Return whether *dialect* preserves timezone-aware timestamps natively."""
return dialect.name in _POSTGRES_FAMILY_DIALECTS
class PortableJSON(TypeDecorator):
@@ -87,17 +93,17 @@ class BeijingDateTime(TypeDecorator):
def process_bind_param(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
if _is_postgres_family(dialect):
return value
if value.tzinfo is None:
# Naive values follow the codebase convention of being UTC.
value = value.replace(tzinfo=timezone.utc)
value = value.replace(tzinfo=UTC)
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
def process_result_value(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
if _is_postgres_family(dialect):
return value
if value.tzinfo is None:
# Stored as naive Beijing wall-clock; re-attach the zone.
@@ -0,0 +1,10 @@
"""Skill knowledge distillation primitives.
The package is deliberately independent from ``app.*``. Gateway composition
injects persistence and remote adapters at runtime.
"""
from deerflow.skill_knowledge.extractor import build_skill_ir
from deerflow.skill_knowledge.scanner import ScanResult, enrich_converted_documents, scan_skill
__all__ = ["ScanResult", "build_skill_ir", "enrich_converted_documents", "scan_skill"]
@@ -0,0 +1,152 @@
"""IR construction and validation.
V1 intentionally starts with deterministic, evidence-backed extraction. The
schema is compatible with a future model-backed extractor, while never letting
unvalidated model output flow directly into persistence or graph writers.
"""
from __future__ import annotations
import hashlib
import re
from typing import Any
from deerflow.skill_knowledge.scanner import ScanResult
EXTRACTOR_VERSION = "skill-knowledge-v1"
PROMPT_HASH = f"sha256:{hashlib.sha256(b'skill-knowledge-v1-evidence-only').hexdigest()}"
PREDICATE_PATTERNS = {
"depends_on": re.compile(r"(?:依赖|需要|requires?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"uses": re.compile(r"(?:调用|使用|uses?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"produces": re.compile(r"(?:生成|输出|produces?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"consumes": re.compile(r"(?:读取|输入|consumes?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
}
def _evidence(path: str, text: str, start: int, end: int) -> dict[str, Any]:
line_start = text.count("\n", 0, start) + 1
line_end = text.count("\n", 0, end) + 1
quote = " ".join(text[start:end].split())[:240]
return {"file_path": path, "line_start": line_start, "line_end": line_end, "quote": quote}
def build_skill_ir(scan: ScanResult, *, display_name: str | None = None, description: str = "") -> dict[str, Any]:
entities: list[dict[str, Any]] = [
{
"local_id": "ent_skill",
"type": "skill",
"name": display_name or scan.skill_name,
"aliases": [scan.skill_name] if display_name and display_name != scan.skill_name else [],
"description": description,
"attributes": {},
"evidence": [],
"confidence": 1.0,
}
]
relations: list[dict[str, Any]] = []
by_key: dict[tuple[str, str], str] = {("skill", (display_name or scan.skill_name).casefold()): "ent_skill"}
for file in scan.files:
if not file.text:
continue
for predicate, pattern in PREDICATE_PATTERNS.items():
for match in pattern.finditer(file.text):
name = match.group(1).strip(" `\"'“”()[]")
if not name or len(name) > 80:
continue
key = ("concept", name.casefold())
entity_id = by_key.get(key)
if entity_id is None:
entity_id = f"ent_{len(entities):04d}"
by_key[key] = entity_id
entities.append(
{
"local_id": entity_id,
"type": "concept",
"name": name,
"aliases": [],
"description": "从技能业务说明中抽取的知识对象",
"attributes": {},
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
"confidence": 0.82,
}
)
relations.append(
{
"local_id": f"rel_{len(relations):04d}",
"source_local_id": "ent_skill",
"predicate": predicate,
"target_local_id": entity_id,
"description": " ".join(match.group(0).split())[:240],
"relation_source": "explicit_text",
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
"confidence": 0.82,
}
)
for ref in file.references:
name = ref["target_path"]
key = ("file", name.casefold())
entity_id = by_key.get(key)
if entity_id is None:
entity_id = f"ent_{len(entities):04d}"
by_key[key] = entity_id
entities.append(
{
"local_id": entity_id,
"type": "file",
"name": name,
"aliases": [],
"description": "技能文档引用的资料文件",
"attributes": {},
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
"confidence": 1.0,
}
)
relations.append(
{
"local_id": f"rel_{len(relations):04d}",
"source_local_id": "ent_skill",
"predicate": "references",
"target_local_id": entity_id,
"description": f"{file.path} 引用 {name}",
"relation_source": "deterministic_structure",
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
"confidence": 1.0,
}
)
title = display_name or scan.skill_name
summaries = [f"- **{item.title}**(`{item.path}`)" for item in scan.files[:100]]
content = "\n".join(
[
f"# {title}",
"",
description or "由技能知识文件安全归纳生成。",
"",
"## 知识资料",
"",
*(summaries or ["- 暂无可归纳知识文件"]),
"",
f"> 源摘要:`{scan.source_digest}`;代码与脚本文件未读取。",
]
)
return {
"extractor_version": EXTRACTOR_VERSION,
"prompt_hash": PROMPT_HASH,
"model_name": "deterministic-v1",
"skill": {
"name": scan.skill_name,
"display_name": title,
"description": description,
"source_root": scan.source_root,
"source_digest": scan.source_digest,
},
"files": scan.manifest,
"entities": entities,
"relations": relations,
"wiki": {
"skill_page": {"slug": f"skills/{scan.skill_name}/overview", "title": title, "content": content},
"pages": [],
},
"safety": {"blocked_files": scan.blocked_files},
}
@@ -0,0 +1,115 @@
"""Guarded graph-write plans.
No caller can submit Cypher. The writer accepts only normalized node/edge
records and executes fixed parameterized templates when explicitly enabled.
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from typing import Any, Protocol
class GraphTransaction(Protocol):
async def run(self, query: str, parameters: dict[str, Any]) -> Any: ...
@dataclass(frozen=True, slots=True)
class GraphWritePlan:
scope: str
nodes: tuple[dict[str, Any], ...]
edges: tuple[dict[str, Any], ...]
NODE_QUERY = """
MERGE (n:DeerFlowKnowledge {scope: $scope, entity_key: $entity_key})
SET n.display_name = $display_name, n.entity_type = $entity_type,
n.binding_id = $binding_id, n.snapshot_id = $snapshot_id, n.job_id = $job_id,
n.confidence = $confidence, n.updated_at = datetime()
""".strip()
EDGE_QUERY = """
MATCH (a:DeerFlowKnowledge {scope: $scope, entity_key: $source_key})
MATCH (b:DeerFlowKnowledge {scope: $scope, entity_key: $target_key})
MERGE (a)-[r:DEERFLOW_RELATION {edge_key: $edge_key}]->(b)
SET r.predicate = $predicate, r.binding_id = $binding_id,
r.snapshot_id = $snapshot_id, r.job_id = $job_id, r.confidence = $confidence,
r.evidence = $evidence, r.updated_at = datetime()
""".strip()
def _key(*parts: str) -> str:
return hashlib.sha256("\x1f".join(parts).encode()).hexdigest()
def plan_from_ir(
ir: dict[str, Any],
*,
target_type: str,
target_id: str,
binding_id: str,
snapshot_id: str,
job_id: str,
) -> GraphWritePlan:
"""Build a stable write plan; callers cannot supply executable Cypher."""
scope = f"{target_type}:{target_id}"
local_keys: dict[str, str] = {}
nodes: list[dict[str, Any]] = []
for entity in ir.get("entities") or []:
entity_type = str(entity.get("type") or "concept")
normalized_name = str(entity.get("normalized_name") or entity.get("name") or "").strip().casefold()
if not normalized_name:
continue
entity_key = _key(entity_type, normalized_name)
local_keys[str(entity.get("local_id") or normalized_name)] = entity_key
nodes.append(
{
"entity_key": entity_key,
"display_name": str(entity.get("name") or normalized_name),
"entity_type": entity_type,
"binding_id": binding_id,
"snapshot_id": snapshot_id,
"job_id": job_id,
"confidence": float(entity.get("confidence") or 1.0),
}
)
edges: list[dict[str, Any]] = []
for relation in ir.get("relations") or []:
source_key = local_keys.get(str(relation.get("source_local_id") or ""))
target_key = local_keys.get(str(relation.get("target_local_id") or ""))
evidence = list(relation.get("evidence") or [])
confidence = float(relation.get("confidence") or 0)
predicate = str(relation.get("predicate") or "related_to")
if not source_key or not target_key or confidence < 0.75 or not evidence:
continue
edges.append(
{
"edge_key": _key(source_key, predicate, target_key, binding_id, snapshot_id),
"source_key": source_key,
"target_key": target_key,
"predicate": predicate,
"binding_id": binding_id,
"snapshot_id": snapshot_id,
"job_id": job_id,
"confidence": confidence,
"evidence": evidence,
}
)
return GraphWritePlan(scope=scope, nodes=tuple(nodes), edges=tuple(edges))
def parameterized_statements(plan: GraphWritePlan) -> list[dict[str, Any]]:
"""Return Neo4j transactional HTTP statements using only fixed templates."""
statements = [{"statement": NODE_QUERY, "parameters": {"scope": plan.scope, **node}} for node in plan.nodes]
statements.extend({"statement": EDGE_QUERY, "parameters": {"scope": plan.scope, **edge}} for edge in plan.edges)
return statements
async def execute_plan(tx: GraphTransaction, plan: GraphWritePlan, *, enabled: bool) -> dict[str, int]:
if not enabled:
return {"nodes": 0, "edges": 0}
for node in plan.nodes:
await tx.run(NODE_QUERY, {"scope": plan.scope, **node})
for edge in plan.edges:
await tx.run(EDGE_QUERY, {"scope": plan.scope, **edge})
return {"nodes": len(plan.nodes), "edges": len(plan.edges)}
@@ -0,0 +1,31 @@
"""Portable entity/relation normalization used by both knowledge products."""
from __future__ import annotations
import re
import unicodedata
def normalize_name(value: str) -> str:
text = unicodedata.normalize("NFKC", value or "").strip().casefold()
text = text.strip("`'\"“”‘’《》〈〉[](){}")
text = re.sub(r"\s+", " ", text)
return text
def normalize_predicate(value: str) -> str:
normalized = normalize_name(value).replace(" ", "_")
aliases = {
"依赖": "depends_on",
"需要": "depends_on",
"requires": "depends_on",
"调用": "uses",
"使用": "uses",
"uses": "uses",
"生成": "produces",
"输出": "produces",
"读取": "consumes",
"输入": "consumes",
"引用": "references",
}
return aliases.get(normalized, normalized or "related_to")
@@ -0,0 +1,351 @@
"""Safe, deterministic scanner for the knowledge-bearing part of a skill."""
from __future__ import annotations
import csv
import hashlib
import json
import re
import zipfile
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
CODE_SUFFIXES = {
".py",
".pyc",
".pyo",
".ts",
".tsx",
".js",
".jsx",
".mjs",
".cjs",
".sh",
".bash",
".zsh",
".fish",
".bat",
".cmd",
".ps1",
".psm1",
".java",
".kt",
".kts",
".go",
".rs",
".c",
".cc",
".cpp",
".h",
".hpp",
".cs",
".php",
".rb",
".pl",
".lua",
".swift",
".scala",
".sql",
".vue",
".svelte",
}
TEXT_SUFFIXES = {".md", ".mdx", ".txt", ".json", ".yaml", ".yml", ".toml", ".ini", ".csv", ".tsv"}
BINARY_KNOWLEDGE_SUFFIXES = {".xlsx", ".xls", ".pdf", ".docx", ".pptx", ".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".zip"}
EXCLUDED_DIRS = {".git", "node_modules", "__pycache__", ".pytest_cache", "dist", "build", ".mypy_cache", ".ruff_cache", ".address-edits"}
EXCLUDED_FILENAMES = {
"package-lock.json",
"pnpm-lock.yaml",
"yarn.lock",
"bun.lockb",
"poetry.lock",
"uv.lock",
"pipfile.lock",
"cargo.lock",
"composer.lock",
}
SENSITIVE_NAME_RE = re.compile(
r"(^|[._-])(\.env|id_rsa|id_dsa|id_ecdsa|id_ed25519|private[_-]?key|credentials?|secrets?)([._-]|$)",
re.IGNORECASE,
)
SECRET_CONTENT_RE = re.compile(r"(?im)^\s*(?:api[_-]?key|access[_-]?key|secret(?:[_-]?key)?|password|token|private[_-]?key)\s*[:=]\s*[^\s]{8,}\s*$")
PEM_PRIVATE_RE = re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")
FENCE_RE = re.compile(r"(?ms)^\s*(```|~~~).*?^\s*\1\s*$")
MARKDOWN_LINK_RE = re.compile(r"\[[^\]]+\]\(([^)#?]+)(?:#[^)]*)?\)")
MAX_TEXT_BYTES = 2 * 1024 * 1024
MAX_ARCHIVE_FILES = 1_000
MAX_ARCHIVE_ENTRY_BYTES = 20 * 1024 * 1024
MAX_ARCHIVE_TOTAL_BYTES = 200 * 1024 * 1024
@dataclass(slots=True)
class ScannedFile:
path: str
kind: str
sha256: str
size: int
text: str = ""
title: str = ""
references: list[dict[str, Any]] = field(default_factory=list)
safety: dict[str, Any] = field(default_factory=lambda: {"status": "allowed", "warnings": []})
def manifest(self) -> dict[str, Any]:
return {
"path": self.path,
"kind": self.kind,
"sha256": self.sha256,
"size": self.size,
"text_digest": f"sha256:{hashlib.sha256(self.text.encode('utf-8')).hexdigest()}" if self.text else None,
"title": self.title,
"references": self.references,
"safety": self.safety,
}
@dataclass(slots=True)
class ScanResult:
skill_name: str
source_root: str
source_digest: str
files: list[ScannedFile]
blocked_files: list[dict[str, str]]
skipped_code_count: int
skipped_other_count: int
@property
def manifest(self) -> list[dict[str, Any]]:
return [item.manifest() for item in self.files] + [{"path": item["path"], "kind": "blocked", "safety": {"status": "blocked", "warnings": [item["reason"]]}} for item in self.blocked_files]
@property
def counts(self) -> dict[str, int]:
return {
"knowledge_files": len(self.files),
"blocked_files": len(self.blocked_files),
"skipped_code_files": self.skipped_code_count,
"skipped_other_files": self.skipped_other_count,
}
def _safe_relative(path: Path, root: Path) -> str:
resolved = path.resolve()
resolved_root = root.resolve()
try:
relative = resolved.relative_to(resolved_root)
except ValueError as exc:
raise ValueError("Skill file escapes its source root") from exc
return relative.as_posix()
def _strip_markdown_code(text: str) -> str:
return FENCE_RE.sub("\n[示例代码已按安全策略省略]\n", text)
def _read_text(path: Path, suffix: str) -> str:
raw = path.read_bytes()
if len(raw) > MAX_TEXT_BYTES:
raise ValueError("text file exceeds 2 MiB parsing limit")
text = raw.decode("utf-8-sig", errors="replace")
if suffix in {".md", ".mdx"}:
return _strip_markdown_code(text)
if suffix in {".csv", ".tsv"}:
delimiter = "\t" if suffix == ".tsv" else ","
rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
return "\n".join(" | ".join(cell.strip() for cell in row) for row in rows[:500])
if suffix == ".json":
try:
return json.dumps(json.loads(text), ensure_ascii=False, indent=2)
except json.JSONDecodeError:
return text
return text
def _title(text: str, fallback: str) -> str:
match = re.search(r"(?m)^#\s+(.+?)\s*$", text)
return match.group(1).strip() if match else fallback
def _kind(suffix: str) -> str:
return {
".md": "markdown",
".mdx": "markdown",
".txt": "text",
".csv": "table",
".tsv": "table",
".xlsx": "spreadsheet",
".xls": "spreadsheet",
".pdf": "pdf",
".docx": "office",
".pptx": "office",
".png": "image",
".jpg": "image",
".jpeg": "image",
".webp": "image",
".gif": "image",
".bmp": "image",
".zip": "archive",
}.get(suffix, "structured_data")
def _zip_members(path: Path, archive_relative: str) -> tuple[list[ScannedFile], list[dict[str, str]], int, int]:
files: list[ScannedFile] = []
blocked: list[dict[str, str]] = []
skipped_code = 0
skipped_other = 0
with zipfile.ZipFile(path) as archive:
members = archive.infolist()
if len(members) > MAX_ARCHIVE_FILES:
return [], [{"path": archive_relative, "reason": "压缩包文件数超过安全上限"}], 0, 0
total_size = sum(member.file_size for member in members)
if total_size > MAX_ARCHIVE_TOTAL_BYTES:
return [], [{"path": archive_relative, "reason": "压缩包展开体积超过安全上限"}], 0, 0
for member in members:
member_path = Path(member.filename.replace("\\", "/"))
virtual_path = f"{archive_relative}!/{member_path.as_posix()}"
unix_mode = member.external_attr >> 16
if member.is_dir():
continue
if member_path.is_absolute() or ".." in member_path.parts or (unix_mode & 0o170000) == 0o120000:
blocked.append({"path": virtual_path, "reason": "压缩包包含路径穿越或符号链接"})
continue
suffix = member_path.suffix.lower()
lower_name = member_path.name.lower()
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
skipped_code += 1
continue
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
blocked.append({"path": virtual_path, "reason": "疑似密钥或敏感配置文件"})
continue
if suffix not in TEXT_SUFFIXES or member.file_size > MAX_ARCHIVE_ENTRY_BYTES:
skipped_other += 1
continue
raw = archive.read(member)
text = raw.decode("utf-8-sig", errors="replace")
if suffix in {".md", ".mdx"}:
text = _strip_markdown_code(text)
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
blocked.append({"path": virtual_path, "reason": "内容命中高置信敏感信息规则"})
continue
files.append(
ScannedFile(
path=virtual_path,
kind=_kind(suffix),
sha256=hashlib.sha256(raw).hexdigest(),
size=len(raw),
text=text,
title=_title(text, member_path.stem),
)
)
return files, blocked, skipped_code, skipped_other
def scan_skill(skill_name: str, skill_root: Path) -> ScanResult:
"""Scan one already-resolved skill directory without following symlinks."""
root = skill_root.resolve()
if not root.is_dir():
raise FileNotFoundError(f"Skill '{skill_name}' source directory does not exist")
files: list[ScannedFile] = []
blocked: list[dict[str, str]] = []
skipped_code = 0
skipped_other = 0
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix().lower()):
if path.is_symlink() or not path.is_file():
continue
relative = _safe_relative(path, root)
if any(part.lower() in EXCLUDED_DIRS for part in Path(relative).parts[:-1]):
continue
lower_name = path.name.lower()
suffix = path.suffix.lower()
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
skipped_code += 1
continue
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
blocked.append({"path": relative, "reason": "疑似密钥或敏感配置文件"})
continue
if suffix not in TEXT_SUFFIXES | BINARY_KNOWLEDGE_SUFFIXES:
skipped_other += 1
continue
# Never upload an archive wholesale: it may contain excluded code or a
# blocked secret. Only independently validated knowledge members enter
# the manifest and digest.
if suffix == ".zip":
try:
nested, nested_blocked, nested_code, nested_other = _zip_members(path, relative)
files.extend(nested)
blocked.extend(nested_blocked)
skipped_code += nested_code
skipped_other += nested_other
except (OSError, zipfile.BadZipFile):
blocked.append({"path": relative, "reason": "压缩包损坏或格式不受支持"})
continue
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
text = ""
warnings: list[str] = []
if suffix in TEXT_SUFFIXES:
try:
text = _read_text(path, suffix)
except ValueError as exc:
warnings.append(str(exc))
if text and (SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text)):
blocked.append({"path": relative, "reason": "内容命中高置信敏感信息规则"})
continue
references = []
if suffix in {".md", ".mdx"}:
for target in MARKDOWN_LINK_RE.findall(text):
normalized = target.replace("\\", "/").lstrip("./")
if normalized:
references.append({"target_path": normalized, "evidence": "markdown_link", "confidence": 1.0})
files.append(
ScannedFile(
path=relative,
kind=_kind(suffix),
sha256=digest,
size=len(raw),
text=text,
title=_title(text, path.stem),
references=references,
safety={"status": "allowed", "warnings": warnings},
)
)
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in files).encode("utf-8")
source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
return ScanResult(
skill_name=skill_name,
source_root=str(root),
source_digest=source_digest,
files=files,
blocked_files=blocked,
skipped_code_count=skipped_code,
skipped_other_count=skipped_other,
)
async def enrich_converted_documents(scan: ScanResult) -> ScanResult:
"""Extract Office/PDF/spreadsheet text without modifying the skill tree."""
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, extract_file_to_markdown
allowed: list[ScannedFile] = []
root = Path(scan.source_root)
for item in scan.files:
if Path(item.path).suffix.lower() in CONVERTIBLE_EXTENSIONS:
text = await extract_file_to_markdown(root / item.path)
if text:
text = _strip_markdown_code(text)
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
scan.blocked_files.append({"path": item.path, "reason": "转换文本命中高置信敏感信息规则"})
continue
item.text = text
item.title = _title(text, item.title)
else:
item.safety["warnings"].append("文档转换不可用,仅保留原始附件")
allowed.append(item)
scan.files = allowed
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in scan.files).encode("utf-8")
scan.source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
return scan
@@ -256,6 +256,24 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
return None
async def extract_file_to_markdown(file_path: Path) -> str | None:
"""Extract a supported document to Markdown without creating a sidecar file.
Knowledge-ingest scanners use this variant because running a read-only
distillation task must never add generated ``.md`` files to a live skill.
Conversion always runs off the event loop; Office/PDF parsers can be CPU
intensive even for deceptively small inputs.
"""
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
return None
try:
return await asyncio.to_thread(_do_convert, file_path, _get_pdf_converter())
except Exception as exc:
logger.error("Failed to extract %s as markdown: %s", file_path.name, exc)
return None
# Regex for bold-only lines that look like section headings.
# Targets SEC filing structural headings that pymupdf4llm renders as **bold**
# rather than # Markdown headings (because they use same font size as body text,
@@ -53,6 +53,10 @@ postgres = [
"psycopg-pool>=3.3.0",
]
mysql = ["asyncmy>=0.2.10"]
gauss = [
"asyncpg>=0.29",
"opengauss-sqlalchemy>=2.4,<3",
]
pymupdf = ["pymupdf4llm>=0.0.17"]
[build-system]
@@ -30,6 +30,7 @@ dependencies = [
[project.optional-dependencies]
postgres = ["deerflow-harness[postgres]"]
mysql = ["deerflow-harness[mysql]"]
gauss = ["deerflow-harness[gauss]"]
[dependency-groups]
dev = [
@@ -0,0 +1,180 @@
from __future__ import annotations
import os
from pathlib import Path
from types import SimpleNamespace
import pytest
from app.gateway import deps
from app.gateway.routers import artifacts
class _FakeThreadStore:
def __init__(self, rows: list[dict]) -> None:
self.rows = rows
self.calls: list[dict] = []
async def search(self, **kwargs) -> list[dict]:
self.calls.append(kwargs)
offset = int(kwargs.get("offset", 0))
limit = int(kwargs.get("limit", 100))
return self.rows[offset : offset + limit]
class _FakeAgentStore:
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict]:
assert user_id == "owner"
agents = {
"analysis-agent": {
"id": "analysis-agent",
"name": "舆情分析智能体",
}
}
return [agents[agent_id] for agent_id in agent_ids if agent_id in agents]
@pytest.mark.asyncio
async def test_artifact_library_aggregates_generated_non_media_files(tmp_path: Path, monkeypatch):
thread_one = tmp_path / "thread-one" / "outputs"
thread_two = tmp_path / "thread-two" / "outputs"
thread_one.mkdir(parents=True)
thread_two.mkdir(parents=True)
report = thread_one / "研判报告.docx"
table = thread_one / "数据汇总.xlsx"
image = thread_one / "配图.png"
audio = thread_two / "录音.mp3"
notes = thread_two / "结论.md"
archive = thread_two / "证据.zip"
for path, content in (
(report, b"doc"),
(table, b"sheet"),
(image, b"image"),
(audio, b"audio"),
(notes, b"# notes"),
(archive, b"zip"),
):
path.write_bytes(content)
os.utime(report, (100, 100))
os.utime(table, (200, 200))
os.utime(notes, (300, 300))
os.utime(archive, (400, 400))
rows = [
{
"thread_id": "thread-one",
"display_name": "战场环境综合研判",
"metadata": {"agent_id": "analysis-agent"},
},
{
"thread_id": "thread-two",
"display_name": "生成发展报告",
"metadata": {},
},
]
store = _FakeThreadStore(rows)
class _Paths:
def sandbox_outputs_dir(self, thread_id: str, *, user_id: str) -> Path:
assert user_id == "owner"
return tmp_path / thread_id / "outputs"
async def _resolve_owner(_thread_id: str) -> str:
return "owner"
monkeypatch.setattr(artifacts, "get_paths", lambda: _Paths())
monkeypatch.setattr(artifacts, "aresolve_path_user_id", _resolve_owner)
items = await artifacts._scan_artifact_library(store)
assert [item["name"] for item in items] == ["证据.zip", "结论.md", "数据汇总.xlsx", "研判报告.docx"]
assert [item["file_kind"] for item in items] == ["archive", "text_data", "spreadsheet", "document"]
assert [item["source_type"] for item in items] == ["general_qa", "general_qa", "agent", "agent"]
assert items[-1]["thread_title"] == "战场环境综合研判"
assert items[-1]["agent_id"] == "analysis-agent"
assert all(item["name"] not in {"配图.png", "录音.mp3"} for item in items)
assert store.calls[0]["exclude_system"] is True
@pytest.mark.asyncio
async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Path, monkeypatch):
output_dir = tmp_path / "thread-one" / "outputs"
agent_output_dir = tmp_path / "thread-agent" / "outputs"
output_dir.mkdir(parents=True)
agent_output_dir.mkdir(parents=True)
(output_dir / "任务报告.md").write_text("report", encoding="utf-8")
(output_dir / "任务数据.csv").write_text("a,b", encoding="utf-8")
(agent_output_dir / "智能体结论.md").write_text("agent report", encoding="utf-8")
store = _FakeThreadStore(
[
{
"thread_id": "thread-one",
"display_name": "任务研判对话",
"metadata": {},
},
{
"thread_id": "thread-agent",
"display_name": "智能体研判对话",
"metadata": {"agent_id": "analysis-agent"},
},
]
)
class _Paths:
def sandbox_outputs_dir(self, thread_id: str, *, user_id: str) -> Path:
return tmp_path / thread_id / "outputs"
async def _resolve_owner(_thread_id: str) -> str:
return "owner"
monkeypatch.setattr(artifacts, "get_paths", lambda: _Paths())
monkeypatch.setattr(artifacts, "aresolve_path_user_id", _resolve_owner)
monkeypatch.setattr(artifacts, "get_effective_user_id", lambda: "owner")
monkeypatch.setattr(deps, "get_thread_store", lambda _request: store)
monkeypatch.setattr(deps, "get_agent_store", lambda _request: _FakeAgentStore())
artifacts._artifact_library_cache.clear()
raw_endpoint = artifacts.list_artifact_library.__wrapped__
response = await raw_endpoint(
request=SimpleNamespace(),
query="研判对话",
file_kind="text_data",
source_type="general_qa",
offset=0,
limit=1,
refresh=True,
)
assert response.total == 1
assert response.offset == 0
assert response.limit == 1
assert [item.name for item in response.items] == ["任务报告.md"]
spreadsheet_response = await raw_endpoint(
request=SimpleNamespace(),
query="任务数据",
file_kind="spreadsheet",
source_type="all",
offset=0,
limit=50,
refresh=False,
)
assert spreadsheet_response.total == 1
assert spreadsheet_response.items[0].name == "任务数据.csv"
agent_response = await raw_endpoint(
request=SimpleNamespace(),
query=None,
file_kind="all",
source_type="agent",
offset=0,
limit=50,
refresh=False,
)
assert agent_response.total == 1
assert agent_response.items[0].source_type == "agent"
assert agent_response.items[0].agent_name == "舆情分析智能体"
assert agent_response.items[0].name == "智能体结论.md"
@@ -0,0 +1,193 @@
import pytest
import pytest_asyncio
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
)
from deerflow.persistence.base import Base
class FakeEmbeddingClient:
dimensions = 3
async def embed_query(self, _query: str) -> list[float]:
return [1.0, 0.0, 0.0]
@pytest_asyncio.fixture
async def repository():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as connection:
await connection.run_sync(Base.metadata.create_all)
yield AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
await engine.dispose()
@pytest.mark.asyncio
async def test_single_base_entity_alignment_search_and_revision(repository: AssistantKnowledgeRepository) -> None:
first = await repository.initialize(created_by="admin")
second = await repository.initialize(created_by="someone-else")
assert first["id"] == second["id"]
package = {
"source": {"type": "skill"},
"wiki_pages": [{"slug": "skills/demo/overview", "title": "示例知识", "content": "张三负责测试。"}],
"entities": [{"local_id": "e1", "type": "person", "name": "张三", "aliases": ["Zhang San"]}],
"relations": [],
"documents": [],
"chunks": [],
"graph": {"nodes": [], "edges": []},
}
await repository.import_package(
package=package,
source_type="skill",
source_key="demo",
source_name="示例技能",
trigger="skill_direct",
created_by="admin",
skill_name="demo",
)
await repository.import_package(
package=package,
source_type="weknora",
source_key="kb-1",
source_name="普通库",
trigger="manual_from_weknora",
created_by="admin",
)
results = await repository.list_pages(query="张三")
alias_results = await repository.list_pages(query="Zhang San")
revisions = await repository.list_revisions("skills/demo/overview")
assert {row["page_type"] for row in results} == {"source", "entity"}
assert [row["page_type"] for row in alias_results] == ["entity"]
assert len(revisions) == 2
@pytest.mark.asyncio
async def test_delete_source_keeps_entity_until_last_contribution_is_removed(
repository: AssistantKnowledgeRepository,
) -> None:
package = {
"source": {},
"wiki_pages": [],
"entities": [{"local_id": "e1", "type": "concept", "name": "共享实体"}],
"relations": [],
}
for source_key in ("one", "two"):
await repository.import_package(
package=package,
source_type="skill",
source_key=source_key,
source_name=source_key,
trigger="skill_direct",
created_by="admin",
skill_name=source_key,
)
sources = await repository.list_sources()
await repository.delete_source(next(row["id"] for row in sources if row["source_key"] == "one"))
async with repository._sf() as session:
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow))).scalars().all()
assert entity.status == "active"
assert len(contributions) == 1
await repository.delete_source(next(row["id"] for row in sources if row["source_key"] == "two"))
async with repository._sf() as session:
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
assert entity.status == "deleted"
@pytest.mark.asyncio
async def test_custom_base_import_is_also_mirrored_to_global_base(
repository: AssistantKnowledgeRepository,
) -> None:
global_base = await repository.initialize(created_by="admin")
custom_base = await repository.create_base(
created_by="admin",
name="专题助手库",
description="只承接某个专题的 WeKnora 导出数据",
)
package = {
"source": {"type": "weknora"},
"wiki_pages": [{"slug": "wiki/topic", "title": "专题 Wiki", "content": "王五负责统筹。"}],
"entities": [{"local_id": "e1", "type": "person", "name": "王五"}],
"relations": [],
}
job = await repository.import_package(
package=package,
source_type="weknora",
source_key="kb-topic",
source_name="专题普通库",
trigger="manual_from_weknora",
created_by="admin",
mapping_id="kb-topic",
base_id=custom_base["id"],
)
assert job["base_id"] == custom_base["id"]
assert job["mirrored_global_base_id"] == global_base["id"]
assert len(await repository.list_pages(base_id=custom_base["id"], query="王五")) == 2
assert len(await repository.list_pages(base_id=global_base["id"], query="王五")) == 2
bases = await repository.list_bases()
assert [base["id"] for base in bases[:2]] == [global_base["id"], custom_base["id"]]
@pytest.mark.asyncio
async def test_imported_vectors_are_reused_for_assistant_search(
repository: AssistantKnowledgeRepository,
) -> None:
base = await repository.initialize(created_by="admin")
job = await repository.import_package(
package={
"source": {"type": "weknora"},
"wiki_pages": [{"slug": "wiki/vector", "title": "向量页", "content": "页面正文"}],
"vectors": [
{
"wiki_slug": "wiki/vector",
"section_index": 0,
"section_content": "张三负责向量化检索。",
"embedding_model": "bge-embedding-m3",
"embedding_dimensions": 3,
"vector": [1.0, 0.0, 0.0],
},
{
"wiki_slug": "wiki/vector",
"section_index": 1,
"section_content": "无关片段。",
"embedding_model": "bge-embedding-m3",
"embedding_dimensions": 3,
"vector": [0.0, 1.0, 0.0],
},
],
},
source_type="weknora",
source_key="kb-vector",
source_name="向量普通库",
trigger="manual_package_upload",
created_by="admin",
)
assert job["counts"]["vectors"] == 2
results = await repository.search_pages(
base_ids=[base["id"]],
query="这个查询不靠关键词",
embedding_client=FakeEmbeddingClient(),
)
assert results[0]["retrieval_mode"] == "vector"
assert results[0]["matched_section"]["embedding_model"] == "bge-embedding-m3"
assert results[0]["content_markdown"] == "张三负责向量化检索。"
sources = await repository.list_sources()
assert await repository.delete_source(sources[0]["id"])
assert await repository.search_pages(
base_ids=[base["id"]],
query="这个查询不靠关键词",
embedding_client=FakeEmbeddingClient(),
) == []
@@ -0,0 +1,81 @@
"""Regression coverage for the openGauss/GaussDB persistence backend."""
from __future__ import annotations
from datetime import UTC, datetime
import pytest
from pydantic import ValidationError
from sqlalchemy.engine import make_url
from sqlalchemy.schema import CreateIndex, CreateTable
from deerflow.config.database_config import DatabaseConfig
from deerflow.persistence.types import BeijingDateTime
@pytest.mark.parametrize(
("configured", "expected"),
[
(
"opengauss://user:pass@127.0.0.1:26000/deerflow",
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
),
(
"postgresql://user:pass@127.0.0.1:26000/deerflow?ssl=require",
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow?ssl=require",
),
(
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
"opengauss+asyncpg://user:pass@127.0.0.1:26000/deerflow",
),
],
)
def test_gauss_urls_always_select_the_opengauss_async_dialect(
configured: str,
expected: str,
) -> None:
config = DatabaseConfig(backend="gauss", gauss_url=configured)
assert config.app_sqlalchemy_url == expected
def test_gauss_backend_requires_a_supported_nonempty_url() -> None:
with pytest.raises(ValueError, match="gauss_url is required"):
_ = DatabaseConfig(backend="gauss").app_sqlalchemy_url
with pytest.raises(ValueError, match="must use opengauss"):
_ = DatabaseConfig(
backend="gauss",
gauss_url="mysql://user:pass@127.0.0.1/deerflow",
).app_sqlalchemy_url
with pytest.raises(ValidationError):
DatabaseConfig(backend="unknown")
def test_gauss_timestamps_keep_timezone_information() -> None:
dialect = type("GaussDialect", (), {"name": "opengauss"})()
value = datetime(2026, 9, 2, 8, 30, tzinfo=UTC)
column_type = BeijingDateTime()
assert column_type.process_bind_param(value, dialect) is value
assert column_type.process_result_value(value, dialect) is value
def test_official_async_dialect_compiles_the_complete_orm_schema() -> None:
"""Every registered table/index must compile before startup touches GaussDB."""
pytest.importorskip("opengauss_sqlalchemy")
import deerflow.persistence.models # noqa: F401 -- register ORM metadata
from deerflow.persistence.base import Base
dialect_cls = make_url("opengauss+asyncpg://user:pass@localhost/db").get_dialect(
_is_async=True
)
dialect = dialect_cls()
assert dialect.__class__.__module__.startswith("opengauss_sqlalchemy")
assert Base.metadata.sorted_tables
for table in Base.metadata.sorted_tables:
assert str(CreateTable(table).compile(dialect=dialect)).startswith("\nCREATE TABLE")
for index in table.indexes:
assert str(CreateIndex(index).compile(dialect=dialect)).startswith("CREATE")
@@ -66,6 +66,9 @@ async def test_sql_store_atomically_replaces_vectors_and_marks_missing_deleted(t
assert store.write_batch_size == 1
assert len(snapshot["rows"]) == 3
assert decode_vector(snapshot["rows"][0]["vector_blob"], 2).tolist() == pytest.approx([0.6, 0.8])
streamed = [row async for row in store.iter_vector_snapshot_rows("mapping-1", "fp", batch_size=2)]
assert [row["section_index"] for row in streamed] == [0, 1, 2]
assert streamed[0]["page"]["slug"] == "people/alice"
page_statuses = await store.list_page_index_status("mapping-1")
assert len(page_statuses) == 1
assert page_statuses[0] == {
@@ -68,7 +68,7 @@ def _published_row() -> dict[str, object]:
@pytest.mark.asyncio
async def test_public_knowledge_base_list_exposes_only_published_wiki_identity() -> None:
async def test_public_knowledge_base_list_exposes_live_safe_metadata_and_generated_index(monkeypatch) -> None:
store = PublicListStore(
[
{
@@ -83,21 +83,116 @@ async def test_public_knowledge_base_list_exposes_only_published_wiki_identity()
"owner_user_id": "system",
"name": "对话沉淀",
},
{
**_published_row(),
"id": "stale-kb",
"weknora_id": "remote-missing",
},
]
)
class Client:
async def list_knowledge_bases(self):
return [
{
"id": "remote-one",
"name": "远端公开知识库",
"description": "远端公开说明",
"type": "wiki",
"knowledge_count": 12,
"chunk_count": 34,
"processing_count": 1,
"share_count": 3,
"created_at": "2026-07-01T08:00:00+08:00",
"updated_at": "2026-08-01T08:00:00+08:00",
"tenant_id": 10001,
"creator_id": "secret-creator",
"storage_config": {"secret_key": "must-not-leak"},
}
]
async def get_knowledge_base(self, knowledge_base_id: str):
assert knowledge_base_id == "remote-one"
return {
"id": knowledge_base_id,
"vector_store_name": "System default",
"vector_store_source": "env",
"vector_store_engine_type": "elasticsearch",
"vector_store_status": "available",
"wiki_config": {"enabled": True},
}
async def list_wiki_pages(self, knowledge_base_id: str, **kwargs):
assert knowledge_base_id == "remote-one"
assert kwargs == {"page": 1, "page_size": 500, "query": ""}
return {
"pages": [
{
"id": "page-one",
"slug": "concept/start",
"title": "开始使用",
"page_type": "concept",
"status": "completed",
"created_at": "2026-07-02T08:00:00+08:00",
"updated_at": "2026-08-02T08:00:00+08:00",
},
{
"id": "page-two",
"slug": "summary/overview",
"title": "知识概览",
"page_type": "summary",
"status": "completed",
"created_at": "2026-07-03T08:00:00+08:00",
"updated_at": "2026-08-03T08:00:00+08:00",
},
],
"total": 2,
"page": 1,
"page_size": 500,
"total_pages": 1,
}
monkeypatch.setattr(llmwiki, "_client_or_503", lambda: Client())
payload = await llmwiki.list_public_knowledge_bases(_request(store))
assert store.calls == [("__public_wiki__", "public", False)]
assert payload == {
"knowledge_bases": [
{
"id": "1063edee-332a-43b4-bbd9-73bf0a708b94",
"name": "公开知识库",
}
],
"total": 1,
assert payload["total"] == 1
item = payload["knowledge_bases"][0]
assert item["id"] == "1063edee-332a-43b4-bbd9-73bf0a708b94"
assert item["name"] == "远端公开知识库"
assert item["description"] == "远端公开说明"
assert item["type"] == "wiki"
assert item["knowledge_count"] == 12
assert item["chunk_count"] == 34
assert item["processing_count"] == 1
assert item["share_count"] == 3
assert item["is_processing"] is True
assert item["remote_created_at"] == "2026-07-01T08:00:00+08:00"
assert item["remote_updated_at"] == "2026-08-01T08:00:00+08:00"
assert item["vector_store_engine_type"] == "elasticsearch"
assert item["wiki"] == {
"status": "ready",
"page_count": 2,
"loaded_page_count": 2,
"stats_complete": True,
"page_type_counts": {"concept": 1, "summary": 1},
"page_status_counts": {"completed": 2},
"index_mode": "generated",
"has_explicit_index": False,
"index_slug": "__index__",
"default_slug": "__index__",
"created_at": "2026-07-02T08:00:00+08:00",
"updated_at": "2026-08-03T08:00:00+08:00",
"embed_route": "/embed/knowledge/1063edee-332a-43b4-bbd9-73bf0a708b94?wikiSlug=__index__",
}
assert item["local_index"]["state"] == "not_synced"
assert item["local_index"]["fully_vectorized"] is False
assert item["owner_user_id"] is None
assert "weknora_id" not in item
assert "tenant_id" not in item
assert "creator_id" not in item
assert "storage_config" not in item
@pytest.mark.asyncio
@@ -178,6 +273,35 @@ async def test_public_wiki_page_list_is_normalized(monkeypatch) -> None:
assert payload["pages"][0]["aliases"] == []
def test_public_wiki_summary_prefers_an_explicit_index_page() -> None:
payload = llmwiki._public_wiki_summary(
"mapping-one",
[
{
"slug": "guide/start",
"title": "开始使用",
"page_type": "article",
"status": "completed",
},
{
"slug": "home",
"title": "知识首页",
"page_type": "index",
"status": "completed",
},
],
total=2,
complete=True,
)
assert payload["status"] == "ready"
assert payload["index_mode"] == "explicit"
assert payload["has_explicit_index"] is True
assert payload["index_slug"] == "home"
assert payload["default_slug"] == "home"
assert payload["embed_route"] == "/embed/knowledge/mapping-one?wikiSlug=home"
@pytest.mark.asyncio
async def test_public_wiki_file_is_authorized_and_streamed(monkeypatch) -> None:
store = PublicStore(_published_row())
@@ -0,0 +1,62 @@
import pytest
from deerflow.skill_knowledge.graph_writer import (
EDGE_QUERY,
NODE_QUERY,
execute_plan,
parameterized_statements,
plan_from_ir,
)
class RecordingTransaction:
def __init__(self) -> None:
self.calls: list[tuple[str, dict]] = []
async def run(self, query: str, parameters: dict) -> None:
self.calls.append((query, parameters))
def _plan():
return plan_from_ir(
{
"entities": [
{"local_id": "a", "type": "person", "name": "张三", "confidence": 1},
{"local_id": "b", "type": "system", "name": "知识库", "confidence": 1},
],
"relations": [
{
"source_local_id": "a",
"target_local_id": "b",
"predicate": "uses",
"confidence": 0.9,
"evidence": [{"file": "SKILL.md", "line": 2}],
}
],
},
target_type="weknora",
target_id="kb-1",
binding_id="binding-1",
snapshot_id="snapshot-1",
job_id="job-1",
)
def test_direct_graph_plan_uses_only_fixed_parameterized_templates() -> None:
statements = parameterized_statements(_plan())
assert len(statements) == 3
assert {row["statement"] for row in statements} == {NODE_QUERY, EDGE_QUERY}
assert all("parameters" in row for row in statements)
assert all("张三" not in row["statement"] for row in statements)
@pytest.mark.asyncio
async def test_direct_graph_writer_obeys_explicit_enable_switch() -> None:
transaction = RecordingTransaction()
disabled = await execute_plan(transaction, _plan(), enabled=False)
assert disabled == {"nodes": 0, "edges": 0}
assert transaction.calls == []
enabled = await execute_plan(transaction, _plan(), enabled=True)
assert enabled == {"nodes": 2, "edges": 1}
assert len(transaction.calls) == 3
@@ -0,0 +1,95 @@
import pytest
import pytest_asyncio
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from deerflow.persistence.base import Base
from deerflow.persistence.skill_knowledge import SkillKnowledgeRepository
@pytest_asyncio.fixture
async def repository():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as connection:
await connection.run_sync(Base.metadata.create_all)
yield SkillKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
await engine.dispose()
@pytest.mark.asyncio
async def test_matrix_job_creates_one_item_and_binding_per_pair(repository: SkillKnowledgeRepository) -> None:
job = await repository.create_job(
skill_names=["a", "b", "c"],
targets=[
{"target_type": "weknora", "target_mode": "wiki", "target_id": "kb-1", "target_name": "一"},
{"target_type": "assistant", "target_mode": "assistant", "target_id": "base", "target_name": "知识梳理"},
],
created_by="admin",
)
detail = await repository.get_job(job["id"])
bindings = await repository.list_bindings()
assert detail is not None
assert detail["total_count"] == 6
assert len(detail["items"]) == 6
assert len(bindings) == 6
@pytest.mark.asyncio
async def test_detach_delete_only_removes_binding_contributions(repository: SkillKnowledgeRepository) -> None:
job = await repository.create_job(
skill_names=["a", "b"],
targets=[{"target_type": "assistant", "target_mode": "assistant", "target_id": "base", "target_name": "知识梳理"}],
created_by="admin",
review_mode="off",
)
detail = await repository.get_job(job["id"])
assert detail is not None
binding = (await repository.list_bindings(skill_names=["a"]))[0]
detached = await repository.detach_binding(binding["id"], delete_contributions=True)
assert detached and detached["status"] == "detached_deleted"
assert len(await repository.list_bindings()) == 1
@pytest.mark.asyncio
async def test_item_claim_is_atomic_and_records_worker_lease(repository: SkillKnowledgeRepository) -> None:
job = await repository.create_job(
skill_names=["a"],
targets=[
{
"target_type": "assistant",
"target_mode": "assistant",
"target_id": "base",
"target_name": "知识梳理",
}
],
created_by="admin",
)
detail = await repository.get_job(job["id"])
item_id = detail["items"][0]["id"]
first = await repository.claim_item(item_id, owner="worker-a")
second = await repository.claim_item(item_id, owner="worker-b")
assert first and first["lease_owner"] == "worker-a"
assert first["attempts"] == 1
assert second is None
@pytest.mark.asyncio
async def test_binding_allows_only_one_active_item(repository: SkillKnowledgeRepository) -> None:
target = {
"target_type": "assistant",
"target_mode": "assistant",
"target_id": "base",
"target_name": "知识梳理",
}
first = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
duplicate = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
assert first["total_count"] == 1
assert duplicate["total_count"] == 0
assert duplicate["status"] == "completed"
detail = await repository.get_job(first["id"])
await repository.update_item(detail["items"][0]["id"], status="failed")
retry = await repository.create_job(skill_names=["a"], targets=[target], created_by="admin")
assert retry["total_count"] == 1
@@ -0,0 +1,52 @@
from pathlib import Path
from zipfile import ZipFile
from deerflow.skill_knowledge import build_skill_ir, scan_skill
def test_scanner_excludes_code_blocks_source_and_sensitive_files(tmp_path: Path) -> None:
(tmp_path / "SKILL.md").write_text(
"# 示例\n本技能使用 客户画像数据。\n```python\nPASSWORD = 'must-not-enter-ir'\n```\n",
encoding="utf-8",
)
(tmp_path / "worker.py").write_text("# uses hidden system\nSECRET='never-read'", encoding="utf-8")
(tmp_path / ".env").write_text("TOKEN=super-secret-value", encoding="utf-8")
scan = scan_skill("example", tmp_path)
ir = build_skill_ir(scan)
assert scan.skipped_code_count == 1
assert scan.blocked_files == [{"path": ".env", "reason": "疑似密钥或敏感配置文件"}]
assert "must-not-enter-ir" not in str(ir)
assert "never-read" not in str(ir)
assert any(relation["predicate"] == "uses" for relation in ir["relations"])
def test_code_only_change_does_not_change_digest(tmp_path: Path) -> None:
(tmp_path / "SKILL.md").write_text("# Stable knowledge", encoding="utf-8")
code = tmp_path / "worker.py"
code.write_text("print(1)", encoding="utf-8")
before = scan_skill("example", tmp_path).source_digest
code.write_text("print(2)", encoding="utf-8")
after = scan_skill("example", tmp_path).source_digest
assert before == after
def test_zip_is_inspected_and_only_safe_knowledge_members_are_read(tmp_path: Path) -> None:
archive = tmp_path / "materials.zip"
with ZipFile(archive, "w") as output:
output.writestr("guide.md", "# 操作手册\n仅提炼这段知识。")
output.writestr("worker.py", "SECRET = 'code-must-not-be-read'")
output.writestr("../escape.md", "path traversal")
scan = scan_skill("example", tmp_path)
assert [file.path for file in scan.files] == ["materials.zip!/guide.md"]
assert scan.skipped_code_count == 1
assert scan.blocked_files == [
{
"path": "materials.zip!/../escape.md",
"reason": "压缩包包含路径穿越或符号链接",
}
]
assert "code-must-not-be-read" not in str(scan.manifest)