Co-authored-by: Cursor <cursoragent@cursor.com>

# Conflicts:
#	frontend-web/src/components/workspace/workspace-nav-chat-list.tsx
#	frontend-web/src/pages/WorkspaceRoutes.tsx
#	offline-backend-20260512/backend/app/gateway/app.py
#	offline-backend-20260512/backend/app/gateway/deps.py
#	offline-backend-20260512/backend/pyproject.toml
This commit is contained in:
董宏凯 committed 2026-09-07 14:46:36 +08:00
commit cf9fe58905
109 files changed
+18863 -982

No files matched your search

+9 -2
View File
@@ -88,10 +88,15 @@ skills/
hermes-agent-main/
.gitignore
offline-backend-20260512/backend/tests/*
# Keep the openGauss URL/dialect/schema compatibility regression coverage versioned.
!offline-backend-20260512/backend/tests/test_gauss_database.py
!offline-backend-20260512/backend/tests/test_llmwiki_weknora.py
!offline-backend-20260512/backend/tests/test_llmwiki_knowledge_management.py
!offline-backend-20260512/backend/tests/test_health_auth_bypass.py
# Keep formal Word OOXML and embedded-font regression coverage versioned.
!offline-backend-20260512/backend/tests/test_word_export.py
# Keep generated-file library aggregation and filtering regression coverage versioned.
!offline-backend-20260512/backend/tests/test_artifact_library.py
# Keep the model configuration regression coverage versioned even though
# this deployment repository otherwise ignores newly added backend tests.
!offline-backend-20260512/backend/tests/test_model_management.py
@@ -171,8 +176,10 @@ docs/backups
# Frontend skills components must be tracked (excluded from the broad skills/ ignore above)
!frontend-web/src/components/workspace/skills/
# uv lockfile (not tracked per project preference)
offline-backend-20260512/backend/uv.lock
# uv lockfile (not tracked per project preference)
offline-backend-20260512/backend/uv.lock
!offline-backend-20260512/backend/tests/test_wiki_package_roundtrip.py
# Local-only scratch directory (not committed)
0622/
@@ -5,6 +5,9 @@ const LEGACY_LOCAL_API_BASE_URL = "http://localhost:8001";
const LEGACY_LOCAL_WEB_BASE_URL = "http://localhost:5174";
let pollTimer = null;
let isPolling = false;
let startupPromise = null;
const POLLING_SCHEDULE_SETTING_KEYS = new Set(["taskPollingEnabled", "taskPollIntervalSeconds"]);
function delay(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
@@ -12,6 +15,14 @@ function delay(ms) {
async function getSettings() {
const settings = await chrome.storage.sync.get(DEER_CLIPPER_DEFAULTS);
const discovered = await chrome.storage.local.get({
discoveredApiBaseUrl: "",
discoveredWebBaseUrl: "",
});
if (settings.autoDiscoverServiceUrl) {
settings.apiBaseUrl = discovered.discoveredApiBaseUrl || settings.apiBaseUrl;
settings.webBaseUrl = discovered.discoveredWebBaseUrl || settings.webBaseUrl;
}
const updates = {};
if (!settings.apiBaseUrl || settings.apiBaseUrl === LEGACY_LOCAL_API_BASE_URL) {
updates.apiBaseUrl = DEER_CLIPPER_DEFAULTS.apiBaseUrl;
@@ -69,10 +80,7 @@ function apiCandidatesForWebBase(webBase, currentApiBase) {
}
};
add(currentApiBase);
add(webBase);
add(DEER_CLIPPER_DEFAULTS.apiBaseUrl);
add(LEGACY_LOCAL_API_BASE_URL);
try {
const url = new URL(normalizeServiceBase(webBase));
@@ -85,6 +93,10 @@ function apiCandidatesForWebBase(webBase, currentApiBase) {
// Ignore invalid candidates.
}
add(currentApiBase);
add(DEER_CLIPPER_DEFAULTS.apiBaseUrl);
add(LEGACY_LOCAL_API_BASE_URL);
return candidates;
}
@@ -93,6 +105,12 @@ async function browserContextStatus(apiBaseUrl) {
const response = await fetch(`${apiBaseUrl}/api/browser-context/status`, {
credentials: "include",
});
// The gateway may enforce authentication globally before this otherwise
// public discovery endpoint is reached. A 401/403 still proves that this
// candidate is the zncm API paired with the open frontend tab.
if (response.status === 401 || response.status === 403) {
return { enabled: true, authenticationRequired: true };
}
if (!response.ok) {
return null;
}
@@ -201,9 +219,12 @@ async function discoverServiceBases(settings, force = false) {
settings.webBaseUrl !== next.webBaseUrl ||
settings.lastServiceSyncSource !== next.lastServiceSyncSource
) {
await chrome.storage.sync.set({
apiBaseUrl: next.apiBaseUrl,
webBaseUrl: next.webBaseUrl,
// Discovered endpoints are runtime state, not user-authored preferences.
// Keep them in storage.local so Chrome sync write quotas cannot break task
// polling. The options page may still persist explicit user choices.
await chrome.storage.local.set({
discoveredApiBaseUrl: next.apiBaseUrl,
discoveredWebBaseUrl: next.webBaseUrl,
lastServiceSyncAt: next.lastServiceSyncAt,
lastServiceSyncSource: next.lastServiceSyncSource,
});
@@ -826,19 +847,50 @@ async function pollOnce() {
}
}
async function startup() {
async function initializePolling() {
let settings = await getSettings();
settings = await discoverServiceBases(settings);
settings = await syncAuthToken(settings);
try {
settings = await discoverServiceBases(settings);
} catch (error) {
// Service discovery persists diagnostic metadata in storage.sync. Chrome can
// temporarily reject those writes after its quota is reached; polling must
// still start with the last known settings.
console.warn("zncm service discovery failed during startup", error);
}
try {
settings = await syncAuthToken(settings);
} catch (error) {
// A failed status/token write must not prevent alarms from being installed.
// pollOnce() will retry token synchronization before claiming each task.
console.warn("zncm token sync failed during startup", error);
}
await refreshAlarm(settings);
await scheduleNextPoll(1000);
}
chrome.runtime.onInstalled.addListener(() => void startup());
chrome.runtime.onStartup.addListener(() => void startup());
function startup() {
if (!startupPromise) {
startupPromise = initializePolling().finally(() => {
startupPromise = null;
});
}
return startupPromise;
}
async function restartPollingSchedule() {
const settings = await getSettings();
await refreshAlarm(settings);
await scheduleNextPoll(0);
}
chrome.runtime.onInstalled.addListener(() => void startup().catch((error) => console.warn("zncm startup failed", error)));
chrome.runtime.onStartup.addListener(() => void startup().catch((error) => console.warn("zncm startup failed", error)));
chrome.storage.onChanged.addListener((changes, areaName) => {
if (areaName === "sync") {
void startup();
if (
areaName === "sync" &&
Object.keys(changes || {}).some((key) => POLLING_SCHEDULE_SETTING_KEYS.has(key))
) {
void restartPollingSchedule().catch((error) => console.warn("zncm polling reschedule failed", error));
}
});
chrome.alarms.onAlarm.addListener((alarm) => {
@@ -902,4 +954,4 @@ chrome.runtime.onMessage.addListener((message, sender, sendResponse) => {
return true;
});
void startup();
void startup().catch((error) => console.warn("zncm startup failed", error));
@@ -0,0 +1,130 @@
const assert = require("node:assert/strict");
const fs = require("node:fs");
const path = require("node:path");
const test = require("node:test");
const vm = require("node:vm");
const BACKGROUND_PATH = path.join(__dirname, "background.js");
function createHarness() {
let storageChangedListener;
let alarmCreateCount = 0;
const settings = {
apiBaseUrl: "http://localhost:8001",
webBaseUrl: "http://localhost:5174",
autoDiscoverServiceUrl: false,
accessToken: "",
tokenSyncEnabled: false,
taskPollingEnabled: true,
workerId: "test-worker",
taskPollIntervalSeconds: 5,
};
const chrome = {
alarms: {
clear: async () => {},
create: async () => {
alarmCreateCount += 1;
},
onAlarm: { addListener: () => {} },
},
cookies: { get: async () => null },
runtime: {
onInstalled: { addListener: () => {} },
onMessage: { addListener: () => {} },
onStartup: { addListener: () => {} },
},
storage: {
local: {
get: async (defaults) => ({ ...defaults }),
set: async () => {},
},
session: { get: async () => ({ actSessions: {} }), set: async () => {} },
sync: {
get: async () => ({ ...settings }),
set: async (updates) => Object.assign(settings, updates),
},
onChanged: {
addListener: (listener) => {
storageChangedListener = listener;
},
},
},
tabs: { query: async () => [] },
};
const context = {
chrome,
console,
fetch: async () => {
throw new Error("offline test");
},
importScripts: () => {},
setTimeout: () => 1,
clearTimeout: () => {},
URL,
DEER_CLIPPER_DEFAULTS: settings,
deerNormalizeBaseUrl: (value) => String(value || "").replace(/\/+$/, ""),
deerMakeWorkerId: () => "test-worker",
deerAuthHeaders: () => ({}),
deerExtractAuthTokenFromPage: () => ({ token: "" }),
};
context.globalThis = context;
vm.runInNewContext(fs.readFileSync(BACKGROUND_PATH, "utf8"), context, {
filename: BACKGROUND_PATH,
});
return {
emitStorageChange(changes) {
storageChangedListener(changes, "sync");
},
getAlarmCreateCount: () => alarmCreateCount,
resetAlarmCreateCount: () => {
alarmCreateCount = 0;
},
getApiCandidates: (webBase, currentApiBase) =>
vm.runInContext(`apiCandidatesForWebBase(${JSON.stringify(webBase)}, ${JSON.stringify(currentApiBase)})`, context),
};
}
const settle = () => new Promise((resolve) => setImmediate(resolve));
test("self-written sync metadata does not restart browser task polling", async () => {
const harness = createHarness();
await settle();
harness.resetAlarmCreateCount();
harness.emitStorageChange({
lastTokenSyncAt: { oldValue: "before", newValue: "after" },
});
await settle();
assert.equal(harness.getAlarmCreateCount(), 0);
});
test("polling configuration changes restart browser task polling", async () => {
const harness = createHarness();
await settle();
harness.resetAlarmCreateCount();
harness.emitStorageChange({
taskPollingEnabled: { oldValue: false, newValue: true },
});
await settle();
assert.equal(harness.getAlarmCreateCount(), 1);
});
test("service discovery prefers the API paired with the open zncm tab", async () => {
const harness = createHarness();
assert.deepEqual(
Array.from(harness.getApiCandidates("http://localhost:5174", "http://47.94.209.59:2026")),
[
"http://localhost:5174",
"http://localhost:8001",
"http://47.94.209.59:2026",
"http://localhost:8001",
].filter((value, index, values) => values.indexOf(value) === index),
);
});
+45 -5
View File
@@ -7,6 +7,8 @@
const HOST_ID = "zncm-page-assistant-host";
const state = {
enabled: true,
taskPollingEnabled: true,
webBaseUrl: "",
open: false,
busy: false,
threadId: "",
@@ -709,20 +711,58 @@
async function loadSettings() {
const settings = await chrome.storage.sync.get(DEER_CLIPPER_DEFAULTS);
state.enabled = Boolean(settings.pageAssistantEnabled);
state.taskPollingEnabled = Boolean(settings.taskPollingEnabled);
state.webBaseUrl = deerNormalizeBaseUrl(settings.webBaseUrl || DEER_CLIPPER_DEFAULTS.webBaseUrl);
ensureWidget();
}
chrome.storage.onChanged.addListener((changes, areaName) => {
if (areaName !== "sync" || !changes.pageAssistantEnabled) {
if (areaName !== "sync") {
return;
}
state.enabled = Boolean(changes.pageAssistantEnabled.newValue);
ensureWidget();
if (state.enabled) {
setStatus("页面问答框已开启。");
if (changes.pageAssistantEnabled) {
state.enabled = Boolean(changes.pageAssistantEnabled.newValue);
ensureWidget();
if (state.enabled) {
setStatus("页面问答框已开启。");
}
}
if (changes.taskPollingEnabled) {
state.taskPollingEnabled = Boolean(changes.taskPollingEnabled.newValue);
}
if (changes.webBaseUrl) {
state.webBaseUrl = deerNormalizeBaseUrl(changes.webBaseUrl.newValue || DEER_CLIPPER_DEFAULTS.webBaseUrl);
}
});
function isZncmPage() {
const appPath = `${location.pathname}${location.hash}`.toLowerCase();
if (
appPath.includes("/page/workspace") ||
appPath.includes("/chats/") ||
appPath.includes("#/login")
) {
return true;
}
try {
const configured = new URL(state.webBaseUrl);
return configured.origin === location.origin;
} catch {
return false;
}
}
// MV3 service workers may be suspended between their short polling timers,
// while chrome.alarms cannot reliably run more often than every 30 seconds.
// A visible zncm tab provides a cheap wake-up signal so browser_act requests
// are normally claimed within the backend/model timeout window.
setInterval(() => {
if (!state.taskPollingEnabled || !isZncmPage()) {
return;
}
chrome.runtime.sendMessage({ type: "zncm-poll-now" }).catch(() => {});
}, 3000);
setInterval(() => {
if (!state.enabled || !ui.pageTitle) {
return;
+117
View File
@@ -0,0 +1,117 @@
# Wiki 数据流转与本地编码验收(2026-09-03)
## 结论
本地 BGE-M3 小批量编码已安装、配置并实际运行。已有 Wiki 的向量化、自动沉淀、
普通库文件导出、助手库异步导入、助手库文件导出、空普通库回流及两端向量检索已实测通过。
**技能自动生成 Wiki 尚未完成验收**:WeKnora 的两个生成模型都返回 HTTP 402
`Insufficient Balance`。不能把后半段通过等同于全链路通过。
本次修改运行在本地 DeerFlow 开发服务(前端 5175、后端 8001),连接用户提供的远端
WeKnora;没有把业务代码发布到服务器,没有提交 Git,也没有直接写 WeKnora 图数据库。
## 本地编码配置
- 实现:FastEmbed / ONNX Runtime CPU,官方 `BAAI/bge-m3` 权重。
- 配置名:`bge-embedding-m3`,1024 维,每批 2 条,2 个 CPU 推理线程。
- 运行依赖:FastEmbed 随标准后端依赖安装;模型必须提前放在离线目录中,仍按需加载,不影响主服务启动。
- 权重:约 2.3 GB,持久缓存 `C:\Users\ji\.cache\deerflow\embeddings`,不进入业务仓库。
- 配置:`llmwiki.local_wiki_index.embedding.provider: local`;可用
`local_model_path` 指定离线权重目录,详见后端 README。
- 查询向量在本地实时编码。已有 Wiki 在内容哈希、模型指纹匹配且向量非空时直接复用。
不同模型或维度不能混用;导入不静默重新编码。
- 原始文档分段向量与模型生成后的 Wiki 文本不同,不能相互冒用。本次新建的是 Wiki
正文的 BGE-M3 索引,不是把既有 512 维文档向量改名为 1024 维。
## 实际数据验收
来源是已有普通知识库“图片测试111”,没有改写其 Wiki 内容。
| 阶段 | Wiki | Wiki 向量 | 实体 | 引用关系 |
| --- | ---: | ---: | ---: | ---: |
| 普通知识库导出 | 19 | 101 | 19 | 108 |
| 助手知识库导入后导出 | 19 | 101 | 19 | 108 |
| 新建普通知识库回流后导出 | 19 | 101 | 19 | 108 |
| 知识梳理总库导出(7 个来源记录) | 19 | 101 | 19 | 108 |
逐项断言通过:
1. 三份包的 Wiki slug、标题、正文、摘要、页面类型、别名、分类路径、出链一致。
2. 101 条向量的原始 float32 字节、分段编号、维度和模型指纹完全一致。
3. 将导入前后实体 ID 映射回原 Wiki slug 后,108 条关系的端点及谓词集合完全一致。
4. 总库导出没有因多个来源重复同步而重复导出同一当前页面的分段向量。
5. 元数据回归测试另覆盖 tags、目录恢复、原始文档片段不计入 Wiki 数量、原生技能无
ownership 记录时可被选中、短 Wiki 不漏向量、模型指纹不兼容时不假装向量检索。
普通包约 1.01 MB,助手包约 0.79 MB,回流包约 0.83 MB。普通包还带 2 个原始文档及
108 个原始片段;助手导出提供 Wiki、Wiki 向量和关系,不重新上传原始文档触发二次生成。
### 检索
实际查询:“军方用什么工具评估极端天气带来的风险?”
- 回流普通库 `/api/llmwiki/search` 返回 `mode=vector`,首条“国防气候评估工具”。
- 助手库 `/api/assistant-knowledge/search` 返回 `retrieval_mode=vector`,首条同上;
首个命中分段余弦相似度约 0.6343。
- 浏览器实际提交总库检索,显示“向量检索 · 余弦相似度”和相同命中;不是只检查返回字段。
- 普通库采用已有页面级聚合评分,因此页面最终分数与单分段余弦值不必相等。
- 回流的向量写入 **DeerFlow 管理的本地 Wiki 索引**。未写入 WeKnora 原生向量数据库,
本次结论适用于本系统问答检索,不能声称原生 WeKnora 自身的检索也已完成该向量导入。
### 页面
普通与助手库复用同一 Wiki 阅读组件:目录树、类型/标签筛选、分类面包屑、别名标签、
正文内链、出链、反向链接及邻接关系视图。窄窗口可收起目录。
浏览器验证了“索引 → 国防气候评估工具”、展开关系视图、助手 Wiki 跳回普通库来源,
以及普通 Wiki 的“在知识梳理中查看”反向跳转。
### 验收数据位置
- 原有普通库:`48f669dc-433e-4aab-b5ea-897b6f7a3d2f`
- 知识梳理总库:`c3fd9841-0718-44dc-860e-aae3020fd485`
- 最终助手测试库:`489be905-08c8-40fe-8d60-42ac21dd945a`
- 最终普通回流测试库:`428162aa-03f6-4c61-8994-360f2f39e0c9`
新建测试库名称均以“Wiki链路验收-0903-1826”开头,保留用于复核;未删除用户已有库。
本地 `.runtime/wiki-live-result.json` 保存验收结果,`.runtime/wiki_live_acceptance.py`
保存本次验收流程,均为忽略的运行产物,不包含认证口令。
## 这次实际修复的问题
- 内置技能只有磁盘目录、没有归属记录时,被错误判定为不存在。
- 技能应上传去代码后的知识正文,让 WeKnora 生成 Wiki,而不是伪造一个 `skill` 类型页面。
- Wiki 生成等待使用实际 stats 接口;零页结果明确失败,不无期限等待。
- 向量化完成后自动导出沉淀到总库,并阻断导出回调重复触发全量同步。
- 短摘要和短正文同时存在时,旧分段阈值会将整页过滤为零向量。
- 分类路径必须先创建 WeKnora folder 并绑定 folder_id;只传 category_path 会被服务端清空。
- Wiki 的标签/分类/别名/原始引用保留;图节点绑定真实 Wiki,避免生成重复的空实体页面。
- 保留归一化 float32 原始字节;导入中不能用 `section_index or index`,因为合法编号 0 会被改写。
- 总库多个来源保留审计,但检索与导出只使用当前 Wiki 版本对应的向量,避免旧贡献污染检索。
- 包解析改为同盘临时 SQLite,上传先落盘再异步导入,导出完成后才提供文件下载。
## 阻塞与未通过的范围
1. 技能 `knowledge-base-ingest` 已送入专门创建的普通测试库,但远端
`deepseek-v4-flash` 和 `qwen3.6-flash` 均因余额不足失败。WeKnora 即使失败也可能把
文档 parse_status 标为 completed。需恢复可用生成模型后重新运行技能全链路验收。
2. 已采用磁盘暂存和有界批处理,但本次只有约 1 MB 实测,**未做数 GB 压测、断点续传、
进程中断恢复验收**。大包部署仍需磁盘容量、代理 body size 和超时配置。
3. 当前实体去重主要按类型及规范化名称;这次没有完成跨别名、歧义实体的模型对齐及
多来源事实冲突合并验收,不能视为这一历史需求已全面完成。
4. 重复上传仍会保留独立来源记录。向量导出已消除旧版本重复,但上传身份幂等、部分远端
写入失败后的无损重试、跨实例图片资源打包仍需独立验证。
5. 当前版本已补充自动增量链路:服务启动立即补扫,之后默认每 30 秒扫描全部普通
知识库;Wiki 抽取完成后无需手工点击向量化,并自动以来源快照覆盖同步到知识梳理总库。
6. 回流当前只允许新建空 Wiki 普通库,避免覆盖已有页面。普通库现有页面列表的超大规模
分页与助手问答的最终生成回答不属于这轮通过结论。
## 自动化检查
- 后端相关测试:118 passed(assistant repository、Wiki sync、package roundtrip、
WeKnora 接口、向量检索与知识库管理)。
- 修改的 Python 功能文件 Ruff 检查通过;app.py 既有启动顺序导致的 E402 不在此次改动中处理。
- `git diff --check` 通过(只有仓库换行提示)。
- 前端完整 `pnpm typecheck` 未通过:57 个既有/本轮范围外错误,涉及 nextra 缺失、Markdown
插件类型及其他模块;本轮新建/修改的 Wiki 组件、助手 API、技能面板没有出现在错误清单中。
浏览器实际页面与跳转已验证,但不能声称全项目构建通过。
@@ -0,0 +1,7 @@
# 技能知识归纳与“知识梳理”助手知识库
管理员在「技能管理 → 技能知识归纳」中选择一个或多个技能、一个或多个目标,系统按“技能 × 目标”创建后台任务。界面提供总体进度、单项失败重试、实体/关系审核、绑定重归纳、自动同步和两种解绑动作。技能列表通过一次 bindings 查询展示归纳目标,避免逐技能请求。
扫描仅处理知识资料。代码、脚本、锁文件、密钥文件以及 Markdown 代码块不会进入上传、摘要、提示词或知识指纹;ZIP 只读取逐项验证通过的知识成员。自动监听只比较该知识指纹,所以纯代码改动不会产生 stale 状态。
知识库页面新增「普通知识库 / 助手知识库」并行入口。管理员可将系统默认入口设为 v1(普通 WeKnora)或 v2(全局“知识梳理”)。助手库无需 WeKnora 即可初始化,所有用户可只读查看和搜索 Wiki;管理员可上传本地文件、从普通库导入、重新导入或删除来源、查看版本并回滚。来源删除只移除对应贡献,共享实体与关系继续保留。
File diff suppressed because it is too large. Load diff
+13 -1
View File
@@ -56,7 +56,7 @@ type PageSidebarProps = {
headerHidden?: boolean;
};
type SettingsSection = "account" | "appearance" | "memory" | "tools" | "about";
type SettingsSection = "account" | "appearance" | "memory" | "tools" | "encoder" | "about";
const hasActiveChild = (item: MenuItem, pathname: string): boolean =>
menuItemContainsPath(item, pathname);
@@ -270,6 +270,12 @@ function SecondaryNavSidebar({ headerHidden }: { headerHidden: boolean }) {
模型配置
</DropdownMenuItem>
)}
{isAdmin && (
<DropdownMenuItem onClick={() => openSettings("encoder")}>
<Cpu className="size-4" />
本地编码器
</DropdownMenuItem>
)}
{isAdmin && (
<DropdownMenuItem
onClick={() => navigate("/page/strategy/settings/model-concurrency")}
@@ -585,6 +591,12 @@ function LegacyAccordionSidebar({
模型配置
</DropdownMenuItem>
)}
{isAdmin && (
<DropdownMenuItem onClick={() => openSettings("encoder")}>
<Cpu className="size-4" />
本地编码器
</DropdownMenuItem>
)}
{isAdmin && (
<DropdownMenuItem onClick={() => navigate("/page/strategy/admin/login-whitelist")}>
<ShieldCheck className="size-4" />
@@ -1,7 +1,7 @@
import type { Message } from "@langchain/langgraph-sdk";
import { useQuery } from "@tanstack/react-query";
import { FilesIcon, XIcon } from "lucide-react";
import { usePathname } from "next/navigation";
import { usePathname, useSearchParams } from "next/navigation";
import { useEffect, useMemo, useRef, useState } from "react";
import type { GroupImperativeHandle } from "react-resizable-panels";
@@ -15,6 +15,7 @@ import {
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { apiFetch } from "@/core/api";
import {
artifactPathname,
listThreadArtifacts,
mergeArtifactPaths,
} from "@/core/artifacts/listing";
@@ -93,7 +94,13 @@ const ChatBox: React.FC<{
knowledgeReferencesEnabled = false,
} = useThread();
const pathname = usePathname();
const searchParams = useSearchParams();
const requestedArtifact = useMemo(() => {
const value = searchParams.get("artifact")?.trim();
return value ? artifactPathname(value) : null;
}, [searchParams]);
const threadIdRef = useRef(threadId);
const openedRequestedArtifactRef = useRef<string | null>(null);
const artifactsCountRef = useRef(0);
const layoutRef = useRef<GroupImperativeHandle>(null);
const latestReferenceBatchIdRef = useRef<string | null>(null);
@@ -133,7 +140,7 @@ const ChatBox: React.FC<{
const { skills } = useSkills();
const canScanArtifactDirectory =
artifactDirectoryModeEnabled &&
(artifactDirectoryModeEnabled || !!requestedArtifact) &&
!disableArtifacts &&
!isMock &&
!!threadId &&
@@ -151,17 +158,21 @@ const ChatBox: React.FC<{
);
const checkpointArtifactSignature = checkpointArtifacts.join("\n");
const availableArtifacts = useMemo(
() =>
artifactDirectoryModeEnabled
? mergeArtifactPaths(
checkpointArtifacts,
(artifactListingQuery.data?.files ?? []).map((file) => file.path),
)
: checkpointArtifacts,
() => {
if (!artifactDirectoryModeEnabled && !requestedArtifact) {
return checkpointArtifacts;
}
const discoveredArtifacts = (artifactListingQuery.data?.files ?? []).map(
(file) => file.path,
);
if (requestedArtifact) discoveredArtifacts.push(requestedArtifact);
return mergeArtifactPaths(checkpointArtifacts, discoveredArtifacts);
},
[
artifactDirectoryModeEnabled,
artifactListingQuery.data?.files,
checkpointArtifacts,
requestedArtifact,
],
);
@@ -205,6 +216,29 @@ const ChatBox: React.FC<{
thread.isLoading,
);
}, [referencesEnabled, skillDisplayMap, thread.isLoading, thread.values.messages]);
useEffect(() => {
if (
disableArtifacts ||
!requestedArtifact ||
!threadId ||
threadId === "new"
) {
return;
}
const requestKey = `${threadId}:${artifactPathname(requestedArtifact)}`;
if (openedRequestedArtifactRef.current === requestKey) return;
openedRequestedArtifactRef.current = requestKey;
selectArtifact(requestedArtifact);
setArtifactsOpen(true);
setSideTab("document");
}, [
disableArtifacts,
requestedArtifact,
selectArtifact,
setArtifactsOpen,
threadId,
]);
useEffect(() => {
if (!shouldFetchBackendReferenceBatches) {
setBackendReferenceBatches([]);
@@ -0,0 +1,57 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { BookOpen, Database } from "lucide-react";
import { useLocation, useNavigate } from "react-router-dom";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { getStoredAuth } from "@/core/auth";
import {
loadKnowledgeBaseProductVersion,
type KnowledgeBaseProductVersion,
updateKnowledgeBaseProductVersion,
} from "@/core/system-settings/api";
export const KNOWLEDGE_PRODUCT_KEY = ["system-settings", "knowledge-base-product-version"] as const;
export const KNOWLEDGE_NORMAL_VIEW_VALUE = "normal";
export const KNOWLEDGE_NORMAL_VIEW_SEARCH = `?view=${KNOWLEDGE_NORMAL_VIEW_VALUE}`;
export function useKnowledgeBaseProductVersion() {
return useQuery({ queryKey: KNOWLEDGE_PRODUCT_KEY, queryFn: loadKnowledgeBaseProductVersion, staleTime: 60_000 });
}
export function KnowledgeProductNav() {
const location = useLocation();
const navigate = useNavigate();
const client = useQueryClient();
const version = useKnowledgeBaseProductVersion();
const isAdmin = getStoredAuth()?.system_role === "admin";
const save = useMutation({
mutationFn: updateKnowledgeBaseProductVersion,
onSuccess: (value) => {
client.setQueryData(KNOWLEDGE_PRODUCT_KEY, value);
toast.success(`默认知识库体验已切换为 ${value}`);
},
});
const assistant = location.pathname.includes("/knowledge/assistant");
return (
<div className="flex items-center gap-2 border-b bg-card px-6 py-2">
<Button size="sm" variant={assistant ? "ghost" : "secondary"} onClick={() => navigate(`/page/workspace/knowledge${KNOWLEDGE_NORMAL_VIEW_SEARCH}`)}>
<BookOpen />普通知识库
</Button>
<Button size="sm" variant={assistant ? "secondary" : "ghost"} onClick={() => navigate("/page/workspace/knowledge/assistant")}>
<Database />助手知识库
</Button>
{isAdmin && (
<div className="ml-auto flex items-center gap-2 text-xs text-muted-foreground">
默认产品版本
<Select value={version.data ?? "v1"} onValueChange={(value) => save.mutate(value as KnowledgeBaseProductVersion)}>
<SelectTrigger className="h-8 w-40"><SelectValue /></SelectTrigger>
<SelectContent><SelectItem value="v1">v1 普通知识库</SelectItem><SelectItem value="v2">v2 助手知识库</SelectItem></SelectContent>
</Select>
</div>
)}
</div>
);
}
@@ -22,6 +22,10 @@ import {
} from "@/components/ui/dropdown-menu";
import { Input } from "@/components/ui/input";
import { cn } from "@/lib/utils";
import {
type AssistantKnowledgeBase,
useAssistantKnowledgeBases,
} from "@/strategy-components/api/assistant-knowledge";
import {
type LlmWikiKnowledgeBase,
useAgentLlmWikiKnowledgeBases,
@@ -66,10 +70,26 @@ function SpaceIcon({ item }: { item: LlmWikiKnowledgeBase }) {
);
}
function AssistantSpaceIcon({ item }: { item: AssistantKnowledgeBase }) {
return (
<span
className={cn(
"flex size-7 shrink-0 items-center justify-center rounded-md",
item.is_global ? "bg-amber-500/10 text-amber-600 dark:text-amber-400" : "bg-sky-500/10 text-sky-600 dark:text-sky-400",
)}
>
<Database className="size-3.5" />
</span>
);
}
function SelectionContent({
rows,
assistantRows,
value,
assistantValue,
onChange,
onAssistantChange,
search,
onSearchChange,
loading,
@@ -79,8 +99,11 @@ function SelectionContent({
readOnly,
}: {
rows: LlmWikiKnowledgeBase[];
assistantRows: AssistantKnowledgeBase[];
value: string[];
assistantValue: string[];
onChange: (value: string[]) => void;
onAssistantChange: (value: string[]) => void;
search: string;
onSearchChange: (value: string) => void;
loading: boolean;
@@ -90,6 +113,7 @@ function SelectionContent({
readOnly: boolean;
}) {
const selected = useMemo(() => new Set(value), [value]);
const selectedAssistant = useMemo(() => new Set(assistantValue), [assistantValue]);
const filtered = useMemo(() => {
const query = search.trim().toLocaleLowerCase();
if (!query) return rows;
@@ -97,11 +121,28 @@ function SelectionContent({
`${item.name} ${item.description}`.toLocaleLowerCase().includes(query),
);
}, [rows, search]);
const filteredAssistant = useMemo(() => {
const query = search.trim().toLocaleLowerCase();
if (!query) return assistantRows;
return assistantRows.filter((item) =>
`${item.name} ${item.description ?? ""}`.toLocaleLowerCase().includes(query),
);
}, [assistantRows, search]);
const groups = useMemo(() => groupRows(filtered), [filtered]);
const toggle = (id: string) => {
if (readOnly) return;
onChange(selected.has(id) ? value.filter((item) => item !== id) : [...value, id]);
};
const toggleAssistant = (id: string) => {
if (readOnly) return;
onAssistantChange(
selectedAssistant.has(id)
? assistantValue.filter((item) => item !== id)
: [...assistantValue, id],
);
};
const selectedCount = value.length + assistantValue.length;
const hasRows = groups.length > 0 || filteredAssistant.length > 0;
return (
<>
@@ -127,7 +168,7 @@ function SelectionContent({
<div className="m-2 rounded-lg border border-destructive/30 bg-destructive/5 p-3 text-xs text-destructive">
{error instanceof Error ? error.message : "知识空间读取失败"}
</div>
) : groups.length === 0 ? (
) : !hasRows ? (
<div className="flex h-28 items-center justify-center px-4 text-center text-xs text-muted-foreground">
{search
? "没有匹配的知识空间"
@@ -136,91 +177,181 @@ function SelectionContent({
: "当前账号暂无可用知识空间"}
</div>
) : (
groups.map((group, groupIndex) => (
<div key={group.key}>
{groupIndex > 0 && (
variant === "popover"
? <DropdownMenuSeparator className="my-1.5" />
: <div className="my-1.5 border-t" />
)}
{variant === "popover" ? (
<DropdownMenuLabel className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
{group.key === "personal" ? <Folder className="size-3" /> : <Globe2 className="size-3" />}{group.label}
</DropdownMenuLabel>
) : (
<div className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
{group.key === "personal" ? <Folder className="size-3" /> : <Globe2 className="size-3" />}{group.label}
</div>
)}
{group.rows.map((item) => {
const rowContent = (
<>
<SpaceIcon item={item} />
<div className="min-w-0 flex-1">
<div className="truncate text-xs font-medium">{item.name}</div>
<div className="mt-0.5 truncate text-[10px] text-muted-foreground">
{item.type === "faq" ? "FAQ" : "文档库"} · {item.knowledge_count ?? 0} 项知识
</div>
</div>
{item.publication_status === "published" && (
<Badge variant="outline" className="h-4 px-1 text-[9px]">公共</Badge>
)}
</>
);
return variant === "popover" ? (
<DropdownMenuCheckboxItem
key={item.id}
checked={selected.has(item.id)}
disabled={readOnly}
onCheckedChange={() => toggle(item.id)}
onSelect={(event) => event.preventDefault()}
className={cn(
"my-0.5 min-h-10 rounded-md py-1.5 pr-2 pl-7",
selected.has(item.id) && "bg-primary/8 focus:bg-primary/12",
)}
>
{rowContent}
</DropdownMenuCheckboxItem>
<>
{groups.map((group, groupIndex) => (
<div key={group.key}>
{groupIndex > 0 && (
variant === "popover"
? <DropdownMenuSeparator className="my-1.5" />
: <div className="my-1.5 border-t" />
)}
{variant === "popover" ? (
<DropdownMenuLabel className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
{group.key === "personal" ? <Folder className="size-3" /> : <Globe2 className="size-3" />}{group.label}
</DropdownMenuLabel>
) : (
<button
key={item.id}
type="button"
role="checkbox"
aria-checked={selected.has(item.id)}
disabled={readOnly}
onClick={() => toggle(item.id)}
className={cn(
"my-0.5 flex min-h-11 w-full items-center gap-2 rounded-md px-2 py-1.5 text-left outline-none transition-colors hover:bg-muted/60 focus-visible:ring-2 focus-visible:ring-primary",
selected.has(item.id) && "bg-primary/8",
)}
>
<span className={cn(
"flex size-4 shrink-0 items-center justify-center rounded border",
selected.has(item.id) && "border-primary bg-primary text-primary-foreground",
)}>
{selected.has(item.id) && <Check className="size-3" />}
</span>
{rowContent}
</button>
);
})}
</div>
))
<div className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
{group.key === "personal" ? <Folder className="size-3" /> : <Globe2 className="size-3" />}{group.label}
</div>
)}
{group.rows.map((item) => {
const rowContent = (
<>
<SpaceIcon item={item} />
<div className="min-w-0 flex-1">
<div className="truncate text-xs font-medium">{item.name}</div>
<div className="mt-0.5 truncate text-[10px] text-muted-foreground">
{item.type === "faq" ? "FAQ" : "文档库"} · {item.knowledge_count ?? 0} 项知识
</div>
</div>
{item.publication_status === "published" && (
<Badge variant="outline" className="h-4 px-1 text-[9px]">公共</Badge>
)}
</>
);
return variant === "popover" ? (
<DropdownMenuCheckboxItem
key={item.id}
checked={selected.has(item.id)}
disabled={readOnly}
onCheckedChange={() => toggle(item.id)}
onSelect={(event) => event.preventDefault()}
className={cn(
"my-0.5 min-h-10 rounded-md py-1.5 pr-2 pl-7",
selected.has(item.id) && "bg-primary/8 focus:bg-primary/12",
)}
>
{rowContent}
</DropdownMenuCheckboxItem>
) : (
<button
key={item.id}
type="button"
role="checkbox"
aria-checked={selected.has(item.id)}
disabled={readOnly}
onClick={() => toggle(item.id)}
className={cn(
"my-0.5 flex min-h-11 w-full items-center gap-2 rounded-md px-2 py-1.5 text-left outline-none transition-colors hover:bg-muted/60 focus-visible:ring-2 focus-visible:ring-primary",
selected.has(item.id) && "bg-primary/8",
)}
>
<span className={cn(
"flex size-4 shrink-0 items-center justify-center rounded border",
selected.has(item.id) && "border-primary bg-primary text-primary-foreground",
)}>
{selected.has(item.id) && <Check className="size-3" />}
</span>
{rowContent}
</button>
);
})}
</div>
))}
{filteredAssistant.length > 0 && (
<div>
{(groups.length > 0) && (
variant === "popover"
? <DropdownMenuSeparator className="my-1.5" />
: <div className="my-1.5 border-t" />
)}
{variant === "popover" ? (
<DropdownMenuLabel className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
<Database className="size-3" />助手知识库
</DropdownMenuLabel>
) : (
<div className="flex items-center gap-1.5 px-2 py-1 text-[11px] font-semibold text-muted-foreground">
<Database className="size-3" />助手知识库
</div>
)}
{filteredAssistant.map((item) => {
const rowContent = (
<>
<AssistantSpaceIcon item={item} />
<div className="min-w-0 flex-1">
<div className="truncate text-xs font-medium">{item.name}</div>
<div className="mt-0.5 truncate text-[10px] text-muted-foreground">
Wiki · {item.page_count ?? 0} 页 · {item.entity_count ?? 0} 实体
</div>
</div>
<Badge variant="outline" className="h-4 px-1 text-[9px]">
{item.is_global ? "总库" : "助手"}
</Badge>
</>
);
return variant === "popover" ? (
<DropdownMenuCheckboxItem
key={`assistant:${item.id}`}
checked={selectedAssistant.has(item.id)}
disabled={readOnly}
onCheckedChange={() => toggleAssistant(item.id)}
onSelect={(event) => event.preventDefault()}
className={cn(
"my-0.5 min-h-10 rounded-md py-1.5 pr-2 pl-7",
selectedAssistant.has(item.id) && "bg-primary/8 focus:bg-primary/12",
)}
>
{rowContent}
</DropdownMenuCheckboxItem>
) : (
<button
key={`assistant:${item.id}`}
type="button"
role="checkbox"
aria-checked={selectedAssistant.has(item.id)}
disabled={readOnly}
onClick={() => toggleAssistant(item.id)}
className={cn(
"my-0.5 flex min-h-11 w-full items-center gap-2 rounded-md px-2 py-1.5 text-left outline-none transition-colors hover:bg-muted/60 focus-visible:ring-2 focus-visible:ring-primary",
selectedAssistant.has(item.id) && "bg-primary/8",
)}
>
<span className={cn(
"flex size-4 shrink-0 items-center justify-center rounded border",
selectedAssistant.has(item.id) && "border-primary bg-primary text-primary-foreground",
)}>
{selectedAssistant.has(item.id) && <Check className="size-3" />}
</span>
{rowContent}
</button>
);
})}
</div>
)}
</>
)}
</div>
<div className="flex items-center justify-between gap-2 border-t bg-muted/20 px-2.5 py-2">
<span className="text-[11px] text-muted-foreground">
{readOnly ? `本对话已选 ${value.length} 个` : `已选择 ${value.length} 个`}
{readOnly ? `本对话已选 ${selectedCount} 个` : `已选择 ${selectedCount} 个`}
</span>
{readOnly ? (
<span className="text-[11px] text-muted-foreground">对话开始后不可更改</span>
) : (
<div className="flex items-center gap-1">
<Button type="button" variant="ghost" size="sm" className="h-7 px-2 text-xs" onClick={() => onChange(filtered.map((item) => item.id))}>
<Button
type="button"
variant="ghost"
size="sm"
className="h-7 px-2 text-xs"
onClick={() => {
onChange(filtered.map((item) => item.id));
onAssistantChange(filteredAssistant.map((item) => item.id));
}}
>
全选
</Button>
<Button type="button" variant="ghost" size="sm" className="h-7 px-2 text-xs" onClick={() => onChange([])}>
<Button
type="button"
variant="ghost"
size="sm"
className="h-7 px-2 text-xs"
onClick={() => {
onChange([]);
onAssistantChange([]);
}}
>
清空
</Button>
</div>
@@ -232,7 +363,9 @@ function SelectionContent({
export function LlmWikiKnowledgeSelector({
value,
assistantValue = [],
onChange,
onAssistantChange = () => undefined,
source = "all",
agentId,
autoSelectAll = false,
@@ -243,7 +376,9 @@ export function LlmWikiKnowledgeSelector({
variant = "popover",
}: {
value: string[];
assistantValue?: string[];
onChange: (value: string[]) => void;
onAssistantChange?: (value: string[]) => void;
source?: SelectorSource;
agentId?: string | null;
autoSelectAll?: boolean;
@@ -257,28 +392,39 @@ export function LlmWikiKnowledgeSelector({
const [open, setOpen] = useState(false);
const [search, setSearch] = useState("");
const runtime = useLlmWikiRuntime();
const weknoraEnabled = runtime.data?.provider === "weknora";
const all = useLlmWikiKnowledgeBases(
"all",
runtime.data?.provider === "weknora",
weknoraEnabled,
);
const agent = useAgentLlmWikiKnowledgeBases(
agentId,
runtime.data?.provider === "weknora" && source === "agent" && !!agentId,
weknoraEnabled && source === "agent" && !!agentId,
);
const rows = all.data ?? [];
const assistantBases = useAssistantKnowledgeBases();
const rows = weknoraEnabled ? (all.data ?? []) : [];
const assistantRows = assistantBases.data ?? [];
const defaultIds = useMemo(() => {
if (source !== "agent") {
return rows.map((item) => item.id);
}
return (agent.data ?? []).map((item) => item.id);
}, [agent.data, rows, source]);
const loading = all.isLoading || (source === "agent" && !!agentId && agent.isLoading);
const error = all.error ?? (source === "agent" ? agent.error : null);
const loading = runtime.isLoading || (weknoraEnabled && all.isLoading) || assistantBases.isLoading || (source === "agent" && !!agentId && agent.isLoading);
const error = (weknoraEnabled ? all.error ?? (source === "agent" ? agent.error : null) : null) ?? assistantBases.error;
const availableIds = useMemo(() => new Set(rows.map((item) => item.id)), [rows]);
const assistantAvailableIds = useMemo(
() => new Set(assistantRows.map((item) => item.id)),
[assistantRows],
);
const initializationKey = `${source}:${agentId ?? "all"}:${defaultIds.join(",")}`;
const initialized = useRef<string | null>(null);
useEffect(() => {
if (!runtime.isLoading && !weknoraEnabled) {
if (value.length > 0) onChange([]);
return;
}
if (!all.data || (source === "agent" && agentId && !agent.data) || initialized.current === initializationKey) return;
initialized.current = initializationKey;
const valid = value.filter((id) => availableIds.has(id));
@@ -288,19 +434,30 @@ export function LlmWikiKnowledgeSelector({
if (next.length !== value.length || next.some((id, index) => id !== value[index])) {
onChange(next);
}
}, [agent.data, agentId, all.data, autoSelectAll, availableIds, defaultIds, initializationKey, onChange, source, value]);
}, [agent.data, agentId, all.data, autoSelectAll, availableIds, defaultIds, initializationKey, onChange, runtime.isLoading, source, value, weknoraEnabled]);
useEffect(() => {
if (!assistantBases.data) return;
const valid = assistantValue.filter((id) => assistantAvailableIds.has(id));
if (valid.length !== assistantValue.length || valid.some((id, index) => id !== assistantValue[index])) {
onAssistantChange(valid);
}
}, [assistantAvailableIds, assistantBases.data, assistantValue, onAssistantChange]);
useEffect(() => {
if (!open) setSearch("");
}, [open]);
if (runtime.isError || runtime.data?.provider !== "weknora") return null;
if ((runtime.isError || !weknoraEnabled) && !assistantBases.isLoading && assistantRows.length === 0) return null;
const content = (
<SelectionContent
rows={rows}
assistantRows={assistantRows}
value={value}
assistantValue={assistantValue}
onChange={onChange}
onAssistantChange={onAssistantChange}
search={search}
onSearchChange={setSearch}
loading={loading}
@@ -323,7 +480,7 @@ export function LlmWikiKnowledgeSelector({
新建该智能体对话时,只能从已绑定空间中选择本次检索范围。
</p>
</div>
<Badge variant="secondary" className="shrink-0">{value.length} 个</Badge>
<Badge variant="secondary" className="shrink-0">{value.length + assistantValue.length} 个</Badge>
</div>
{content}
</div>
@@ -333,10 +490,15 @@ export function LlmWikiKnowledgeSelector({
const selectedNames = value
.map((id) => rows.find((item) => item.id === id)?.name)
.filter((name): name is string => !!name);
const selectedLabel = selectedNames.length > 0
? selectedNames.join("、")
: value.length > 0
? `已选 ${value.length} 个知识空间`
const selectedAssistantNames = assistantValue
.map((id) => assistantRows.find((item) => item.id === id)?.name)
.filter((name): name is string => !!name);
const allSelectedNames = [...selectedNames, ...selectedAssistantNames];
const selectedCount = value.length + assistantValue.length;
const selectedLabel = allSelectedNames.length > 0
? allSelectedNames.join("、")
: selectedCount > 0
? `已选 ${selectedCount} 个知识空间`
: label;
return (
@@ -348,20 +510,20 @@ export function LlmWikiKnowledgeSelector({
size="sm"
className={cn(
"h-8 max-w-52 gap-1.5 rounded-full border-0 bg-transparent px-2.5 text-muted-foreground shadow-none hover:bg-muted/70 hover:text-foreground",
value.length > 0 && "bg-primary/8 text-primary hover:bg-primary/12 hover:text-primary",
selectedCount > 0 && "bg-primary/8 text-primary hover:bg-primary/12 hover:text-primary",
className,
)}
disabled={disabled || loading}
title={value.length > 0 ? selectedLabel : undefined}
title={selectedCount > 0 ? selectedLabel : undefined}
>
{loading ? <Loader2 className="size-3.5 animate-spin" /> : <Database className="size-3.5" />}
<span className="truncate text-xs font-normal">{selectedLabel}</span>
{value.length > 0 && (
{selectedCount > 0 && (
<span className="flex size-4 items-center justify-center rounded-full bg-primary text-[9px] font-semibold text-primary-foreground">
{value.length}
{selectedCount}
</span>
)}
{value.length > 0 && value.length === rows.length && <Check className="size-3" />}
{selectedCount > 0 && selectedCount === rows.length + assistantRows.length && <Check className="size-3" />}
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="start" sideOffset={8} className="w-80 overflow-hidden rounded-xl p-0 shadow-xl">
@@ -1,6 +1,6 @@
"use client";
import { BellIcon, PaletteIcon, SparklesIcon, UserIcon, WrenchIcon } from "lucide-react";
import { BellIcon, CpuIcon, PaletteIcon, SparklesIcon, UserIcon, WrenchIcon } from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import {
@@ -14,16 +14,19 @@ import { AboutSettingsPage } from "@/components/workspace/settings/about-setting
import { AccountSettingsPage } from "@/components/workspace/settings/account-settings-page";
import { AppearanceSettingsPage } from "@/components/workspace/settings/appearance-settings-page";
import { MemorySettingsPage } from "@/components/workspace/settings/memory-settings-page";
import { ToolSettingsPage } from "@/components/workspace/settings/tool-settings-page";
import { useI18n } from "@/core/i18n/hooks";
import { cn } from "@/lib/utils";
import { ToolSettingsPage } from "@/components/workspace/settings/tool-settings-page";
import { getStoredAuth } from "@/core/auth";
import { useI18n } from "@/core/i18n/hooks";
import { cn } from "@/lib/utils";
import { LocalEncoderSettingsCard } from "@/strategy-components/components/resource-management/llmwiki/WeKnoraKnowledgeWorkspace";
type SettingsSection =
| "account"
| "appearance"
| "memory"
| "tools"
| "about";
| "memory"
| "tools"
| "encoder"
| "about";
type SettingsDialogProps = React.ComponentProps<typeof Dialog> & {
defaultSection?: SettingsSection;
@@ -31,7 +34,8 @@ type SettingsDialogProps = React.ComponentProps<typeof Dialog> & {
export function SettingsDialog(props: SettingsDialogProps) {
const { defaultSection = "appearance", ...dialogProps } = props;
const { t } = useI18n();
const { t } = useI18n();
const isAdmin = getStoredAuth()?.system_role === "admin";
const [activeSection, setActiveSection] =
useState<SettingsSection>(defaultSection);
@@ -63,7 +67,10 @@ export function SettingsDialog(props: SettingsDialogProps) {
// label: t.settings.sections.memory,
// icon: BrainIcon,
// },
{ id: "tools", label: t.settings.sections.tools, icon: WrenchIcon },
{ id: "tools", label: t.settings.sections.tools, icon: WrenchIcon },
...(isAdmin
? [{ id: "encoder", label: "本地编码器", icon: CpuIcon }]
: []),
{
id: "account",
label: "个人信息",
@@ -75,7 +82,8 @@ export function SettingsDialog(props: SettingsDialogProps) {
t.settings.sections.appearance,
t.settings.sections.memory,
t.settings.sections.tools,
t.settings.sections.about,
t.settings.sections.about,
isAdmin,
],
);
return (
@@ -125,7 +133,8 @@ export function SettingsDialog(props: SettingsDialogProps) {
<AppearanceSettingsPage onClose={() => props.onOpenChange?.(false)} />
)}
{activeSection === "memory" && <MemorySettingsPage />}
{activeSection === "tools" && <ToolSettingsPage />}
{activeSection === "tools" && <ToolSettingsPage />}
{activeSection === "encoder" && isAdmin && <LocalEncoderSettingsCard />}
{activeSection === "about" && <AboutSettingsPage />}
</div>
</ScrollArea>
@@ -93,6 +93,7 @@ import {
useViewMode,
} from "@/core/system-settings/hooks";
import type { Skill } from "@/core/skills/type";
import type { SkillKnowledgeBinding } from "@/strategy-components/api/skill-knowledge";
import { formatDateTime } from "@/core/utils/datetime";
import { env } from "@/env";
import { desensitize } from "@/lib/desensitize";
@@ -101,9 +102,16 @@ import { cn } from "@/lib/utils";
import { SettingsSection } from "./settings-section";
import { SkillStateSheet } from "./skill-state-sheet";
export function SkillSettingsPage({ onClose }: { onClose?: () => void } = {}) {
const { t } = useI18n();
return (
export function SkillSettingsPage({
onClose,
embedded = false,
}: {
onClose?: () => void;
embedded?: boolean;
} = {}) {
const { t } = useI18n();
if (embedded) return <SkillSettingsList onClose={onClose} />;
return (
<SettingsSection
title={t.settings.skills.title}
description={t.settings.skills.description}
@@ -601,6 +609,7 @@ export function SkillRow({
isAdmin,
layout = "row",
onPublisherClick,
knowledgeBindings = [],
}: {
skill: Skill;
currentUserId: string | null;
@@ -610,6 +619,8 @@ export function SkillRow({
/** Click the 发布者 line to filter the list to that publisher. Receives the
* owner_user_id + its desensitized display label. */
onPublisherClick?: (publisher: { id: string; label: string }) => void;
/** Admin list can inject its single batched binding query here (never N+1). */
knowledgeBindings?: SkillKnowledgeBinding[];
}) {
const isCard = layout === "card";
const { t } = useI18n();
@@ -858,6 +869,20 @@ export function SkillRow({
常驻
</Badge>
)}
{knowledgeBindings.slice(0, 3).map((binding) => (
<Badge
key={binding.id}
variant="outline"
className="border-indigo-500/30 text-indigo-700 dark:text-indigo-300 text-xs"
title={`${binding.target_name} · ${binding.status}${binding.remote_business_key ? ` · ${binding.remote_business_key}` : ""}`}
>
知识库:{binding.target_name || binding.target_id}
{binding.status === "stale" ? "(待更新)" : ""}
</Badge>
))}
{knowledgeBindings.length > 3 && (
<Badge variant="secondary" className="text-xs">+{knowledgeBindings.length - 3}</Badge>
)}
</div>
);
@@ -0,0 +1,280 @@
import { CheckCircle2, Database, Loader2, RefreshCw, Search, Sparkles } from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { Input } from "@/components/ui/input";
import { Progress } from "@/components/ui/progress";
import { ScrollArea } from "@/components/ui/scroll-area";
import { useAllSkillsAdmin, useSkills } from "@/core/skills/hooks";
import { useLlmWikiKnowledgeBases, useLlmWikiRuntime } from "@/strategy-components/api/llmwiki";
import {
type SkillKnowledgeTarget,
useSkillKnowledgeActions,
useSkillKnowledgeBindings,
useSkillKnowledgeJobs,
} from "@/strategy-components/api/skill-knowledge";
const PHASE_LABELS: Record<string, string> = {
scanning: "扫描知识文件",
parsing: "解析资料",
extracting: "抽取实体关系",
awaiting_review: "等待审核",
merging: "合并实体关系",
writing_wiki: "写入 Wiki",
writing_sources: "上传知识资料",
generating_wiki: "分析并生成分类 Wiki",
vectorizing_wiki: "Wiki 向量化",
syncing_assistant: "同步知识梳理总库",
activating: "激活新版本",
completed: "完成",
};
const STATUS_LABELS: Record<string, string> = {
queued: "等待处理",
running: "处理中",
awaiting_review: "等待管理员审核",
completed: "已完成",
partial_failed: "部分失败",
failed: "失败",
canceled: "已取消",
};
export function SkillKnowledgePanel({ adminMode = false }: { adminMode?: boolean }) {
const visibleSkills = useSkills();
const allSkills = useAllSkillsAdmin(adminMode);
const skills = adminMode ? allSkills.skills : visibleSkills.skills;
const isLoading = adminMode ? allSkills.isLoading : visibleSkills.isLoading;
const liveSkills = useMemo(() => skills.filter((skill) => (skill.state ?? "active") === "active"), [skills]);
const [selectedSkills, setSelectedSkills] = useState<string[]>([]);
const [skillQuery, setSkillQuery] = useState("");
const [selectedTargetKey, setSelectedTargetKey] = useState<string | null>(null);
const runtime = useLlmWikiRuntime();
const knowledgeBases = useLlmWikiKnowledgeBases("all", runtime.data?.provider === "weknora");
const bindings = useSkillKnowledgeBindings(undefined, !adminMode);
const jobs = useSkillKnowledgeJobs(!adminMode);
const actions = useSkillKnowledgeActions();
const targets = useMemo<SkillKnowledgeTarget[]>(() => {
return (knowledgeBases.data ?? [])
.filter((kb) => {
const strategy = kb.indexing_strategy ?? {};
const capabilities = kb.capabilities ?? {};
const wikiEnabled =
kb.type === "wiki" ||
kb.wiki_index_enabled === true ||
strategy.wiki_enabled === true ||
capabilities.wiki_enabled === true ||
(kb.wiki_config && Object.keys(kb.wiki_config).length > 0);
return kb.can_write && kb.remote_status !== "missing" && !kb.is_conversation_deposit && kb.type !== "faq" && wikiEnabled;
})
.map((kb) => ({
target_type: "weknora",
target_mode: "wiki",
target_id: kb.id,
target_name: kb.name,
remote_business_key: `weknora:kb:${kb.id}`,
}));
}, [knowledgeBases.data]);
const filteredSkills = useMemo(() => {
const needle = skillQuery.trim().toLowerCase();
if (!needle) return liveSkills;
return liveSkills.filter((skill) => {
const tagText = (skill.tags ?? []).map((tag) => tag.name).join(" ");
const haystack = [
skill.name,
skill.name_zh,
skill.description,
skill.category,
tagText,
]
.filter(Boolean)
.join("\n")
.toLowerCase();
return haystack.includes(needle);
});
}, [liveSkills, skillQuery]);
const filteredSkillNames = useMemo(() => filteredSkills.map((skill) => skill.name), [filteredSkills]);
const allFilteredSelected = filteredSkillNames.length > 0 && filteredSkillNames.every((name) => selectedSkills.includes(name));
const toggleFilteredSkills = () => {
if (!filteredSkillNames.length) return;
setSelectedSkills((current) => {
if (allFilteredSelected) {
const filtered = new Set(filteredSkillNames);
return current.filter((name) => !filtered.has(name));
}
return [...current, ...filteredSkillNames.filter((name) => !current.includes(name))];
});
};
useEffect(() => {
if (selectedTargetKey && !targets.some((target) => `${target.target_type}:${target.target_id}` === selectedTargetKey)) {
setSelectedTargetKey(null);
}
}, [selectedTargetKey, targets]);
const toggle = (values: string[], value: string, setter: (next: string[]) => void) =>
setter(values.includes(value) ? values.filter((item) => item !== value) : [...values, value]);
const submit = async () => {
const selectedTargets = targets.filter((target) =>
selectedTargetKey === `${target.target_type}:${target.target_id}`,
);
if (!selectedSkills.length || !selectedTargets.length) {
toast.error("请至少选择一个技能和一个目标知识库");
return;
}
try {
const job = await actions.createJob.mutateAsync({
skill_names: selectedSkills,
targets: selectedTargets,
review_mode: "off",
});
toast.success(`已创建 ${job.total_count} 个归纳项`);
setSelectedSkills([]);
} catch (error) {
toast.error(error instanceof Error ? error.message : String(error));
}
};
return (
<div className="grid size-full min-h-0 gap-4 overflow-auto p-4 lg:grid-cols-[minmax(320px,0.9fr)_minmax(420px,1.1fr)] lg:overflow-hidden lg:p-6">
<section className="flex min-h-0 flex-col rounded-lg border bg-card">
<div className="border-b p-4">
<h2 className="font-semibold">归纳技能知识</h2>
<p className="mt-1 text-xs leading-5 text-muted-foreground">选择技能和一个目标知识库后,系统会在后台提取 Markdown 等知识资料,忽略 Python 等代码文件,自动生成 Wiki、关系和向量索引。</p>
</div>
<ScrollArea className="min-h-0 flex-1">
<div className="space-y-5 p-4">
<div>
<div className="mb-2 flex items-center justify-between">
<span className="text-sm font-medium">第一步:选择技能(已选 {selectedSkills.length})</span>
<Button variant="ghost" size="sm" onClick={toggleFilteredSkills}>
{allFilteredSelected ? "清空当前结果" : "选择当前结果"}
</Button>
</div>
<div className="relative mb-2">
<Search className="absolute left-2.5 top-1/2 size-4 -translate-y-1/2 text-muted-foreground" />
<Input
value={skillQuery}
onChange={(event) => setSkillQuery(event.target.value)}
placeholder="按技能中文名、英文名、描述、分类或标签搜索"
className="pl-8"
/>
</div>
<div className="max-h-60 space-y-1 overflow-y-auto rounded-md border p-2">
{isLoading ? <Loader2 className="m-4 size-5 animate-spin" /> : filteredSkills.map((skill) => (
<label key={skill.name} className="flex cursor-pointer items-center gap-2 rounded px-2 py-1.5 hover:bg-muted">
<Checkbox checked={selectedSkills.includes(skill.name)} onCheckedChange={() => toggle(selectedSkills, skill.name, setSelectedSkills)} />
<span className="truncate text-sm">{skill.name_zh || skill.name}</span>
<span className="ml-auto truncate text-xs text-muted-foreground">{skill.name}</span>
{(bindings.data ?? []).some((binding) => binding.skill_name === skill.name) && (
<Badge variant="secondary" className="shrink-0 text-[10px]">
{(bindings.data ?? []).some((binding) => binding.skill_name === skill.name && `${binding.target_type}:${binding.target_id}` === selectedTargetKey)
? "已在目标库"
: `已归纳 ${(bindings.data ?? []).filter((binding) => binding.skill_name === skill.name).length} 库`}
</Badge>
)}
</label>
))}
{!isLoading && !filteredSkills.length && (
<p className="p-3 text-xs text-muted-foreground">没有匹配的技能,请换个关键词。</p>
)}
</div>
<p className="mt-1 text-xs text-muted-foreground">
当前显示 {filteredSkills.length}/{liveSkills.length} 个技能。
</p>
</div>
<div>
<div className="mb-2 flex items-center justify-between">
<span className="text-sm font-medium">第二步:选择一个目标知识库</span>
</div>
<div className="space-y-1 rounded-md border p-2">
{targets.map((target) => {
const key = `${target.target_type}:${target.target_id}`;
return (
<label key={key} className="flex cursor-pointer items-center gap-2 rounded px-2 py-2 hover:bg-muted">
<Checkbox
checked={selectedTargetKey === key}
onCheckedChange={() => setSelectedTargetKey((current) => (current === key ? null : key))}
/>
<Database className="size-4 text-muted-foreground" />
<span className="text-sm">{target.target_name}</span>
<Badge variant="outline" className="ml-auto text-[10px]">WeKnora Wiki</Badge>
</label>
);
})}
{(runtime.isLoading || knowledgeBases.isLoading) && <Loader2 className="m-4 size-5 animate-spin" />}
{!runtime.isLoading && !knowledgeBases.isLoading && knowledgeBases.error && (
<p className="p-3 text-xs text-destructive">
读取 WeKnora 普通知识库失败:
{knowledgeBases.error instanceof Error ? knowledgeBases.error.message : String(knowledgeBases.error)}
</p>
)}
{!runtime.isLoading && !knowledgeBases.isLoading && !knowledgeBases.error && !targets.length && (
<p className="p-3 text-xs text-muted-foreground">
{runtime.data?.provider !== "weknora"
? "当前未配置 WeKnora 普通知识库地址;技能需要先归纳到普通知识库,再导入助手知识库。"
: "当前没有可写且已开启 Wiki 的普通知识库;请确认该库不是 FAQ/会话沉淀库,并已开启 Wiki。"}
</p>
)}
</div>
</div>
<div className="rounded-lg border bg-muted/30 p-3 text-xs leading-5 text-muted-foreground">
第三步:提交后可离开本页。归纳、Wiki 生成、向量化及同步到“知识梳理”总库均在后台完成。
</div>
<Button className="w-full" disabled={actions.createJob.isPending || !selectedSkills.length || !selectedTargetKey} onClick={() => void submit()}>
{actions.createJob.isPending ? <Loader2 className="animate-spin" /> : <Sparkles />}
开始归纳 {selectedSkills.length ? `${selectedSkills.length} 个技能` : ""}
</Button>
</div>
</ScrollArea>
</section>
<section className="flex min-h-0 flex-col gap-4 overflow-hidden">
<div className="flex min-h-0 flex-1 flex-col rounded-lg border bg-card">
<div className="flex items-center justify-between border-b p-4">
<div><h2 className="font-semibold">我的归纳任务</h2><p className="text-xs text-muted-foreground">任务会自动刷新,失败项可直接重试</p></div>
<Button size="icon" variant="ghost" onClick={() => void jobs.refetch()}><RefreshCw className="size-4" /></Button>
</div>
<ScrollArea className="min-h-0 flex-1">
<div className="space-y-3 p-4">
{(jobs.data ?? []).map((job) => {
const countedDone = job.success_count + job.partial_failed_count + job.failed_count;
const itemDone = (job.items ?? []).filter((item) =>
["completed", "partial_failed", "failed", "canceled"].includes(item.status),
).length;
const done = Math.max(countedDone, itemDone);
const progress = job.total_count ? Math.round((done / job.total_count) * 100) : 0;
return (
<div key={job.id} className="rounded-md border p-3">
<div className="flex items-center gap-2 text-sm"><Badge variant={job.status === "completed" ? "default" : "secondary"}>{STATUS_LABELS[job.status] || job.status}</Badge><span>{done}/{job.total_count} 项</span><span className="ml-auto text-xs text-muted-foreground">{new Date(job.created_at).toLocaleString()}</span></div>
<Progress className="my-2" value={progress} />
{job.items?.map((item) => (
<div key={item.id} className="mt-2 flex items-center gap-2 border-t pt-2 text-xs">
<span className="max-w-36 truncate font-medium">{item.skill_name}</span>
<span className="text-muted-foreground">{PHASE_LABELS[item.phase] || STATUS_LABELS[item.status] || item.phase}</span>
<span className="ml-auto">{item.progress}%</span>
{item.status === "failed" && <Button size="sm" variant="outline" onClick={() => actions.retry.mutate(item.id)}>重试</Button>}
</div>
))}
</div>
);
})}
{!jobs.data?.length && <p className="py-10 text-center text-sm text-muted-foreground">暂无归纳任务</p>}
</div>
</ScrollArea>
</div>
<div className="max-h-80 overflow-auto rounded-lg border bg-card p-3 text-xs">
<div className="mb-2 flex items-center gap-2 text-muted-foreground"><CheckCircle2 className="size-4 text-emerald-600" />已归纳关联:{bindings.data?.length ?? 0}</div>
<div className="space-y-2">{bindings.data?.map((binding) => <div key={binding.id} className="flex flex-wrap items-center gap-2 rounded border p-2"><span className="max-w-40 truncate font-medium">{binding.skill_name}</span><span className="max-w-48 truncate text-muted-foreground">已归纳到 {binding.target_name}</span><Badge variant="outline">{STATUS_LABELS[binding.status] || binding.status}</Badge><Button size="sm" variant="outline" className="ml-auto" disabled={actions.resync.isPending} onClick={() => actions.resync.mutate(binding.id)}>重新归纳</Button><Button size="sm" variant="ghost" onClick={() => { if (window.confirm("解除关联并保留目标知识内容?")) actions.detach.mutate({ id: binding.id, deleteContributions: false }); }}>解除关联</Button>{adminMode && <Button size="sm" variant="destructive" onClick={() => { if (window.confirm("解除关联并删除此技能贡献?共享知识仍会保留其他来源。")) actions.detach.mutate({ id: binding.id, deleteContributions: true }); }}>删除贡献</Button>}</div>)}</div>
</div>
</section>
</div>
);
}
@@ -14,7 +14,7 @@ import {
SidebarMenuItem,
} from "@/components/ui/sidebar";
import { useI18n } from "@/core/i18n/hooks";
import { BookOpen, BotIcon, BrainIcon, FileSearch, FileText, ListChecks, MessageSquarePlus, MessagesSquare, PenLine, Plus, SparklesIcon, Telescope, Users } from "lucide-react";
import { BookOpen, BotIcon, BrainIcon, FileSearch, FileText, Library, ListChecks, MessageSquarePlus, MessagesSquare, PenLine, Plus, SparklesIcon, Telescope, Users } from "lucide-react";
import { KnowledgeRagToggle } from "./knowledge-rag-toggle";
import { SidebarCollapseButton } from "./workspace-header";
@@ -65,6 +65,13 @@ export const WORKSPACE_COMPACT_NAV_ITEMS: WorkspaceCompactNavItem[] = [
path: "/page/workspace/chats",
match: (pathname) => pathname === "/page/workspace/chats",
},
{
id: "workspace-library",
label: "资料库",
icon: Library,
path: "/page/workspace/library",
match: (pathname) => pathname === "/page/workspace/library",
},
{
id: "workspace-agents",
label: "智能体",
@@ -83,7 +83,7 @@ export function WorkspaceNavMenu() {
const isAdmin = getStoredAuth()?.system_role === "admin";
const [settingsOpen, setSettingsOpen] = useState(false);
const [settingsDefaultSection, setSettingsDefaultSection] = useState<
"appearance" | "memory" | "tools" | "about"
"appearance" | "memory" | "tools" | "encoder" | "about"
>("appearance");
const [notificationOpen, setNotificationOpen] = useState(false);
const [mounted, setMounted] = useState(false);
@@ -189,6 +189,17 @@ export function WorkspaceNavMenu() {
<Settings2Icon />
{t.common.settings}
</DropdownMenuItem>
{isAdmin && (
<DropdownMenuItem
onClick={() => {
setSettingsDefaultSection("encoder");
setSettingsOpen(true);
}}
>
<CpuIcon />
本地编码器
</DropdownMenuItem>
)}
{isAdmin && (
<DropdownMenuItem
onClick={() =>
+24 -5
View File
@@ -1,5 +1,6 @@
import { apiFetch } from "../api";
import { getFileName } from "../utils/files";
import { apiFetch } from "../api";
import { downloadMarkdownAsDocx } from "../scheduled-tasks/markdown-docx";
import { getFileName } from "../utils/files";
import { urlOfArtifact } from "./utils";
@@ -39,7 +40,7 @@ async function errorMessageFromResponse(res: Response): Promise<string> {
return res.statusText || "Request failed";
}
export async function downloadArtifactFile({
export async function downloadArtifactFile({
filepath,
threadId,
isMock,
@@ -73,8 +74,26 @@ export async function downloadArtifactFile({
document.body.appendChild(a);
a.click();
a.remove();
window.setTimeout(() => URL.revokeObjectURL(objectUrl), 30_000);
}
window.setTimeout(() => URL.revokeObjectURL(objectUrl), 30_000);
}
export async function downloadArtifactMarkdownAsWord({
filepath,
threadId,
}: {
filepath: string;
threadId: string;
}): Promise<void> {
const url = urlOfArtifact({ filepath, threadId, download: false });
const res = await apiFetch(url);
if (!res.ok) {
throw new Error(await errorMessageFromResponse(res));
}
const fileName = getFileName(filepath);
const title = fileName.replace(/\.(md|markdown)$/i, "");
await downloadMarkdownAsDocx(await res.text(), fileName, title);
}
export async function openArtifactInNewTab({
filepath,
+113
View File
@@ -0,0 +1,113 @@
import { apiFetch } from "@/core/api";
import { getBackendBaseURL } from "@/core/config";
export type ArtifactLibraryKind =
| "all"
| "document"
| "spreadsheet"
| "text_data"
| "archive"
| "other";
export type ArtifactLibrarySource = "all" | "general_qa" | "agent";
export interface ArtifactLibraryItem {
threadId: string;
threadTitle: string;
agentId: string | null;
agentName: string | null;
threadType: string | null;
sourceType: Exclude<ArtifactLibrarySource, "all">;
path: string;
name: string;
sizeBytes: number;
mimeType: string | null;
modifiedAt: string;
fileKind: Exclude<ArtifactLibraryKind, "all">;
}
export interface ArtifactLibraryResponse {
total: number;
offset: number;
limit: number;
items: ArtifactLibraryItem[];
}
export async function fetchArtifactLibrary({
query,
fileKind,
sourceType,
offset,
limit,
refresh = false,
signal,
}: {
query?: string;
fileKind: ArtifactLibraryKind;
sourceType: ArtifactLibrarySource;
offset: number;
limit: number;
refresh?: boolean;
signal?: AbortSignal;
}): Promise<ArtifactLibraryResponse> {
const search = new URLSearchParams({
file_kind: fileKind,
source_type: sourceType,
offset: String(offset),
limit: String(limit),
});
const normalizedQuery = query?.trim();
if (normalizedQuery) search.set("query", normalizedQuery);
if (refresh) search.set("refresh", "true");
const response = await apiFetch(
`${getBackendBaseURL()}/api/artifact-library?${search.toString()}`,
{ method: "GET", signal },
);
if (!response.ok) {
const detail = await response.text().catch(() => "");
throw new Error(
`Failed to load artifact library (${response.status})${detail ? `: ${detail}` : ""}`,
);
}
const data = (await response.json()) as {
total?: number;
offset?: number;
limit?: number;
items?: Array<{
thread_id: string;
thread_title: string;
agent_id?: string | null;
agent_name?: string | null;
thread_type?: string | null;
source_type?: Exclude<ArtifactLibrarySource, "all">;
path: string;
name: string;
size_bytes: number;
mime_type?: string | null;
modified_at: string;
file_kind: Exclude<ArtifactLibraryKind, "all">;
}>;
};
return {
total: data.total ?? 0,
offset: data.offset ?? offset,
limit: data.limit ?? limit,
items: (data.items ?? []).map((item) => ({
threadId: item.thread_id,
threadTitle: item.thread_title,
agentId: item.agent_id ?? null,
agentName: item.agent_name ?? item.agent_id ?? null,
threadType: item.thread_type ?? null,
sourceType: item.source_type ?? (item.agent_id ? "agent" : "general_qa"),
path: item.path,
name: item.name,
sizeBytes: item.size_bytes,
mimeType: item.mime_type ?? null,
modifiedAt: item.modified_at,
fileKind: item.file_kind,
})),
};
}
@@ -149,7 +149,7 @@ export function buildWikiIndexCatalogMarkdown(
buildPageUrl: WikiPageUrlBuilder = buildWeKnoraWikiPageUrl,
indexGroups: WikiIndexGroup[] = [],
): string {
if (indexGroups.length > 0) {
if (indexGroups.some((group) => group.items.length > 0)) {
const sections: string[] = [];
for (const group of indexGroups) {
if (group.items.length === 0) continue;
@@ -61,6 +61,7 @@ export const sidebarMenuItems: MenuItem[] = [
icon: HelpCircle,
children: [
{ id: "general-qa", label: "通用问答", icon: CircleHelp, path: "/page/workspace/chats/new" },
{ id: "artifact-library", label: "资料库", icon: Library, path: "/page/workspace/library" },
{
id: "multi-agent-roundtable",
label: "多智能体会商",
@@ -9,6 +9,31 @@ export interface PromptPrefixSettings {
ordinary_qa_markdown_format_enabled: boolean;
}
export type KnowledgeBaseProductVersion = "v1" | "v2";
export async function loadKnowledgeBaseProductVersion(): Promise<KnowledgeBaseProductVersion> {
const response = await apiFetch(
`${getBackendBaseURL()}/api/system-settings/knowledge-base-product-version`,
);
if (!response.ok) throw new Error(`Failed to load knowledge base product version: ${response.status}`);
return ((await response.json()) as { version: KnowledgeBaseProductVersion }).version;
}
export async function updateKnowledgeBaseProductVersion(
version: KnowledgeBaseProductVersion,
): Promise<KnowledgeBaseProductVersion> {
const response = await apiFetch(
`${getBackendBaseURL()}/api/system-settings/knowledge-base-product-version`,
{
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ version }),
},
);
if (!response.ok) throw new Error(`Failed to update knowledge base product version: ${response.status}`);
return ((await response.json()) as { version: KnowledgeBaseProductVersion }).version;
}
export interface PluginDomainAgentMappingDto {
id: string;
domain: string;
+9 -3
View File
@@ -1512,10 +1512,16 @@ export function useThreadStream({
)
? mergedContext.llmwiki_knowledge_base_ids
: [];
const selectedAssistantKnowledgeIds = Array.isArray(
mergedContext.assistant_knowledge_base_ids,
)
? mergedContext.assistant_knowledge_base_ids
: [];
const ragModeEnabled =
(referenceModeEntryEnabled && localSettings.display.ragModeEnabled) ||
isThreadRagModeLocked(threadId) ||
selectedLlmWikiIds.length > 0;
selectedLlmWikiIds.length > 0 ||
selectedAssistantKnowledgeIds.length > 0;
if (ragModeEnabled) {
lockThreadRagMode(threadId);
}
@@ -1693,7 +1699,7 @@ export function useThreadStream({
? undefined
: thread.error;
const mergedThread = {
const mergedThread = {
...thread,
error: visibleThreadError,
values: mergedValues,
@@ -1703,7 +1709,7 @@ export function useThreadStream({
: mergedMessages,
isLoading: thread.isLoading || streamHoldOpen,
stop: handleThreadStop,
} as typeof thread;
} as typeof thread;
// 第 4 个返回值 `onStreamThreadId` = 底层 useStream 当前实际绑定的线程 id(新建/未保存时为 null)。
// 调用方据此判断「流是否已切到全新线程」,避免在切换会话的那一帧把消息发进上一个会话
+2
View File
@@ -27,6 +27,8 @@ export interface AgentThreadContext extends Record<string, unknown> {
knowledge_rag_enabled?: boolean;
/** DeerFlow LLMWiki mapping ids selected for this conversation. */
llmwiki_knowledge_base_ids?: string[];
/** Assistant knowledge-base ids selected for this conversation. */
assistant_knowledge_base_ids?: string[];
/** Direct RAG modes handled in the frontend; these requests never enter Lead Agent. */
execution_profile?: "single_round_rag" | "multi_round_rag";
agent_id?: string;
+20 -1
View File
@@ -8,6 +8,7 @@ import {
SparklesIcon,
} from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import { useSearchParams } from "react-router-dom";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
@@ -21,14 +22,28 @@ import { SkillRow } from "@/components/workspace/settings/skill-settings-page";
import { getStoredAuth } from "@/core/auth";
import { useAllSkillsAdmin } from "@/core/skills/hooks";
import { desensitize } from "@/lib/desensitize";
import { useSkillKnowledgeBindings } from "@/strategy-components/api/skill-knowledge";
import { SkillAddressesPanel } from "./SkillAddressesPage";
const PAGE_SIZE = 10;
export default function AdminSkillsPage() {
const [searchParams, setSearchParams] = useSearchParams();
const [search, setSearch] = useState("");
const [page, setPage] = useState(1);
const { skills, isLoading, error } = useAllSkillsAdmin(true);
const knowledgeBindings = useSkillKnowledgeBindings();
const activeTab = useMemo(() => {
const tab = searchParams.get("tab");
return tab === "addresses" ? tab : "skills";
}, [searchParams]);
const changeTab = (value: string) => {
const next = new URLSearchParams(searchParams);
if (value === "skills") next.delete("tab");
else next.set("tab", value);
setSearchParams(next, { replace: true });
};
const storedAuth = useMemo(() => getStoredAuth(), []);
const currentUserId = storedAuth?.user_id ?? null;
@@ -56,7 +71,8 @@ export default function AdminSkillsPage() {
<WorkspaceHeader />
<WorkspaceBody>
<Tabs
defaultValue="skills"
value={activeTab}
onValueChange={changeTab}
className="flex size-full min-h-0 flex-col gap-0"
>
<div className="flex items-center justify-between gap-4 border-b px-6 pt-4">
@@ -126,6 +142,9 @@ export default function AdminSkillsPage() {
skill={skill}
currentUserId={currentUserId}
isAdmin
knowledgeBindings={(knowledgeBindings.data ?? []).filter(
(binding) => binding.skill_name === skill.name,
)}
/>
))}
</>
+71 -12
View File
@@ -128,6 +128,26 @@ function readPendingKnowledgeSelection(agentId: string | undefined): string[] |
}
}
function readPendingAssistantKnowledgeSelection(agentId: string | undefined): string[] | null {
if (typeof window === "undefined" || !agentId) return null;
try {
const raw = sessionStorage.getItem("pendingAgentMessage");
if (!raw) return null;
const parsed = JSON.parse(raw) as {
agentId?: string;
assistantKnowledgeBaseIds?: unknown;
};
if (parsed.agentId !== agentId || !Array.isArray(parsed.assistantKnowledgeBaseIds)) {
return null;
}
return parsed.assistantKnowledgeBaseIds
.map((item) => String(item).trim())
.filter(Boolean);
} catch {
return null;
}
}
function stringifyForHiddenContext(value: unknown): string {
if (typeof value === "string") return value;
try {
@@ -344,11 +364,17 @@ export default function AgentChatPage({
const pendingKnowledgeSelection = useRef(
readPendingKnowledgeSelection(agent_id),
);
const pendingAssistantKnowledgeSelection = useRef(
readPendingAssistantKnowledgeSelection(agent_id),
);
const [selectedKnowledgeBaseIds, setSelectedKnowledgeBaseIds] = useState<string[]>(
() => pendingKnowledgeSelection.current ?? [],
);
const [selectedAssistantKnowledgeBaseIds, setSelectedAssistantKnowledgeBaseIds] = useState<string[]>(
() => pendingAssistantKnowledgeSelection.current ?? [],
);
const [knowledgeSelectionReady, setKnowledgeSelectionReady] = useState(
() => pendingKnowledgeSelection.current !== null,
() => pendingKnowledgeSelection.current !== null || pendingAssistantKnowledgeSelection.current !== null,
);
const [knowledgeSelectionLoading, setKnowledgeSelectionLoading] = useState(false);
const knowledgeSelectionThread = useRef<string | null>(null);
@@ -359,6 +385,10 @@ export default function AgentChatPage({
setSelectedKnowledgeBaseIds(ids);
setKnowledgeSelectionReady(true);
}, []);
const changeAssistantKnowledgeSelection = useCallback((ids: string[]) => {
setSelectedAssistantKnowledgeBaseIds(ids);
setKnowledgeSelectionReady(true);
}, []);
useEffect(() => {
activeThreadIdRef.current = threadId;
@@ -368,17 +398,23 @@ export default function AgentChatPage({
if (knowledgeSelectionAgent.current === agent_id) return;
knowledgeSelectionAgent.current = agent_id;
const pending = readPendingKnowledgeSelection(agent_id);
const pendingAssistant = readPendingAssistantKnowledgeSelection(agent_id);
pendingKnowledgeSelection.current = pending;
pendingAssistantKnowledgeSelection.current = pendingAssistant;
knowledgeSelectionThread.current = null;
setSelectedKnowledgeBaseIds(pending ?? []);
setKnowledgeSelectionReady(pending !== null);
setSelectedAssistantKnowledgeBaseIds(pendingAssistant ?? []);
setKnowledgeSelectionReady(pending !== null || pendingAssistant !== null);
}, [agent_id]);
useEffect(() => {
if (isNewThread) {
if (knowledgeSelectionThread.current !== "__new__") {
setSelectedKnowledgeBaseIds(pendingKnowledgeSelection.current ?? []);
setKnowledgeSelectionReady(pendingKnowledgeSelection.current !== null);
setSelectedAssistantKnowledgeBaseIds(pendingAssistantKnowledgeSelection.current ?? []);
setKnowledgeSelectionReady(
pendingKnowledgeSelection.current !== null || pendingAssistantKnowledgeSelection.current !== null,
);
}
knowledgeSelectionThread.current = "__new__";
setKnowledgeSelectionLoading(false);
@@ -393,18 +429,29 @@ export default function AgentChatPage({
if (knowledgeSelectionThread.current !== threadId) return;
const metadata = record.metadata as Record<string, unknown> | undefined;
const raw = metadata?.llmwiki_knowledge_base_ids;
const assistantRaw = metadata?.assistant_knowledge_base_ids;
const llmwikiIds = Array.isArray(raw)
? raw.map((item) => String(item).trim()).filter(Boolean)
: [];
const assistantIds = Array.isArray(assistantRaw)
? assistantRaw.map((item) => String(item).trim()).filter(Boolean)
: [];
if (Array.isArray(raw)) {
setSelectedKnowledgeBaseIds(
raw.map((item) => String(item).trim()).filter(Boolean),
);
setKnowledgeSelectionReady(true);
setSelectedKnowledgeBaseIds(llmwikiIds);
} else {
setSelectedKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
}
if (Array.isArray(assistantRaw)) {
setSelectedAssistantKnowledgeBaseIds(assistantIds);
} else {
setSelectedAssistantKnowledgeBaseIds([]);
}
setKnowledgeSelectionReady(Array.isArray(raw) || Array.isArray(assistantRaw));
})
.catch(() => {
if (knowledgeSelectionThread.current === threadId) {
setSelectedKnowledgeBaseIds([]);
setSelectedAssistantKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
}
})
@@ -427,10 +474,13 @@ export default function AgentChatPage({
() => ({
...(taskId ? taskThreadMetadata(taskContext) : {}),
...(knowledgeSelectionReady
? { llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds }
? {
llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds,
assistant_knowledge_base_ids: selectedAssistantKnowledgeBaseIds,
}
: {}),
}),
[knowledgeSelectionReady, selectedKnowledgeBaseIds, taskContext, taskId],
[knowledgeSelectionReady, selectedAssistantKnowledgeBaseIds, selectedKnowledgeBaseIds, taskContext, taskId],
);
// 任务工作区(taskBind):每次运行都带隐藏任务上下文(任务id/名称/内容),
// 新建对话的消息历史里没有开场信息时模型也能拿到任务id(见 taskBindRunContext)。
@@ -446,7 +496,10 @@ export default function AgentChatPage({
...settings.context,
agent_id,
...(knowledgeSelectionReady
? { llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds }
? {
llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds,
assistant_knowledge_base_ids: selectedAssistantKnowledgeBaseIds,
}
: {}),
...(taskBindRunContextValue ?? {}),
},
@@ -1043,9 +1096,13 @@ export default function AgentChatPage({
source="agent"
agentId={agent_id}
value={selectedKnowledgeBaseIds}
assistantValue={selectedAssistantKnowledgeBaseIds}
onChange={changeKnowledgeSelection}
onAssistantChange={changeAssistantKnowledgeSelection}
autoSelectAll={
isNewThread && pendingKnowledgeSelection.current === null
isNewThread &&
pendingKnowledgeSelection.current === null &&
pendingAssistantKnowledgeSelection.current === null
}
disabled={isDemoMode || knowledgeSelectionLoading}
label="知识空间"
@@ -1130,7 +1187,9 @@ export default function AgentChatPage({
source="agent"
agentId={agent_id}
value={selectedKnowledgeBaseIds}
assistantValue={selectedAssistantKnowledgeBaseIds}
onChange={changeKnowledgeSelection}
onAssistantChange={changeAssistantKnowledgeSelection}
disabled={isDemoMode || knowledgeSelectionLoading}
readOnly
label="知识空间"
@@ -0,0 +1,559 @@
"use client";
import { useQuery } from "@tanstack/react-query";
import {
ArrowDownWideNarrow,
Bot,
ChevronLeft,
ChevronRight,
Download,
FileText,
FileX2,
Library,
Loader2,
MessagesSquare,
MessageSquareShare,
RefreshCw,
Search,
} from "lucide-react";
import Link from "next/link";
import { useEffect, useMemo, useRef, useState } from "react";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { ScrollArea } from "@/components/ui/scroll-area";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select";
import { Skeleton } from "@/components/ui/skeleton";
import {
WorkspaceBody,
WorkspaceContainer,
WorkspaceHeader,
} from "@/components/workspace/workspace-container";
import {
downloadArtifactFile,
downloadArtifactMarkdownAsWord,
} from "@/core/artifacts/download";
import {
fetchArtifactLibrary,
type ArtifactLibraryItem,
type ArtifactLibraryKind,
type ArtifactLibrarySource,
} from "@/core/artifacts/library";
import { useI18n } from "@/core/i18n/hooks";
import { bootstrapResumePath, pathOfThread } from "@/core/threads/utils";
import { formatDateTime } from "@/core/utils/datetime";
import {
getFileExtensionDisplayName,
getFileIcon,
} from "@/core/utils/files";
import { desensitize } from "@/lib/desensitize";
import { cn } from "@/lib/utils";
const PAGE_SIZE = 50;
const FILTERS: Array<{ value: ArtifactLibraryKind; label: string }> = [
{ value: "all", label: "全部" },
{ value: "document", label: "文档" },
{ value: "spreadsheet", label: "表格" },
{ value: "text_data", label: "文本与数据" },
{ value: "archive", label: "压缩包" },
{ value: "other", label: "其他" },
];
const SOURCE_FILTERS: Array<{
value: ArtifactLibrarySource;
label: string;
}> = [
{ value: "all", label: "全部来源" },
{ value: "general_qa", label: "通用问答" },
{ value: "agent", label: "智能体" },
];
type DownloadFormat = "original" | "word";
function isMarkdownArtifact(name: string): boolean {
return /\.(md|markdown)$/i.test(name);
}
function artifactDownloadKey(
item: ArtifactLibraryItem,
format: DownloadFormat,
): string {
return `${item.threadId}:${item.path}:${format}`;
}
function formatBytes(bytes: number): string {
if (!Number.isFinite(bytes) || bytes <= 0) return "0 B";
const units = ["B", "KB", "MB", "GB"];
const index = Math.min(
Math.floor(Math.log(bytes) / Math.log(1024)),
units.length - 1,
);
const value = bytes / 1024 ** index;
const digits = index === 0 || value >= 100 ? 0 : value >= 10 ? 1 : 2;
return `${value.toFixed(digits)} ${units[index]}`;
}
function sourceConversationPath(item: ArtifactLibraryItem): string {
const basePath =
item.threadType === "bootstrap"
? bootstrapResumePath(item.threadId)
: pathOfThread({
thread_id: item.threadId,
metadata: item.agentId ? { agent_id: item.agentId } : undefined,
});
const search = new URLSearchParams({ artifact: item.path });
return `${basePath}?${search.toString()}`;
}
function FileRow({
item,
downloading,
onDownload,
}: {
item: ArtifactLibraryItem;
downloading: DownloadFormat | null;
onDownload: (item: ArtifactLibraryItem, format: DownloadFormat) => void;
}) {
const sourcePath = sourceConversationPath(item);
const displayName = desensitize(item.name);
const sourceTitle = desensitize(item.threadTitle || "新对话");
const sourceLabel = item.sourceType === "agent" ? "智能体" : "通用问答";
const sourceName =
item.sourceType === "agent"
? desensitize(item.agentName || item.agentId || "未知智能体")
: sourceLabel;
const SourceIcon = item.sourceType === "agent" ? Bot : MessagesSquare;
const busy = downloading !== null;
return (
<tr className="group border-b transition-colors hover:bg-muted/35">
<td className="min-w-0 py-3 pr-4 pl-1">
<Link
href={sourcePath}
className="flex min-w-0 items-center gap-3 rounded-md outline-none focus-visible:ring-2 focus-visible:ring-ring"
title={`打开原对话并查看 ${displayName}`}
>
<span className="flex size-10 shrink-0 items-center justify-center rounded-lg bg-muted text-foreground/75">
{getFileIcon(item.name, "size-5")}
</span>
<span className="min-w-0">
<span className="block truncate text-sm font-medium text-foreground group-hover:text-primary">
{displayName}
</span>
<span className="mt-1 block truncate text-xs text-muted-foreground lg:hidden">
{getFileExtensionDisplayName(item.name)} · {sourceName} · 来自「
{sourceTitle}」
</span>
<span className="mt-1 hidden text-xs text-muted-foreground lg:block">
{getFileExtensionDisplayName(item.name)} 文件
</span>
</span>
</Link>
</td>
<td className="hidden min-w-0 px-3 py-3 lg:table-cell">
<div className="flex min-w-0 items-center gap-2.5">
<span className="flex size-8 shrink-0 items-center justify-center rounded-md bg-muted text-muted-foreground">
<SourceIcon className="size-4" />
</span>
<span className="min-w-0">
<span className="block truncate text-sm font-medium text-foreground">
{sourceName}
</span>
<Link
href={sourcePath}
className="block truncate text-xs text-muted-foreground underline-offset-4 hover:text-foreground hover:underline"
title={`打开原对话:${sourceTitle}`}
>
{item.sourceType === "agent" ? "智能体 · " : ""}
{sourceTitle}
</Link>
</span>
</div>
</td>
<td className="whitespace-nowrap px-3 py-3 text-sm text-muted-foreground">
{formatDateTime(item.modifiedAt)}
</td>
<td className="hidden whitespace-nowrap px-3 py-3 text-sm text-muted-foreground md:table-cell">
{formatBytes(item.sizeBytes)}
</td>
<td className="py-3 pr-1 pl-3 text-right">
<div className="flex items-center justify-end gap-1">
<Button
type="button"
size="icon"
variant="ghost"
className="size-8"
disabled={busy}
onClick={() => onDownload(item, "original")}
title={`下载原文件:${displayName}`}
aria-label={`下载原文件:${displayName}`}
>
{downloading === "original" ? (
<Loader2 className="size-4 animate-spin" />
) : (
<Download className="size-4" />
)}
</Button>
{isMarkdownArtifact(item.name) && (
<Button
type="button"
size="sm"
variant="ghost"
className="h-8 gap-1 px-2 text-xs"
disabled={busy}
onClick={() => onDownload(item, "word")}
title={`下载为 Word:${displayName}`}
aria-label={`下载为 Word:${displayName}`}
>
{downloading === "word" ? (
<Loader2 className="size-4 animate-spin" />
) : (
<FileText className="size-4" />
)}
Word
</Button>
)}
<Button
asChild
size="icon"
variant="ghost"
className="size-8"
title={`打开原对话:${sourceTitle}`}
>
<Link href={sourcePath} aria-label={`打开原对话:${sourceTitle}`}>
<MessageSquareShare className="size-4" />
</Link>
</Button>
</div>
</td>
</tr>
);
}
function LoadingRows() {
return Array.from({ length: 5 }, (_, index) => (
<tr className="border-b" key={index}>
<td className="py-3 pr-4 pl-1">
<div className="flex items-center gap-3">
<Skeleton className="size-10 shrink-0" />
<div className="min-w-0 flex-1 space-y-2">
<Skeleton className="h-4 w-2/3" />
<Skeleton className="h-3 w-1/3" />
</div>
</div>
</td>
<td className="hidden px-3 lg:table-cell"><Skeleton className="h-4 w-3/4" /></td>
<td className="px-3"><Skeleton className="h-4 w-24" /></td>
<td className="hidden px-3 md:table-cell"><Skeleton className="h-4 w-16" /></td>
<td className="pr-1 pl-3"><Skeleton className="ml-auto size-8" /></td>
</tr>
));
}
export default function ArtifactLibraryPage() {
const { t } = useI18n();
const [search, setSearch] = useState("");
const [debouncedSearch, setDebouncedSearch] = useState("");
const [fileKind, setFileKind] = useState<ArtifactLibraryKind>("all");
const [sourceType, setSourceType] = useState<ArtifactLibrarySource>("all");
const [page, setPage] = useState(1);
const [downloadingKey, setDownloadingKey] = useState<string | null>(null);
const forceRefreshRef = useRef(false);
useEffect(() => {
document.title = `资料库 - ${t.pages.appName}`;
}, [t.pages.appName]);
useEffect(() => {
const timer = window.setTimeout(() => setDebouncedSearch(search.trim()), 300);
return () => window.clearTimeout(timer);
}, [search]);
useEffect(() => {
setPage(1);
}, [debouncedSearch, fileKind, sourceType]);
const offset = (page - 1) * PAGE_SIZE;
const libraryQuery = useQuery({
queryKey: ["artifact-library", debouncedSearch, fileKind, sourceType, page],
queryFn: async ({ signal }) => {
const refresh = forceRefreshRef.current;
try {
return await fetchArtifactLibrary({
query: debouncedSearch,
fileKind,
sourceType,
offset,
limit: PAGE_SIZE,
refresh,
signal,
});
} finally {
forceRefreshRef.current = false;
}
},
retry: false,
});
const total = libraryQuery.data?.total ?? 0;
const items = libraryQuery.data?.items ?? [];
const totalPages = Math.max(1, Math.ceil(total / PAGE_SIZE));
const hasPrevious = page > 1;
const hasNext = page < totalPages;
useEffect(() => {
if (page > totalPages) setPage(totalPages);
}, [page, totalPages]);
const itemKeys = useMemo(
() =>
new Set(
items.flatMap((item) => [
artifactDownloadKey(item, "original"),
artifactDownloadKey(item, "word"),
]),
),
[items],
);
useEffect(() => {
if (downloadingKey && !itemKeys.has(downloadingKey)) setDownloadingKey(null);
}, [downloadingKey, itemKeys]);
const handleDownload = async (
item: ArtifactLibraryItem,
format: DownloadFormat,
) => {
const key = artifactDownloadKey(item, format);
if (downloadingKey) return;
setDownloadingKey(key);
try {
if (format === "word") {
await downloadArtifactMarkdownAsWord({
filepath: item.path,
threadId: item.threadId,
});
toast.success("Word 文件已生成");
} else {
await downloadArtifactFile({
filepath: item.path,
threadId: item.threadId,
});
}
} catch (error) {
console.error(error);
toast.error(error instanceof Error ? error.message : "下载失败");
} finally {
setDownloadingKey(null);
}
};
const handleRefresh = () => {
forceRefreshRef.current = true;
void libraryQuery.refetch();
};
return (
<WorkspaceContainer>
<WorkspaceHeader />
<WorkspaceBody className="items-stretch">
<div className="flex min-h-0 w-full flex-1 flex-col">
<header className="shrink-0 border-b bg-card/40 px-5 py-6 sm:px-8">
<div className="mx-auto flex w-full max-w-6xl flex-col gap-5">
<div className="flex flex-col justify-between gap-4 sm:flex-row sm:items-start">
<div>
<div className="flex items-center gap-2">
<Library className="size-6 text-primary" />
<h1 className="text-2xl font-semibold tracking-tight">资料库</h1>
</div>
<p className="mt-2 text-sm text-muted-foreground">
汇总当前账号在对话中生成的文件,不包含上传文件、图片、音频和视频。
</p>
</div>
<Button
type="button"
variant="outline"
size="sm"
onClick={handleRefresh}
disabled={libraryQuery.isFetching}
>
<RefreshCw className={cn("size-4", libraryQuery.isFetching && "animate-spin")} />
刷新
</Button>
</div>
<div className="relative max-w-xl">
<Search className="pointer-events-none absolute top-1/2 left-3 size-4 -translate-y-1/2 text-muted-foreground" />
<Input
type="search"
className="h-11 pl-9"
placeholder="搜索文件名、来源对话或智能体名称"
value={search}
onChange={(event) => setSearch(event.target.value)}
/>
</div>
<div className="flex flex-wrap items-center justify-between gap-4">
<div className="flex flex-wrap items-center gap-3">
<div className="flex flex-wrap gap-1" aria-label="文件类型筛选">
{FILTERS.map((filter) => (
<Button
key={filter.value}
type="button"
size="sm"
variant={fileKind === filter.value ? "secondary" : "ghost"}
aria-pressed={fileKind === filter.value}
onClick={() => setFileKind(filter.value)}
>
{filter.label}
</Button>
))}
</div>
<Select
value={sourceType}
onValueChange={(value) =>
setSourceType(value as ArtifactLibrarySource)
}
>
<SelectTrigger
size="sm"
className="w-32"
aria-label="筛选文件来源"
>
<SelectValue />
</SelectTrigger>
<SelectContent>
{SOURCE_FILTERS.map((filter) => (
<SelectItem key={filter.value} value={filter.value}>
{filter.label}
</SelectItem>
))}
</SelectContent>
</Select>
</div>
<div className="flex items-center gap-2 text-xs text-muted-foreground">
<ArrowDownWideNarrow className="size-4" />
最近生成
</div>
</div>
</div>
</header>
<ScrollArea className="min-h-0 flex-1">
<main className="mx-auto w-full max-w-6xl px-5 py-5 sm:px-8">
<div className="mb-2 text-xs text-muted-foreground tabular-nums">
{libraryQuery.isLoading ? "正在统计文件…" : `共 ${total} 个文件`}
</div>
<div className="overflow-x-auto">
<table className="w-full table-fixed border-collapse">
<thead>
<tr className="border-b text-left text-xs font-normal text-muted-foreground">
<th className="w-[48%] py-3 pr-4 pl-1 font-normal lg:w-[35%]">文件名</th>
<th className="hidden w-[25%] px-3 py-3 font-normal lg:table-cell">来源</th>
<th className="w-[34%] px-3 py-3 font-normal md:w-[22%] lg:w-[16%]">生成时间</th>
<th className="hidden w-[13%] px-3 py-3 font-normal lg:w-[10%] md:table-cell">大小</th>
<th className="w-[18%] py-3 pr-1 pl-3 text-right font-normal md:w-[17%] lg:w-[14%]">
<span className="sr-only">操作</span>
</th>
</tr>
</thead>
<tbody>
{libraryQuery.isLoading ? (
<LoadingRows />
) : (
items.map((item) => {
const key = `${item.threadId}:${item.path}`;
const activeDownload = downloadingKey?.startsWith(`${key}:`)
? downloadingKey.endsWith(":word")
? "word"
: "original"
: null;
return (
<FileRow
key={key}
item={item}
downloading={activeDownload}
onDownload={(target, format) =>
void handleDownload(target, format)
}
/>
);
})
)}
</tbody>
</table>
</div>
{!libraryQuery.isLoading && libraryQuery.isError && (
<div className="flex flex-col items-center gap-3 py-16 text-center">
<FileX2 className="size-9 text-muted-foreground" />
<div>
<p className="text-sm font-medium">资料库加载失败</p>
<p className="mt-1 text-xs text-muted-foreground">
{libraryQuery.error instanceof Error
? libraryQuery.error.message
: "请稍后重试"}
</p>
</div>
<Button variant="outline" size="sm" onClick={handleRefresh}>
重新加载
</Button>
</div>
)}
{!libraryQuery.isLoading && !libraryQuery.isError && items.length === 0 && (
<div className="flex flex-col items-center gap-3 py-20 text-center">
<FileX2 className="size-9 text-muted-foreground" />
<div>
<p className="text-sm font-medium">
{debouncedSearch || fileKind !== "all" || sourceType !== "all"
? "没有找到匹配的文件"
: "还没有生成文件"}
</p>
<p className="mt-1 text-xs text-muted-foreground">
对话中生成的文档、表格和数据文件会自动出现在这里。
</p>
</div>
</div>
)}
{!libraryQuery.isLoading && !libraryQuery.isError && totalPages > 1 && (
<div className="flex items-center justify-center gap-3 py-7">
<Button
variant="outline"
size="sm"
disabled={!hasPrevious}
onClick={() => setPage((current) => Math.max(1, current - 1))}
>
<ChevronLeft className="size-4" />
上一页
</Button>
<span className="text-sm text-muted-foreground tabular-nums">
{page} / {totalPages}
</span>
<Button
variant="outline"
size="sm"
disabled={!hasNext}
onClick={() => setPage((current) => current + 1)}
>
下一页
<ChevronRight className="size-4" />
</Button>
</div>
)}
</main>
</ScrollArea>
</div>
</WorkspaceBody>
</WorkspaceContainer>
);
}
@@ -0,0 +1,631 @@
import {
ArrowLeft,
BookOpenText,
Database,
Edit3,
Loader2,
Plus,
RefreshCw,
Search,
Trash2,
Upload,
} from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import { useSearchParams } from "react-router-dom";
import { useQueryClient } from "@tanstack/react-query";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { Input } from "@/components/ui/input";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Textarea } from "@/components/ui/textarea";
import { KnowledgeProductNav } from "@/components/workspace/knowledge-product-nav";
import { getStoredAuth } from "@/core/auth";
import { buildAppHashUrl } from "@/core/llmwiki/source-links";
import {
searchAssistantWiki,
useAssistantImportJobs,
useAssistantKnowledgeActions,
useAssistantKnowledgeBases,
useAssistantKnowledgeSources,
useAssistantWikiPages,
type AssistantKnowledgeBase,
} from "@/strategy-components/api/assistant-knowledge";
import {
WikiLibraryReader,
type LibraryWikiPage,
} from "@/strategy-components/components/resource-management/llmwiki/WikiLibraryReader";
import { WikiPackageTransfers } from "@/strategy-components/components/resource-management/llmwiki/WikiPackageTransfers";
export default function AssistantKnowledgePage() {
const isAdmin = getStoredAuth()?.system_role === "admin";
const [params, setParams] = useSearchParams();
const baseId = params.get("base_id");
const slug = params.get("wikiSlug");
const bases = useAssistantKnowledgeBases();
const actions = useAssistantKnowledgeActions();
const pages = useAssistantWikiPages("", baseId);
const sources = useAssistantKnowledgeSources(baseId);
const jobs = useAssistantImportJobs(baseId);
const client = useQueryClient();
const [tab, setTab] = useState("wiki");
const [createOpen, setCreateOpen] = useState(false);
const [editingBase, setEditingBase] =
useState<AssistantKnowledgeBase | null>(null);
const [name, setName] = useState("");
const [query, setQuery] = useState("");
const [searching, setSearching] = useState(false);
const [searchResult, setSearchResult] = useState<Awaited<
ReturnType<typeof searchAssistantWiki>
> | null>(null);
const upload = useRef<HTMLInputElement>(null);
const base = bases.data?.find((b) => b.id === baseId);
const libraryPages = useMemo<LibraryWikiPage[]>(
() =>
(pages.data || []).map((p) => ({
...p,
content: p.content_markdown,
status: "published",
aliases: p.aliases || [],
source_refs: p.source_refs || [],
in_links: p.in_links || [],
out_links: p.out_links || [],
version: 1,
})),
[pages.data],
);
const active =
libraryPages.find((p) => p.slug === slug) ||
libraryPages.find((p) => p.page_type === "index") ||
libraryPages[0] ||
null;
const navigatePage = (nextSlug: string) => {
setParams({ base_id: baseId!, wikiSlug: nextSlug });
setTab("wiki");
};
const finishedJobs = (jobs.data || [])
.filter((j) => j.status === "completed")
.map((j) => j.id)
.join(",");
useEffect(() => {
void client.invalidateQueries({
queryKey: ["assistant-knowledge", "pages"],
});
void client.invalidateQueries({
queryKey: ["assistant-knowledge", "bases"],
});
}, [finishedJobs, client]);
useEffect(() => {
setSearchResult(null);
setQuery("");
}, [baseId]);
const create = async () => {
if (!name.trim()) return;
try {
const b = await actions.createBase.mutateAsync({ name: name.trim() });
setCreateOpen(false);
setName("");
setParams({ base_id: b.id });
} catch (e) {
toast.error((e as Error).message);
}
};
const search = async () => {
if (!query.trim() || !baseId) return;
setSearching(true);
try {
setSearchResult(await searchAssistantWiki(query, baseId));
} catch (e) {
toast.error((e as Error).message);
} finally {
setSearching(false);
}
};
const refresh = () =>
void client.invalidateQueries({ queryKey: ["assistant-knowledge"] });
const removeBase = async (target: AssistantKnowledgeBase) => {
if (target.is_global) {
toast.error("知识梳理总库承担全库沉淀,不能删除");
return;
}
if (
!window.confirm(
`确定删除助手知识库“${target.name}”吗?库内 Wiki、向量、实体、关系和导入记录都会删除。`,
)
)
return;
try {
await actions.deleteBase.mutateAsync(target.id);
if (baseId === target.id) setParams({});
toast.success("助手知识库已删除");
} catch (error) {
toast.error((error as Error).message);
}
};
return (
<div className="flex size-full min-h-0 flex-col bg-background">
<KnowledgeProductNav />
{!baseId ? (
<main className="flex-1 overflow-auto p-6 lg:p-8">
<header className="mb-8 flex items-center justify-between">
<div>
<h1 className="text-2xl font-semibold">助手知识库</h1>
<p className="mt-2 text-sm text-muted-foreground">
浏览分类
Wiki、关联知识与来源。普通知识库向量化完成后自动沉淀到知识梳理。
</p>
</div>
{isAdmin && (
<Button onClick={() => setCreateOpen(true)}>
<Plus className="mr-2 size-4" />
新建知识库
</Button>
)}
</header>
{bases.isLoading && <Loader2 className="animate-spin" />}
{bases.error && (
<p className="text-destructive">{bases.error.message}</p>
)}
<div className="grid gap-5 md:grid-cols-2 xl:grid-cols-3">
{bases.data?.map((b) => (
<div
key={b.id}
className="group overflow-hidden rounded-2xl border bg-card text-left shadow-sm transition hover:border-primary/40 hover:shadow-md"
>
<button
type="button"
onClick={() => setParams({ base_id: b.id })}
className="w-full p-6 text-left"
>
<div className="mb-5 flex items-center justify-between">
<div className="rounded-xl bg-emerald-100 p-3 text-emerald-700 dark:bg-emerald-950">
<Database className="size-6" />
</div>
{b.is_global && (
<Badge variant="secondary">全库知识梳理</Badge>
)}
</div>
<h2 className="text-lg font-semibold group-hover:text-primary">
{b.name}
</h2>
<p className="mt-2 line-clamp-2 min-h-10 text-sm text-muted-foreground">
{b.description ||
(b.is_global
? "汇聚普通知识库的分析成果,保留分类、标签和知识关联。"
: "独立的 Wiki 知识空间,支持导入、导出与向量检索。")}
</p>
<div className="mt-6 flex gap-5 text-xs text-muted-foreground">
<span>{b.page_count || 0} Wiki</span>
<span>{b.source_count || 0} 来源</span>
<span>{b.relation_count || 0} 关系</span>
</div>
</button>
{isAdmin && (
<div className="flex items-center gap-2 border-t bg-muted/20 px-6 py-3">
<Button
size="sm"
variant="outline"
disabled={actions.updateBase.isPending}
onClick={() => setEditingBase(b)}
>
<Edit3 className="mr-2 size-4" />
编辑
</Button>
{!b.is_global && (
<Button
size="sm"
variant="ghost"
className="ml-auto text-destructive hover:bg-destructive/10 hover:text-destructive"
disabled={actions.deleteBase.isPending}
onClick={() => void removeBase(b)}
>
<Trash2 className="mr-2 size-4" />
删除
</Button>
)}
{b.is_global && (
<span className="ml-auto text-xs text-muted-foreground">
系统大库不可删除
</span>
)}
</div>
)}
</div>
))}
</div>
{!bases.isLoading && !bases.data?.length && isAdmin && (
<Button
className="mt-5"
onClick={() =>
actions.initialize.mutate("知识梳理总库", {
onError: (e) => toast.error(e.message),
})
}
>
初始化知识梳理总库
</Button>
)}
</main>
) : (
<>
<header className="flex shrink-0 flex-wrap items-center gap-3 border-b px-5 py-4">
<Button
variant="ghost"
size="icon"
aria-label="返回助手知识库"
onClick={() => setParams({})}
>
<ArrowLeft className="size-4" />
</Button>
<div className="rounded-xl bg-emerald-100 p-2.5 text-emerald-700 dark:bg-emerald-950">
<BookOpenText className="size-5" />
</div>
<div className="min-w-0">
<h1 className="text-lg font-semibold">
{base?.name || "助手知识库"}
</h1>
<p className="mt-1 text-xs text-muted-foreground">
{base?.is_global
? "全库知识梳理 · 自动同步分析成果"
: "Wiki 知识空间"}
</p>
</div>
<div className="ml-auto flex gap-2">
<Button variant="outline" size="sm" onClick={refresh}>
<RefreshCw className="mr-2 size-4" />
刷新
</Button>
{isAdmin && (
<Button
variant="outline"
size="sm"
disabled={!base || actions.updateBase.isPending}
onClick={() => base && setEditingBase(base)}
>
<Edit3 className="mr-2 size-4" />
编辑
</Button>
)}
{isAdmin && base && !base.is_global && (
<Button
variant="outline"
size="sm"
className="text-destructive hover:bg-destructive/10 hover:text-destructive"
disabled={actions.deleteBase.isPending}
onClick={() => void removeBase(base)}
>
<Trash2 className="mr-2 size-4" />
删除
</Button>
)}
{isAdmin && (
<Button
variant="outline"
size="sm"
disabled={actions.uploadWeKnoraPackage.isPending}
onClick={() => upload.current?.click()}
>
<Upload className="mr-2 size-4" />
导入数据包
</Button>
)}
</div>
<input
ref={upload}
type="file"
accept=".jsonl,.ndjson"
className="hidden"
onChange={async (e) => {
const file = e.target.files?.[0];
e.target.value = "";
if (!file) return;
try {
await actions.uploadWeKnoraPackage.mutateAsync({
file,
assistantBaseId: baseId,
});
toast.success("文件已上传,后台正在导入");
} catch (err) {
toast.error((err as Error).message);
}
}}
/>
</header>
<Tabs
value={tab}
onValueChange={setTab}
className="flex min-h-0 flex-1 flex-col gap-0"
>
<div className="shrink-0 border-b px-5 py-2">
<TabsList>
<TabsTrigger value="wiki">
Wiki({base?.page_count || 0})
</TabsTrigger>
<TabsTrigger value="search">知识检索</TabsTrigger>
<TabsTrigger value="sources">来源与同步</TabsTrigger>
{isAdmin && (
<TabsTrigger value="transfer">导入 / 导出记录</TabsTrigger>
)}
</TabsList>
</div>
<TabsContent value="wiki" className="mt-0 flex min-h-0 flex-1">
<WikiLibraryReader
key={baseId}
pages={libraryPages}
activePage={active}
onNavigate={navigatePage}
pageHref={(s) =>
buildAppHashUrl(
`/page/workspace/knowledge/assistant?${new URLSearchParams({ base_id: baseId, wikiSlug: s })}`,
)
}
mappingId={
active?.origins?.find((o) => o.mapping_id)?.mapping_id
}
initiallyCollapseTree
/>
</TabsContent>
<TabsContent value="search" className="overflow-auto p-6">
<div className="mb-5 flex max-w-3xl gap-2">
<Input
placeholder="用自然语言检索知识,例如:如何处理设备过热?"
value={query}
onChange={(e) => setQuery(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter") void search();
}}
/>
<Button disabled={searching} onClick={() => void search()}>
{searching ? (
<Loader2 className="size-4 animate-spin" />
) : (
<Search className="size-4" />
)}
检索
</Button>
</div>
{searchResult && (
<>
<Badge variant="outline">
{searchResult.retrieval_mode === "vector"
? "知识正文 / 片段 · 向量检索"
: searchResult.retrieval_mode === "empty"
? "没有匹配结果"
: "知识正文 / 片段 · 关键词检索(暂无匹配模型的向量)"}
</Badge>
<div className="mt-4 space-y-3">
{searchResult.results.map((p) => (
<button
key={p.id}
type="button"
onClick={() => {
if (p.page_type !== "chunk") navigatePage(p.slug);
}}
className={`block w-full max-w-4xl rounded-xl border p-5 text-left ${p.page_type === "chunk" ? "cursor-default" : "hover:bg-muted"}`}
>
<div className="font-medium">
{p.title}
{p.page_type === "chunk" && (
<Badge variant="secondary" className="ml-2">
知识片段
</Badge>
)}
<span className="float-right text-xs text-muted-foreground">
{p.score?.toFixed(4)}
</span>
</div>
<p className="mt-2 line-clamp-3 text-sm text-muted-foreground">
{p.matched_section?.content ||
p.matched_content ||
p.content_markdown}
</p>
</button>
))}
</div>
</>
)}
</TabsContent>
<TabsContent value="sources" className="overflow-auto p-6">
<div className="grid gap-4 lg:grid-cols-2">
{sources.data?.map((s) => (
<div className="rounded-xl border p-5" key={s.id}>
<h3 className="font-semibold">{s.source_name}</h3>
<p className="mt-2 text-xs text-muted-foreground">
上次同步:{new Date(s.updated_at).toLocaleString()}
</p>
{isAdmin && (
<Button
className="mt-4"
size="sm"
variant="outline"
disabled={actions.reimportSource.isPending}
onClick={() =>
actions.reimportSource.mutate(s.id, {
onSuccess: () => toast.success("已提交同步任务"),
onError: (e) => toast.error(e.message),
})
}
>
<RefreshCw className="mr-2 size-4" />
同步最新数据
</Button>
)}
</div>
))}
</div>
</TabsContent>
{isAdmin && (
<TabsContent
value="transfer"
className="space-y-5 overflow-auto p-6"
>
<WikiPackageTransfers id={baseId} assistant />
<h2 className="text-base font-semibold">导入与同步记录</h2>
{jobs.data?.map((j) => (
<div key={j.id} className="rounded-xl border p-4">
<div className="flex items-center gap-3">
<Badge variant="outline">
{{
completed: "已完成",
failed: "失败",
running: "处理中",
queued: "排队中",
}[j.status] || j.status}
</Badge>
<span className="text-xs text-muted-foreground">
{new Date(j.created_at).toLocaleString()}
</span>
</div>
<p className="mt-3 text-sm">
{j.counts?.wiki_pages || 0} Wiki ·{" "}
{j.counts?.vectors || 0} 向量 · {j.counts?.entities || 0}{" "}
实体 · {j.counts?.relations || 0} 关系
</p>
{j.last_error_message && (
<p className="mt-2 text-sm text-destructive">
{j.last_error_message}
</p>
)}
</div>
))}
</TabsContent>
)}
</Tabs>
</>
)}
<Dialog open={createOpen} onOpenChange={setCreateOpen}>
<DialogContent>
<DialogHeader>
<DialogTitle>新建助手知识库</DialogTitle>
</DialogHeader>
<Input
placeholder="知识库名称"
value={name}
onChange={(e) => setName(e.target.value)}
/>
<DialogFooter>
<Button
disabled={!name.trim() || actions.createBase.isPending}
onClick={() => void create()}
>
创建
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
{editingBase && (
<EditAssistantBaseDialog
key={editingBase.id}
base={editingBase}
pending={actions.updateBase.isPending}
onClose={() => setEditingBase(null)}
onSave={async (input) => {
try {
await actions.updateBase.mutateAsync({
id: editingBase.id,
...input,
});
setEditingBase(null);
toast.success("助手知识库信息已保存");
} catch (error) {
toast.error((error as Error).message);
}
}}
/>
)}
</div>
);
}
function EditAssistantBaseDialog({
base,
pending,
onClose,
onSave,
}: {
base: AssistantKnowledgeBase;
pending: boolean;
onClose: () => void;
onSave: (input: { name: string; description: string }) => Promise<void>;
}) {
const [name, setName] = useState(base.name);
const [description, setDescription] = useState(base.description || "");
return (
<Dialog
open
onOpenChange={(open) => {
if (!open && !pending) onClose();
}}
>
<DialogContent>
<form
onSubmit={(event) => {
event.preventDefault();
if (!pending && name.trim()) {
void onSave({
name: name.trim(),
description: description.trim(),
});
}
}}
>
<DialogHeader>
<DialogTitle>编辑助手知识库</DialogTitle>
<DialogDescription>
修改知识库名称和描述。知识内容、向量和关联关系不会受到影响。
</DialogDescription>
</DialogHeader>
<div className="space-y-4 py-4">
<div className="space-y-2">
<label className="text-sm font-medium" htmlFor="assistant-base-name">
名称
</label>
<Input
id="assistant-base-name"
value={name}
onChange={(event) => setName(event.target.value)}
maxLength={255}
disabled={pending}
required
/>
</div>
<div className="space-y-2">
<label
className="text-sm font-medium"
htmlFor="assistant-base-description"
>
描述
</label>
<Textarea
id="assistant-base-description"
value={description}
onChange={(event) => setDescription(event.target.value)}
maxLength={2048}
rows={4}
disabled={pending}
/>
</div>
</div>
<DialogFooter>
<Button type="button" variant="outline" disabled={pending} onClick={onClose}>
取消
</Button>
<Button type="submit" disabled={pending || !name.trim()}>
{pending && <Loader2 className="mr-2 size-4 animate-spin" />}
保存
</Button>
</DialogFooter>
</form>
</DialogContent>
</Dialog>
);
}
+44 -9
View File
@@ -141,6 +141,7 @@ export default function ChatPage() {
// per-thread header toggle so stale localStorage can't re-enable it.
const [knowledgeRagEnabled] = useKnowledgeRagEnabled();
const [selectedKnowledgeBaseIds, setSelectedKnowledgeBaseIds] = useState<string[]>([]);
const [selectedAssistantKnowledgeBaseIds, setSelectedAssistantKnowledgeBaseIds] = useState<string[]>([]);
const [knowledgeSelectionReady, setKnowledgeSelectionReady] = useState(false);
const [knowledgeSelectionLoading, setKnowledgeSelectionLoading] = useState(false);
const knowledgeSelectionThread = useRef<string | null>(null);
@@ -149,18 +150,26 @@ export default function ChatPage() {
setSelectedKnowledgeBaseIds(ids);
setKnowledgeSelectionReady(true);
}, []);
const changeAssistantKnowledgeSelection = useCallback((ids: string[]) => {
setSelectedAssistantKnowledgeBaseIds(ids);
setKnowledgeSelectionReady(true);
}, []);
const streamContext = useMemo(
() => ({
...settings.context,
knowledge_rag_enabled: knowledgeRagEnabled,
...(knowledgeSelectionReady
? { llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds }
? {
llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds,
assistant_knowledge_base_ids: selectedAssistantKnowledgeBaseIds,
}
: {}),
}),
[
settings.context,
knowledgeRagEnabled,
knowledgeSelectionReady,
selectedAssistantKnowledgeBaseIds,
selectedKnowledgeBaseIds,
],
);
@@ -199,6 +208,7 @@ export default function ChatPage() {
setHasStartedConversation(false);
setPromptPrefixOn(false);
setSelectedKnowledgeBaseIds([]);
setSelectedAssistantKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
setKnowledgeSelectionLoading(false);
knowledgeSelectionThread.current = "__new__";
@@ -224,18 +234,29 @@ export default function ChatPage() {
if (knowledgeSelectionThread.current !== threadId) return;
const metadata = record.metadata as Record<string, unknown> | undefined;
const raw = metadata?.llmwiki_knowledge_base_ids;
const assistantRaw = metadata?.assistant_knowledge_base_ids;
const llmwikiIds = Array.isArray(raw)
? raw.map((item) => String(item).trim()).filter(Boolean)
: [];
const assistantIds = Array.isArray(assistantRaw)
? assistantRaw.map((item) => String(item).trim()).filter(Boolean)
: [];
if (Array.isArray(raw)) {
setSelectedKnowledgeBaseIds(
raw.map((item) => String(item).trim()).filter(Boolean),
);
setKnowledgeSelectionReady(true);
setSelectedKnowledgeBaseIds(llmwikiIds);
} else {
setSelectedKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
}
if (Array.isArray(assistantRaw)) {
setSelectedAssistantKnowledgeBaseIds(assistantIds);
} else {
setSelectedAssistantKnowledgeBaseIds([]);
}
setKnowledgeSelectionReady(Array.isArray(raw) || Array.isArray(assistantRaw));
})
.catch(() => {
if (knowledgeSelectionThread.current === threadId) {
setSelectedKnowledgeBaseIds([]);
setSelectedAssistantKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
}
})
@@ -258,6 +279,7 @@ export default function ChatPage() {
if (!known.some((a) => a.id === selectedAgentId)) {
setSelectedAgentId(null);
setSelectedKnowledgeBaseIds([]);
setSelectedAssistantKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
}
}, [selectedAgentId, selectorAgents, mineAgents, squareAgents, selectorLoading]);
@@ -268,7 +290,10 @@ export default function ChatPage() {
threadId: isNewThread ? undefined : threadId,
context: streamContext,
threadMetadata: knowledgeSelectionReady
? { llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds }
? {
llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds,
assistant_knowledge_base_ids: selectedAssistantKnowledgeBaseIds,
}
: undefined,
isMock,
onStart: (createdThreadId) => {
@@ -442,7 +467,10 @@ export default function ChatPage() {
metadata: {
agent_id: selectedAgentId,
...(knowledgeSelectionReady
? { llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds }
? {
llmwiki_knowledge_base_ids: selectedKnowledgeBaseIds,
assistant_knowledge_base_ids: selectedAssistantKnowledgeBaseIds,
}
: {}),
},
});
@@ -453,7 +481,10 @@ export default function ChatPage() {
threadId: created.thread_id,
text,
...(knowledgeSelectionReady
? { knowledgeBaseIds: selectedKnowledgeBaseIds }
? {
knowledgeBaseIds: selectedKnowledgeBaseIds,
assistantKnowledgeBaseIds: selectedAssistantKnowledgeBaseIds,
}
: {}),
}),
);
@@ -515,6 +546,7 @@ export default function ChatPage() {
hideAdminPrompt,
preferencesLoading,
selectedAgentId,
selectedAssistantKnowledgeBaseIds,
selectedKnowledgeBaseIds,
knowledgeSelectionReady,
agentHandoffPending,
@@ -618,6 +650,7 @@ export default function ChatPage() {
const handleAgentChange = useCallback((agentId: string | null) => {
setSelectedAgentId(agentId);
setSelectedKnowledgeBaseIds([]);
setSelectedAssistantKnowledgeBaseIds([]);
setKnowledgeSelectionReady(false);
saveLastAgentId(agentId);
}, []);
@@ -900,7 +933,9 @@ export default function ChatPage() {
source={selectedAgentId ? "agent" : "all"}
agentId={selectedAgentId}
value={selectedKnowledgeBaseIds}
assistantValue={selectedAssistantKnowledgeBaseIds}
onChange={changeKnowledgeSelection}
onAssistantChange={changeAssistantKnowledgeSelection}
autoSelectAll={isLandingState && !!selectedAgentId}
disabled={
env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" ||
@@ -3,6 +3,11 @@ import { useState } from "react";
import { Navigate, useParams, useSearchParams } from "react-router-dom";
import { Button } from "@/components/ui/button";
import {
KNOWLEDGE_NORMAL_VIEW_VALUE,
KnowledgeProductNav,
useKnowledgeBaseProductVersion,
} from "@/components/workspace/knowledge-product-nav";
import {
Dialog,
DialogContent,
@@ -70,7 +75,9 @@ export default function LlmWikiKnowledgePage({
const { noteId = "" } = useParams();
const [searchParams] = useSearchParams();
const runtime = useLlmWikiRuntime();
const productVersion = useKnowledgeBaseProductVersion();
const wikiSlug = searchParams.get("wikiSlug")?.trim() || "";
const explicitNormalView = searchParams.get("view") === KNOWLEDGE_NORMAL_VIEW_VALUE;
if (mode === "detail" && noteId && wikiSlug) {
return <Navigate to={buildWeKnoraWikiPagePath(noteId, wikiSlug)} replace />;
@@ -80,7 +87,14 @@ export default function LlmWikiKnowledgePage({
return <RuntimeLoading />;
}
if (mode === "list" && productVersion.data === "v2" && !explicitNormalView) {
return <Navigate to="/page/workspace/knowledge/assistant" replace />;
}
if (runtime.isError || runtime.data?.provider !== "weknora") {
if (mode === "list" || mode === "detail") {
return <Navigate to="/page/workspace/knowledge/assistant" replace />;
}
if (mode === "graph") {
return <KnowledgeGraphPage />;
}
@@ -95,5 +109,5 @@ export default function LlmWikiKnowledgePage({
return <WeKnoraKnowledgeDetailPage />;
}
return <WeKnoraKnowledgeWorkspace />;
return <div className="flex size-full min-h-0 flex-col"><KnowledgeProductNav /><div className="min-h-0 flex-1"><WeKnoraKnowledgeWorkspace /></div></div>;
}
+49 -3
View File
@@ -1,9 +1,55 @@
import { BookOpenCheck, Sparkles } from "lucide-react";
import { useMemo } from "react";
import { useSearchParams } from "react-router-dom";
import { SkillSettingsPage } from "@/components/workspace/settings/skill-settings-page";
import { SkillKnowledgePanel } from "@/components/workspace/skills/SkillKnowledgePanel";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
export default function SkillsPage() {
const [searchParams, setSearchParams] = useSearchParams();
const tab = useMemo(
() => (searchParams.get("tab") === "knowledge" ? "knowledge" : "skills"),
[searchParams],
);
const changeTab = (value: string) => {
const next = new URLSearchParams(searchParams);
if (value === "knowledge") next.set("tab", "knowledge");
else next.delete("tab");
setSearchParams(next, { replace: true });
};
return (
<div className="flex size-full min-h-0 flex-col overflow-auto p-6">
<SkillSettingsPage />
</div>
<Tabs
value={tab}
onValueChange={changeTab}
className="flex size-full min-h-0 flex-col gap-0 overflow-hidden"
>
<header className="flex flex-wrap items-center justify-between gap-4 border-b bg-background px-6 py-4">
<div>
<h1 className="text-xl font-semibold">技能</h1>
<p className="mt-1 text-sm text-muted-foreground">
浏览和维护技能,也可以把多个技能的知识一次归纳到一个 Wiki 知识库。
</p>
</div>
<TabsList>
<TabsTrigger value="skills">
<Sparkles className="mr-2 size-4" />
技能列表
</TabsTrigger>
<TabsTrigger value="knowledge">
<BookOpenCheck className="mr-2 size-4" />
归纳到知识库
</TabsTrigger>
</TabsList>
</header>
<TabsContent value="skills" className="mt-0 min-h-0 flex-1 overflow-auto p-6">
<SkillSettingsPage embedded />
</TabsContent>
<TabsContent value="knowledge" className="mt-0 min-h-0 flex-1 overflow-hidden">
<SkillKnowledgePanel />
</TabsContent>
</Tabs>
);
}
+192 -122
View File
@@ -1,87 +1,96 @@
import { useEffect, useState, type CSSProperties, type ReactNode } from 'react';
import { Navigate, Route, Routes, useLocation, useParams } from 'react-router-dom';
import { useEffect, useState, type CSSProperties, type ReactNode } from "react";
import {
Navigate,
Route,
Routes,
useLocation,
useParams,
} from "react-router-dom";
import { getStoredAuth } from '@/core/auth';
import { getAPIClient } from '@/core/api';
import { isSentimentAgentVisible } from '@/core/sentiment-agent/config';
import { getStoredAuth } from "@/core/auth";
import { getAPIClient } from "@/core/api";
import { isSentimentAgentVisible } from "@/core/sentiment-agent/config";
import {
resolveChatsIframeEntryRedirect,
resolveStandardChatFallback,
} from '@/core/page-layout/chats-iframe-route';
import { isEmbeddedHost } from '@/core/embed/frame-context';
} from "@/core/page-layout/chats-iframe-route";
import { isEmbeddedHost } from "@/core/embed/frame-context";
import { SidebarInset, SidebarProvider } from '@/components/ui/sidebar';
import { PromptInputProvider } from '@/components/ai-elements/prompt-input';
import { SidebarInset, SidebarProvider } from "@/components/ui/sidebar";
import { PromptInputProvider } from "@/components/ai-elements/prompt-input";
import {
CompactShortcutButtons,
LayoutModeSwitcher,
ThemeSwitcher,
} from '@/components/workspace/appearance-controls';
import { WorkspaceSidebar } from '@/components/workspace/workspace-sidebar';
import { UserIdentityBadge } from '@/components/workspace/user-identity-badge';
import { CommandPalette } from '@/components/workspace/command-palette';
import { Toaster } from 'sonner';
import { SubtasksProvider } from '@/core/tasks/context';
import { ArtifactsProvider } from '@/components/workspace/artifacts';
import { DocumentRewritePresentationProvider } from '@/core/document-rewrite/context';
} from "@/components/workspace/appearance-controls";
import { WorkspaceSidebar } from "@/components/workspace/workspace-sidebar";
import { UserIdentityBadge } from "@/components/workspace/user-identity-badge";
import { CommandPalette } from "@/components/workspace/command-palette";
import { Toaster } from "sonner";
import { SubtasksProvider } from "@/core/tasks/context";
import { ArtifactsProvider } from "@/components/workspace/artifacts";
import { DocumentRewritePresentationProvider } from "@/core/document-rewrite/context";
import { RouteBaseProvider } from '@/core/route-base';
import { agentRefOfThread } from '@/core/threads/utils';
import { RouteBaseProvider } from "@/core/route-base";
import { agentRefOfThread } from "@/core/threads/utils";
import ChatsPage from './ChatsPage';
import ChatPage from './ChatPage';
import IframeChatPage from './IframeChatPage';
import AgentsPage from './AgentsPage';
import AgentChatPage from './AgentChatPage';
import SentimentAgentChatPage from './SentimentAgentChatPage';
import NewAgentPage from './NewAgentPage';
import AdminAgentsPage from './AdminAgentsPage';
import AdminScheduledTasksPage from './AdminScheduledTasksPage';
import AdminSkillsPage from './AdminSkillsPage';
import ScheduledTaskRunDetailPage from './ScheduledTaskRunDetailPage';
import ScheduledTasksPage from './ScheduledTasksPage';
import LlmWikiKnowledgePage from './LlmWikiKnowledgePage';
import WeKnoraWikiManagementPage from '@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage';
import WeKnoraWikiPageView from '@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView';
import MarkdownDocxEditorPage from './MarkdownDocxEditorPage';
import LlmMetricsPage from './LlmMetricsPage';
import CheckpointMigrationPage from './CheckpointMigrationPage';
import MemoryPage from './MemoryPage';
import PromptPrefixSettingsPage from './PromptPrefixSettingsPage';
import PrivateSquareSettingsPage from './PrivateSquareSettingsPage';
import ModelConcurrencySettingsPage from './ModelConcurrencySettingsPage';
import ModelManagementPage from './ModelManagementPage';
import PublishSquaresSettingsPage from './PublishSquaresSettingsPage';
import SkillCompressionSettingsPage from './SkillCompressionSettingsPage';
import SkillEvolutionPage from './SkillEvolutionPage';
import SkillsPage from './SkillsPage';
import UsersPage from './UsersPage';
import AdminUserActivityPage from './AdminUserActivityPage';
import AdminMemoryPage from './AdminMemoryPage';
import AdminUserHubPage from './AdminUserHubPage';
import TagsPage from './TagsPage';
import PositionsPage from './PositionsPage';
import PositionRolesPage from './PositionRolesPage';
import PluginManagementPage from './PluginManagementPage';
import RecommendedQuestionsPage from './RecommendedQuestionsPage';
import AIWritingPage from '@/open-canvas/pages/AIWritingPage';
import DeepResearchPage from './DeepResearchPage';
import ReportStructureCreatePage from './deep-research/ReportStructureCreatePage';
import EnterpriseResearchPage from './EnterpriseResearchPage';
import StudioNotebooksPage from './StudioNotebooksPage';
import StudioNotebookDetailPage from './StudioNotebookDetailPage';
import RoundtablePlanningPage from '@/roundtable-planning/pages/RoundtablePlanningPage';
import PositionRoundtablePage from '@/position-roundtable/pages/PositionRoundtablePage';
import EightBfRoundtablePage from '@/eightbf-roundtable/pages/EightBfRoundtablePage';
import BusinessChainEditorPage from '@/roundtable-planning/pages/BusinessChainEditorPage';
import BusinessChainListPage from '@/roundtable-planning/pages/BusinessChainListPage';
import DispatchChainPreviewPage from '@/roundtable-planning/pages/DispatchChainPreviewPage';
import '@/roundtable-planning/styles/roundtable-planning.css';
import ChatsPage from "./ChatsPage";
import ChatPage from "./ChatPage";
import ArtifactLibraryPage from "./ArtifactLibraryPage";
import IframeChatPage from "./IframeChatPage";
import AgentsPage from "./AgentsPage";
import AgentChatPage from "./AgentChatPage";
import SentimentAgentChatPage from "./SentimentAgentChatPage";
import NewAgentPage from "./NewAgentPage";
import AdminAgentsPage from "./AdminAgentsPage";
import AdminScheduledTasksPage from "./AdminScheduledTasksPage";
import AdminSkillsPage from "./AdminSkillsPage";
import ScheduledTaskRunDetailPage from "./ScheduledTaskRunDetailPage";
import ScheduledTasksPage from "./ScheduledTasksPage";
import LlmWikiKnowledgePage from "./LlmWikiKnowledgePage";
import AssistantKnowledgePage from "./AssistantKnowledgePage";
import WeKnoraWikiManagementPage from "@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiManagementPage";
import WeKnoraWikiPageView from "@/strategy-components/components/resource-management/llmwiki/WeKnoraWikiPageView";
import WeKnoraPackageTransfersPage from "@/strategy-components/components/resource-management/llmwiki/WeKnoraPackageTransfersPage";
import MarkdownDocxEditorPage from "./MarkdownDocxEditorPage";
import LlmMetricsPage from "./LlmMetricsPage";
import CheckpointMigrationPage from "./CheckpointMigrationPage";
import MemoryPage from "./MemoryPage";
import PromptPrefixSettingsPage from "./PromptPrefixSettingsPage";
import PrivateSquareSettingsPage from "./PrivateSquareSettingsPage";
import ModelConcurrencySettingsPage from "./ModelConcurrencySettingsPage";
import ModelManagementPage from "./ModelManagementPage";
import PublishSquaresSettingsPage from "./PublishSquaresSettingsPage";
import SkillCompressionSettingsPage from "./SkillCompressionSettingsPage";
import SkillEvolutionPage from "./SkillEvolutionPage";
import SkillsPage from "./SkillsPage";
import UsersPage from "./UsersPage";
import AdminUserActivityPage from "./AdminUserActivityPage";
import AdminMemoryPage from "./AdminMemoryPage";
import AdminUserHubPage from "./AdminUserHubPage";
import TagsPage from "./TagsPage";
import PositionsPage from "./PositionsPage";
import PositionRolesPage from "./PositionRolesPage";
import PluginManagementPage from "./PluginManagementPage";
import RecommendedQuestionsPage from "./RecommendedQuestionsPage";
import AIWritingPage from "@/open-canvas/pages/AIWritingPage";
import DeepResearchPage from "./DeepResearchPage";
import ReportStructureCreatePage from "./deep-research/ReportStructureCreatePage";
import EnterpriseResearchPage from "./EnterpriseResearchPage";
import StudioNotebooksPage from "./StudioNotebooksPage";
import StudioNotebookDetailPage from "./StudioNotebookDetailPage";
import RoundtablePlanningPage from "@/roundtable-planning/pages/RoundtablePlanningPage";
import PositionRoundtablePage from "@/position-roundtable/pages/PositionRoundtablePage";
import EightBfRoundtablePage from "@/eightbf-roundtable/pages/EightBfRoundtablePage";
import BusinessChainEditorPage from "@/roundtable-planning/pages/BusinessChainEditorPage";
import BusinessChainListPage from "@/roundtable-planning/pages/BusinessChainListPage";
import DispatchChainPreviewPage from "@/roundtable-planning/pages/DispatchChainPreviewPage";
import "@/roundtable-planning/styles/roundtable-planning.css";
import ReportCollaborationPage, {
ReportCollaborationSessionPage,
} from '@/report-collaboration/pages/ReportCollaborationPage';
import '@/report-collaboration/styles/report-collaboration.css';
import { cn } from '@/lib/utils';
} from "@/report-collaboration/pages/ReportCollaborationPage";
import "@/report-collaboration/styles/report-collaboration.css";
import { cn } from "@/lib/utils";
function NotebookDetailPageWithKey() {
const { notebookId } = useParams<{ notebookId: string }>();
@@ -119,11 +128,12 @@ function ReportCollaborationSessionRoute({
function shouldShowCompactTopActions(pathname: string): boolean {
return (
pathname === '/page/workspace/agents' ||
pathname === '/page/workspace/skills' ||
pathname === '/page/workspace/memory' ||
pathname === '/page/workspace/knowledge' ||
pathname.startsWith('/page/workspace/knowledge/')
pathname === "/page/workspace/agents" ||
pathname === "/page/workspace/skills" ||
pathname === "/page/workspace/memory" ||
pathname === "/page/workspace/library" ||
pathname === "/page/workspace/knowledge" ||
pathname.startsWith("/page/workspace/knowledge/")
);
}
@@ -181,47 +191,52 @@ export function WorkspaceLayout({
// 全量模式下通用问答(/page/workspace/chats*)的历史侧栏默认展开;其余紧凑页
// (如智能体对话,为聊天区更宽而默认收起)维持原默认。
const pathname = useLocation().pathname;
const isGeneralQa = pathname.startsWith('/page/workspace/chats');
const isGeneralQa = pathname.startsWith("/page/workspace/chats");
const showCompactTopActions =
headerHidden && !compactActionsHidden && shouldShowCompactTopActions(pathname);
headerHidden &&
!compactActionsHidden &&
shouldShowCompactTopActions(pathname);
const pageSidebarWidth = pageSidebarHidden
? '0rem'
? "0rem"
: pageSidebarDark
? pageSidebarCollapsed
? '0rem'
: '118px'
? "0rem"
: "118px"
: pageSidebarCollapsed
? '4rem'
: '16rem';
? "4rem"
: "16rem";
return (
<SidebarProvider
className={[
'relative h-full w-full overflow-x-hidden',
'[&_[data-slot=sidebar-container]]:absolute',
'[&_[data-slot=sidebar-container]]:left-0',
'[&_[data-slot=sidebar-container]]:top-0',
].join(' ')}
"relative h-full w-full overflow-x-hidden",
"[&_[data-slot=sidebar-container]]:absolute",
"[&_[data-slot=sidebar-container]]:left-0",
"[&_[data-slot=sidebar-container]]:top-0",
].join(" ")}
style={
{
'--page-sidebar-width': pageSidebarWidth,
'--header-height': headerHidden ? '0rem' : '4rem',
"--page-sidebar-width": pageSidebarWidth,
"--header-height": headerHidden ? "0rem" : "4rem",
} as CSSProperties
}
// Agent and general-chat routes share this layout component. Keep their
// sidebar state separate: otherwise React preserves the collapsed agent
// state when a history item switches to `/chats/:thread_id`, while the
// new page's header button is meant to control GeneralQaSidebar.
key={`${String(compactWorkspaceSidebar)}:${isGeneralQa ? 'general' : 'workspace'}`}
key={`${String(compactWorkspaceSidebar)}:${isGeneralQa ? "general" : "workspace"}`}
data-page-sidebar-collapsed={pageSidebarCollapsed}
defaultOpen={
forceWorkspaceSidebarCollapsed
? false
: (defaultHistorySidebarOpen ?? (isGeneralQa || !compactWorkspaceSidebar))
: (defaultHistorySidebarOpen ??
(isGeneralQa || !compactWorkspaceSidebar))
}
open={forceWorkspaceSidebarCollapsed ? false : undefined}
onOpenChange={forceWorkspaceSidebarCollapsed ? () => undefined : undefined}
onOpenChange={
forceWorkspaceSidebarCollapsed ? () => undefined : undefined
}
disableToggle={forceWorkspaceSidebarCollapsed}
>
{!workspaceSidebarHidden && (
@@ -231,7 +246,7 @@ export function WorkspaceLayout({
/>
)}
<SidebarInset
className={cn('min-w-0', insetOverflowHidden && 'overflow-hidden')}
className={cn("min-w-0", insetOverflowHidden && "overflow-hidden")}
>
{showCompactTopActions ? (
<>
@@ -268,9 +283,14 @@ export function ChatRuntime({ children }: { children: ReactNode }) {
*/
function NewChatContent() {
const location = useLocation();
const agentId = new URLSearchParams(location.search).get('agent');
const agentId = new URLSearchParams(location.search).get("agent");
if (agentId) {
return <AgentChatPage key={`${agentId}:${location.key}`} embeddedAgentId={agentId} />;
return (
<AgentChatPage
key={`${agentId}:${location.key}`}
embeddedAgentId={agentId}
/>
);
}
return <ChatPage />;
}
@@ -417,8 +437,8 @@ function AgentAwareChatThreadContent() {
};
// Task-bound threads need their task host to preserve the purpose gate,
// action partitioning, and other task-specific controls.
const isTaskThread = typeof record.metadata?.taskId === 'string';
setAgentId(isTaskThread ? null : agentRefOfThread(record) ?? null);
const isTaskThread = typeof record.metadata?.taskId === "string";
setAgentId(isTaskThread ? null : (agentRefOfThread(record) ?? null));
})
.catch(() => {
if (!cancelled) setAgentId(null);
@@ -545,13 +565,34 @@ export default function WorkspaceRoutes({
path="chats/:thread_id"
element={<ChatThreadEntryRoute layoutProps={layoutProps} />}
/>
<Route
path="library"
element={
<WorkspaceLayout
{...layoutProps}
workspaceSidebarHidden={
layoutProps.workspaceSidebarHidden ||
!layoutProps.pageSidebarHidden
}
>
<ArtifactLibraryPage />
</WorkspaceLayout>
}
/>
<Route
path="chats_iframe/new"
element={<IframeChatEntryRoute layoutProps={layoutProps} hasThreadParam={false} />}
element={
<IframeChatEntryRoute
layoutProps={layoutProps}
hasThreadParam={false}
/>
}
/>
<Route
path="chats_iframe/:thread_id"
element={<IframeChatEntryRoute layoutProps={layoutProps} hasThreadParam />}
element={
<IframeChatEntryRoute layoutProps={layoutProps} hasThreadParam />
}
/>
<Route
path="agents"
@@ -585,7 +626,10 @@ export default function WorkspaceRoutes({
path="agents/sentiment-analysis/chats"
element={
isSentimentAgentVisible() ? (
<Navigate to="/page/workspace/agents/sentiment-analysis/chats/new" replace />
<Navigate
to="/page/workspace/agents/sentiment-analysis/chats/new"
replace
/>
) : (
<Navigate to="/page/workspace/agents" replace />
)
@@ -615,7 +659,9 @@ export default function WorkspaceRoutes({
<AgentChatPage
showAgentList
compactAppearanceActions={
headerHidden && !compactActionsHidden ? <CompactAppearanceActions /> : undefined
headerHidden && !compactActionsHidden ? (
<CompactAppearanceActions />
) : undefined
}
/>
</ChatRuntime>
@@ -714,7 +760,9 @@ export default function WorkspaceRoutes({
element={
<WorkspaceLayout {...layoutProps}>
<ChatRuntime>
<RoundtablePlanningPage compactChrome={headerHidden && !compactActionsHidden} />
<RoundtablePlanningPage
compactChrome={headerHidden && !compactActionsHidden}
/>
</ChatRuntime>
</WorkspaceLayout>
}
@@ -724,7 +772,9 @@ export default function WorkspaceRoutes({
element={
<WorkspaceLayout {...layoutProps}>
<ChatRuntime>
<EightBfRoundtablePage compactChrome={headerHidden && !compactActionsHidden} />
<EightBfRoundtablePage
compactChrome={headerHidden && !compactActionsHidden}
/>
</ChatRuntime>
</WorkspaceLayout>
}
@@ -734,7 +784,9 @@ export default function WorkspaceRoutes({
element={
<WorkspaceLayout {...layoutProps}>
<ChatRuntime>
<PositionRoundtablePage compactChrome={headerHidden && !compactActionsHidden} />
<PositionRoundtablePage
compactChrome={headerHidden && !compactActionsHidden}
/>
</ChatRuntime>
</WorkspaceLayout>
}
@@ -794,6 +846,14 @@ export default function WorkspaceRoutes({
</WorkspaceLayout>
}
/>
<Route
path="knowledge/assistant"
element={
<WorkspaceLayout {...layoutProps}>
<AssistantKnowledgePage />
</WorkspaceLayout>
}
/>
<Route
path="knowledge/:noteId/wiki/view"
element={
@@ -810,6 +870,14 @@ export default function WorkspaceRoutes({
</WorkspaceLayout>
}
/>
<Route
path="knowledge/:noteId/packages"
element={
<WorkspaceLayout {...layoutProps}>
<WeKnoraPackageTransfersPage />
</WorkspaceLayout>
}
/>
<Route
path="knowledge/markdown-editor"
element={
@@ -834,7 +902,9 @@ export default function WorkspaceRoutes({
path="scheduled/tasks"
element={
<WorkspaceLayout {...layoutProps}>
<ScheduledTasksPage compactChrome={headerHidden && !compactActionsHidden} />
<ScheduledTasksPage
compactChrome={headerHidden && !compactActionsHidden}
/>
</WorkspaceLayout>
}
/>
@@ -850,7 +920,7 @@ export default function WorkspaceRoutes({
<Route
path="admin"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps} pageSidebarHidden>
<AdminUserHubPage />
</WorkspaceLayout>
@@ -875,7 +945,7 @@ export default function WorkspaceRoutes({
<Route
path="admin/scheduled-tasks"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<AdminScheduledTasksPage />
</WorkspaceLayout>
@@ -887,7 +957,7 @@ export default function WorkspaceRoutes({
<Route
path="admin/agents"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<AdminAgentsPage />
</WorkspaceLayout>
@@ -899,7 +969,7 @@ export default function WorkspaceRoutes({
<Route
path="admin/skills"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<AdminSkillsPage />
</WorkspaceLayout>
@@ -919,7 +989,7 @@ export default function WorkspaceRoutes({
<Route
path="admin/plugins"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<PluginManagementPage />
</WorkspaceLayout>
@@ -931,7 +1001,7 @@ export default function WorkspaceRoutes({
<Route
path="admin/positions"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<PositionsPage />
</WorkspaceLayout>
@@ -943,7 +1013,7 @@ export default function WorkspaceRoutes({
<Route
path="admin/position-roles"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<PositionRolesPage />
</WorkspaceLayout>
@@ -955,7 +1025,7 @@ export default function WorkspaceRoutes({
<Route
path="settings/recommended-questions"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<RecommendedQuestionsPage />
</WorkspaceLayout>
@@ -967,7 +1037,7 @@ export default function WorkspaceRoutes({
<Route
path="settings/prompt-prefix"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<PromptPrefixSettingsPage />
</WorkspaceLayout>
@@ -979,7 +1049,7 @@ export default function WorkspaceRoutes({
<Route
path="settings/private-square"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<PrivateSquareSettingsPage />
</WorkspaceLayout>
@@ -991,7 +1061,7 @@ export default function WorkspaceRoutes({
<Route
path="admin/models"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<ModelManagementPage />
</WorkspaceLayout>
@@ -1003,7 +1073,7 @@ export default function WorkspaceRoutes({
<Route
path="settings/model-concurrency"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<ModelConcurrencySettingsPage />
</WorkspaceLayout>
@@ -1015,7 +1085,7 @@ export default function WorkspaceRoutes({
<Route
path="settings/publish-squares"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<PublishSquaresSettingsPage />
</WorkspaceLayout>
@@ -1027,7 +1097,7 @@ export default function WorkspaceRoutes({
<Route
path="settings/skill-compression"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<SkillCompressionSettingsPage />
</WorkspaceLayout>
@@ -1039,7 +1109,7 @@ export default function WorkspaceRoutes({
<Route
path="settings/llm-metrics"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<LlmMetricsPage />
</WorkspaceLayout>
@@ -1051,7 +1121,7 @@ export default function WorkspaceRoutes({
<Route
path="settings/checkpoint-migration"
element={
getStoredAuth()?.system_role === 'admin' ? (
getStoredAuth()?.system_role === "admin" ? (
<WorkspaceLayout {...layoutProps}>
<CheckpointMigrationPage />
</WorkspaceLayout>
@@ -0,0 +1,514 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { apiFetch } from "@/core/api/fetch-client";
import { getBackendBaseURL } from "@/core/config";
const BASE = () => `${getBackendBaseURL()}/api/assistant-knowledge`;
export interface AssistantKnowledgeBase {
id: string;
name: string;
description?: string;
base_kind?: "global" | "custom";
is_global?: boolean;
status: string;
source_count?: number;
page_count?: number;
entity_count?: number;
relation_count?: number;
created_at: string;
updated_at: string;
}
export interface AssistantWikiPage {
id: string;
base_id: string;
slug: string;
title: string;
page_type: string;
content_markdown: string;
summary: string;
current_revision_id?: string | null;
updated_at: string;
aliases?: string[];
tags?: string[];
category_path?: string[];
parent_slug?: string;
wiki_path?: string;
in_links?: string[];
out_links?: string[];
source_refs?: string[];
origins?: { mapping_id?: string; source_name: string; slug: string }[];
matched_content?: string;
matched_section?: {
id?: string;
section_index: number;
heading?: string | null;
content: string;
score: number;
source_chunk_id?: string | null;
embedding_model?: string | null;
embedding_fingerprint?: string | null;
};
}
export interface AssistantWikiRevision {
id: string;
page_id: string;
revision_no: number;
summary: string;
content_markdown: string;
change_type: string;
created_by?: string | null;
created_at: string;
}
export interface AssistantKnowledgeSource {
id: string;
base_id: string;
source_type: "weknora" | "skill" | "local_file";
source_name: string;
source_key: string;
status: string;
updated_at: string;
}
export interface AssistantImportJob {
id: string;
base_id: string;
source_id: string;
mirrored_global_base_id?: string;
mirrored_global_job_id?: string;
package_download_url?: string;
trigger: string;
status: string;
phase: string;
counts: Record<string, number>;
last_error_message?: string | null;
created_at: string;
completed_at?: string | null;
}
export interface LlmWikiExportPackageJob {
id: string;
mapping_id: string;
status: string;
phase: string;
counts: Record<string, number>;
package_download_url?: string;
status_url?: string;
error?: string | null;
created_at?: string | null;
updated_at?: string | null;
completed_at?: string | null;
file_exists?: boolean;
file_size_bytes?: number;
}
async function readJson<T>(response: Response): Promise<T> {
if (!response.ok) {
const data = await response.json().catch(() => null);
const detail =
data && typeof data === "object" && "detail" in data
? (data as { detail: unknown }).detail
: `请求失败 (${response.status})`;
throw new Error(
typeof detail === "string" ? detail : JSON.stringify(detail),
);
}
return (await response.json()) as T;
}
function normalizeImportJob(job: AssistantImportJob): AssistantImportJob {
const url = normalizeBackendUrl(job.package_download_url);
return url ? { ...job, package_download_url: url } : job;
}
function normalizeBackendUrl(
url: string | undefined | null,
): string | undefined {
if (!url || /^https?:\/\//i.test(url)) return url || undefined;
return `${getBackendBaseURL()}${url.startsWith("/") ? url : `/${url}`}`;
}
function normalizeExportPackageJob(
job: LlmWikiExportPackageJob,
): LlmWikiExportPackageJob {
return {
...job,
package_download_url: normalizeBackendUrl(job.package_download_url),
status_url: normalizeBackendUrl(job.status_url),
};
}
export async function getAssistantKnowledgeBase() {
const data = await readJson<{ base: AssistantKnowledgeBase | null }>(
await apiFetch(`${BASE()}/base`),
);
return data.base;
}
export async function getAssistantKnowledgeBaseById(baseId: string) {
const data = await readJson<{ base: AssistantKnowledgeBase }>(
await apiFetch(`${BASE()}/bases/${encodeURIComponent(baseId)}`),
);
return data.base;
}
export async function listAssistantKnowledgeBases() {
return (
await readJson<{ bases: AssistantKnowledgeBase[] }>(
await apiFetch(`${BASE()}/bases`),
)
).bases;
}
export async function initializeAssistantKnowledgeBase(name = "知识梳理总库") {
const data = await readJson<{ base: AssistantKnowledgeBase }>(
await apiFetch(`${BASE()}/base/initialize`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ name }),
}),
);
return data.base;
}
export async function createAssistantKnowledgeBase(input: {
name: string;
description?: string;
}) {
const data = await readJson<{ base: AssistantKnowledgeBase }>(
await apiFetch(`${BASE()}/bases`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
return data.base;
}
export async function updateAssistantKnowledgeBase(input: {
id: string;
name: string;
description?: string;
}) {
const data = await readJson<{ base: AssistantKnowledgeBase }>(
await apiFetch(`${BASE()}/bases/${encodeURIComponent(input.id)}`, {
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
name: input.name,
description: input.description || "",
}),
}),
);
return data.base;
}
export async function deleteAssistantKnowledgeBase(id: string) {
const response = await apiFetch(
`${BASE()}/bases/${encodeURIComponent(id)}`,
{ method: "DELETE" },
);
if (!response.ok) await readJson(response);
}
function withBaseParam(baseId?: string | null) {
const params = new URLSearchParams();
if (baseId) params.set("base_id", baseId);
return params;
}
export async function listAssistantWikiPages(q = "", baseId?: string | null) {
const params = withBaseParam(baseId);
params.set("limit", "500");
if (q.trim()) params.set("q", q.trim());
const pages: AssistantWikiPage[] = [];
let offset: number | null = 0;
do {
params.set("offset", String(offset));
const batch: { pages: AssistantWikiPage[]; next_offset?: number | null } =
await readJson(await apiFetch(`${BASE()}/wiki-pages?${params}`));
pages.push(...batch.pages);
offset = batch.next_offset ?? null;
} while (offset !== null);
return pages;
}
export async function searchAssistantWiki(q: string, baseId: string) {
return readJson<{
results: (AssistantWikiPage & { score: number; retrieval_mode: string })[];
retrieval_mode: string;
}>(
await apiFetch(
`${BASE()}/search?${new URLSearchParams({ q, base_id: baseId })}`,
),
);
}
export async function listWikiTransfers(id: string, assistant: boolean) {
const url = assistant
? `${BASE()}/bases/${encodeURIComponent(id)}/export-package`
: `${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(id)}/import-package`;
return (
await readJson<{ jobs: LlmWikiExportPackageJob[] }>(await apiFetch(url))
).jobs.map(normalizeExportPackageJob);
}
export async function createAssistantExportPackage(id: string) {
return readJson<LlmWikiExportPackageJob>(
await apiFetch(`${BASE()}/bases/${encodeURIComponent(id)}/export-package`, {
method: "POST",
}),
);
}
export async function importOrdinaryWikiPackage(id: string, file: File) {
const body = new FormData();
body.set("file", file);
return readJson<LlmWikiExportPackageJob>(
await apiFetch(
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(id)}/import-package`,
{ method: "POST", body },
),
);
}
export async function listAssistantKnowledgeSources(baseId?: string | null) {
const params = withBaseParam(baseId);
return (
await readJson<{ sources: AssistantKnowledgeSource[] }>(
await apiFetch(`${BASE()}/sources?${params}`),
)
).sources;
}
export async function listAssistantImportJobs(baseId?: string | null) {
const params = withBaseParam(baseId);
return (
await readJson<{ jobs: AssistantImportJob[] }>(
await apiFetch(`${BASE()}/import-jobs?${params}`),
)
).jobs.map(normalizeImportJob);
}
export async function listAssistantWikiRevisions(
slug: string,
baseId?: string | null,
) {
const params = withBaseParam(baseId);
return (
await readJson<{ revisions: AssistantWikiRevision[] }>(
await apiFetch(
`${BASE()}/wiki-pages/${encodeURIComponent(slug)}/revisions?${params}`,
),
)
).revisions;
}
export async function rollbackAssistantWikiRevision(
slug: string,
revisionId: string,
baseId?: string | null,
) {
const params = withBaseParam(baseId);
return readJson<AssistantWikiRevision>(
await apiFetch(
`${BASE()}/wiki-pages/${encodeURIComponent(slug)}/revisions/${encodeURIComponent(revisionId)}/rollback?${params}`,
{ method: "POST" },
),
);
}
export async function exportLlmWikiToAssistant(
mappingId: string,
assistantBaseId?: string | null,
) {
return normalizeImportJob(
await readJson<AssistantImportJob>(
await apiFetch(
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(mappingId)}/export-to-assistant`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ assistant_base_id: assistantBaseId || null }),
},
),
),
);
}
export async function createLlmWikiExportPackage(mappingId: string) {
return normalizeExportPackageJob(
await readJson<LlmWikiExportPackageJob>(
await apiFetch(
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(mappingId)}/export-package`,
{ method: "POST" },
),
),
);
}
export async function listLlmWikiExportPackages(mappingId: string) {
return (
await readJson<{ jobs: LlmWikiExportPackageJob[] }>(
await apiFetch(
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(mappingId)}/export-package`,
),
)
).jobs.map(normalizeExportPackageJob);
}
export async function getLlmWikiExportPackageStatus(
mappingId: string,
jobId: string,
) {
return normalizeExportPackageJob(
await readJson<LlmWikiExportPackageJob>(
await apiFetch(
`${getBackendBaseURL()}/api/llmwiki/knowledge-bases/${encodeURIComponent(mappingId)}/export-package/${encodeURIComponent(jobId)}`,
),
),
);
}
export async function reimportAssistantKnowledgeSource(sourceId: string) {
return normalizeImportJob(
await readJson<AssistantImportJob>(
await apiFetch(
`${BASE()}/sources/${encodeURIComponent(sourceId)}/reimport`,
{
method: "POST",
},
),
),
);
}
export function getAssistantImportPackageUrl(jobId: string) {
return `${BASE()}/import-jobs/${encodeURIComponent(jobId)}/package`;
}
export async function uploadAssistantWeKnoraPackage(input: {
file: File;
assistantBaseId?: string | null;
}) {
const body = new FormData();
body.set("file", input.file);
if (input.assistantBaseId)
body.set("assistant_base_id", input.assistantBaseId);
return normalizeImportJob(
await readJson<AssistantImportJob>(
await apiFetch(`${BASE()}/imports/weknora-package`, {
method: "POST",
body,
}),
),
);
}
export async function deleteAssistantKnowledgeSource(sourceId: string) {
const response = await apiFetch(
`${BASE()}/sources/${encodeURIComponent(sourceId)}`,
{
method: "DELETE",
},
);
if (!response.ok) await readJson(response);
}
const KEY = "assistant-knowledge";
export function useAssistantKnowledgeBase(baseId?: string | null) {
return useQuery({
queryKey: [KEY, "base", baseId ?? "global"],
queryFn: () =>
baseId
? getAssistantKnowledgeBaseById(baseId)
: getAssistantKnowledgeBase(),
});
}
export function useAssistantKnowledgeBases(enabled = true) {
return useQuery({
queryKey: [KEY, "bases"],
queryFn: listAssistantKnowledgeBases,
enabled,
});
}
export function useAssistantWikiPages(query = "", baseId?: string | null) {
return useQuery({
queryKey: [KEY, "pages", baseId ?? "global", query],
queryFn: () => listAssistantWikiPages(query, baseId),
});
}
export function useAssistantKnowledgeSources(baseId?: string | null, enabled = true) {
return useQuery({
queryKey: [KEY, "sources", baseId ?? "global"],
queryFn: () => listAssistantKnowledgeSources(baseId),
enabled,
});
}
export function useAssistantImportJobs(baseId?: string | null) {
return useQuery({
queryKey: [KEY, "jobs", baseId ?? "global"],
queryFn: () => listAssistantImportJobs(baseId),
refetchInterval: (query) =>
(query.state.data ?? []).some((job) =>
["queued", "running"].includes(job.status),
)
? 3000
: false,
});
}
export function useAssistantKnowledgeActions() {
const client = useQueryClient();
const invalidate = () => client.invalidateQueries({ queryKey: [KEY] });
return {
initialize: useMutation({
mutationFn: initializeAssistantKnowledgeBase,
onSuccess: invalidate,
}),
createBase: useMutation({
mutationFn: createAssistantKnowledgeBase,
onSuccess: invalidate,
}),
updateBase: useMutation({
mutationFn: updateAssistantKnowledgeBase,
onSuccess: invalidate,
}),
deleteBase: useMutation({
mutationFn: deleteAssistantKnowledgeBase,
onSuccess: invalidate,
}),
uploadWeKnoraPackage: useMutation({
mutationFn: uploadAssistantWeKnoraPackage,
onSuccess: invalidate,
}),
reimportSource: useMutation({
mutationFn: reimportAssistantKnowledgeSource,
onSuccess: invalidate,
}),
deleteSource: useMutation({
mutationFn: deleteAssistantKnowledgeSource,
onSuccess: invalidate,
}),
rollback: useMutation({
mutationFn: ({
slug,
revisionId,
baseId,
}: {
slug: string;
revisionId: string;
baseId?: string | null;
}) => rollbackAssistantWikiRevision(slug, revisionId, baseId),
onSuccess: invalidate,
}),
};
}
@@ -1,6 +1,8 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { apiFetch } from "@/core/api/fetch-client";
import { getCsrfToken } from "@/core/api/csrf";
import { getAuthorizationHeaderValue } from "@/core/auth";
import { useLiveAccount } from "@/core/auth/use-live-account";
import { getBackendBaseURL } from "@/core/config";
@@ -29,7 +31,7 @@ export interface LlmWikiKnowledgeBase {
remote_status: "available" | "missing";
name: string;
description: string;
type: "document" | "faq";
type: "document" | "faq" | "wiki";
publication_status: "private" | "pending" | "published" | "rejected";
owner_user_id?: string | null;
is_owner: boolean;
@@ -37,6 +39,12 @@ export interface LlmWikiKnowledgeBase {
knowledge_count: number;
chunk_count: number;
processing_count: number;
share_count?: number;
is_processing?: boolean;
is_temporary?: boolean;
wiki_index_enabled?: boolean;
external_search_enabled?: boolean;
external_search_updated_at?: string | null;
embedding_model_id?: string | null;
vector_store_id?: string | null;
vector_store_name?: string | null;
@@ -44,19 +52,65 @@ export interface LlmWikiKnowledgeBase {
vector_store_engine_type?: string | null;
vector_store_status?: string | null;
chunking_config?: Record<string, unknown>;
image_processing_config?: Record<string, unknown>;
indexing_strategy?: Record<string, boolean>;
capabilities?: Record<string, boolean>;
extract_config?: Record<string, unknown>;
faq_config?: Record<string, unknown>;
wiki_config?: Record<string, unknown>;
question_generation_config?: Record<string, unknown>;
auto_tag_config?: Record<string, unknown>;
vlm_config?: Record<string, unknown>;
asr_config?: Record<string, unknown>;
summary_model_id?: string | null;
storage_provider?: string | null;
created_at?: string | null;
updated_at?: string | null;
published_at?: string | null;
remote_created_at?: string | null;
remote_updated_at?: string | null;
is_conversation_deposit?: boolean;
}
export interface PublicLlmWikiKnowledgeBase extends LlmWikiKnowledgeBase {
wiki: {
status: "ready" | "empty" | "unavailable";
page_count: number;
loaded_page_count: number;
stats_complete: boolean;
page_type_counts: Record<string, number>;
page_status_counts: Record<string, number>;
index_mode: "explicit" | "generated" | "none" | "unknown";
has_explicit_index: boolean;
index_slug: string | null;
default_slug: string | null;
created_at: string | null;
updated_at: string | null;
embed_route: string;
};
local_index: {
enabled: boolean;
wiki_index_enabled: boolean;
state: LlmWikiWikiIndexStatus["state"];
fully_vectorized: boolean;
retrieval_mode: "vector" | "wiki_api";
remote_page_count: number;
local_page_count: number;
ready_page_count: number;
failed_page_count: number;
vector_count: number;
last_started_at?: string | null;
last_completed_at?: string | null;
last_success_at?: string | null;
updated_at?: string | null;
};
}
export interface PublicLlmWikiKnowledgeBaseList {
knowledge_bases: PublicLlmWikiKnowledgeBase[];
total: number;
}
export interface LlmWikiDocument {
id: string;
type?: string;
@@ -115,6 +169,7 @@ export interface LlmWikiGraph {
}
export interface LlmWikiWikiPage {
page_metadata?: { tags?: string[]; [key: string]: unknown };
id: string;
slug: string;
title: string;
@@ -167,7 +222,14 @@ export interface LlmWikiWikiIndex {
export interface LlmWikiWikiIndexStatus {
knowledge_base_id: string;
state: "not_synced" | "idle" | "running" | "partial" | "failed" | "rebuilding" | "rebuild_required";
state:
| "not_synced"
| "idle"
| "running"
| "partial"
| "failed"
| "rebuilding"
| "rebuild_required";
local_index_enabled: boolean;
wiki_index_enabled: boolean;
fully_vectorized: boolean;
@@ -203,6 +265,25 @@ export interface LlmWikiWikiIndexStatus {
}>;
}
export interface LocalWikiEncoderStatus {
provider: string;
model: string;
dimensions?: number | null;
state: "idle" | "starting" | "ready" | "failed";
running: boolean;
last_error?: string | null;
started_at?: string | null;
ready_at?: string | null;
offline_only: boolean;
model_path_configured: boolean;
model_path_exists: boolean;
model_files_ready: boolean;
process_isolation: boolean;
model_path: string;
uploaded_files?: number;
uploaded_bytes?: number;
}
export interface LlmWikiSourceContext {
knowledge_base_name?: string;
document_title?: string;
@@ -292,14 +373,16 @@ function userFacingKnowledgeMessage(value: string): string {
async function readJson<T>(response: Response, context: string): Promise<T> {
if (!response.ok) {
const payload = await response.json().catch(() => null as unknown);
const rawDetail = payload && typeof payload === "object" && "detail" in payload
? (payload as { detail: unknown }).detail
: null;
const detail = rawDetail && typeof rawDetail === "object" && "message" in rawDetail
? String((rawDetail as { message: unknown }).message)
: rawDetail
? String(rawDetail)
: `${context}失败(${response.status})`;
const rawDetail =
payload && typeof payload === "object" && "detail" in payload
? (payload as { detail: unknown }).detail
: null;
const detail =
rawDetail && typeof rawDetail === "object" && "message" in rawDetail
? String((rawDetail as { message: unknown }).message)
: rawDetail
? String(rawDetail)
: `${context}失败(${response.status})`;
throw new Error(userFacingKnowledgeMessage(detail));
}
if (response.status === 204) return undefined as T;
@@ -307,10 +390,7 @@ async function readJson<T>(response: Response, context: string): Promise<T> {
}
export async function getLlmWikiRuntime(): Promise<LlmWikiRuntime> {
return readJson(
await apiFetch(`${API()}/runtime`),
"读取 LLMWiki 运行模式",
);
return readJson(await apiFetch(`${API()}/runtime`), "读取 LLMWiki 运行模式");
}
export async function listLlmWikiKnowledgeBases(
@@ -368,10 +448,10 @@ export async function getLlmWikiSourceContext(
knowledge_base_id: knowledgeBaseId,
chunk_id: chunkId,
});
const payload = await readJson<{ success: boolean; data: LlmWikiSourceContext }>(
await apiFetch(`${API()}/sources/context?${params}`),
"加载参考文献原文",
);
const payload = await readJson<{
success: boolean;
data: LlmWikiSourceContext;
}>(await apiFetch(`${API()}/sources/context?${params}`), "加载参考文献原文");
return payload.data;
}
@@ -484,9 +564,13 @@ export async function getLlmWikiProtectedFileUrl(
return `${PUBLIC_API()}/files/${encodeURIComponent(access.token)}`;
}
export async function getLlmWikiGraph(knowledgeBaseId: string): Promise<LlmWikiGraph> {
export async function getLlmWikiGraph(
knowledgeBaseId: string,
): Promise<LlmWikiGraph> {
return readJson(
await apiFetch(`${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/graph?limit=200`),
await apiFetch(
`${API()}/knowledge-bases/${encodeURIComponent(knowledgeBaseId)}/graph?limit=200`,
),
"读取知识图谱",
);
}
@@ -497,7 +581,10 @@ export async function listLlmWikiWikiPages(
pageSize: number,
query = "",
): Promise<LlmWikiWikiPageList> {
const params = new URLSearchParams({ page: String(page), page_size: String(pageSize) });
const params = new URLSearchParams({
page: String(page),
page_size: String(pageSize),
});
if (query.trim()) params.set("query", query.trim());
return readJson(
await apiFetch(
@@ -543,6 +630,80 @@ export async function vectorizeLlmWikiKnowledgeBase(
);
}
export async function getLocalWikiEncoderStatus(): Promise<LocalWikiEncoderStatus> {
return readJson(
await apiFetch(`${API()}/local-encoder/status`),
"读取本地编码器状态",
);
}
export async function startLocalWikiEncoder(): Promise<LocalWikiEncoderStatus> {
return readJson(
await apiFetch(`${API()}/local-encoder/start`, { method: "POST" }),
"启动本地编码器",
);
}
export async function configureLocalWikiEncoder(
modelPath: string,
): Promise<LocalWikiEncoderStatus> {
return readJson(
await apiFetch(`${API()}/local-encoder/config`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ model_path: modelPath }),
}),
"保存本地编码器目录",
);
}
export async function uploadLocalWikiEncoder(
files: File[],
onProgress?: (percent: number) => void,
): Promise<LocalWikiEncoderStatus> {
const body = new FormData();
for (const file of files) {
const webkitPath = (file as File & { webkitRelativePath?: string }).webkitRelativePath || file.name;
const parts = webkitPath.replace(/\\/g, "/").split("/").filter(Boolean);
const relativePath = parts.length > 1 ? parts.slice(1).join("/") : parts[0];
body.append("files", file, file.name);
body.append("paths", relativePath);
}
return new Promise((resolve, reject) => {
const request = new XMLHttpRequest();
request.open("POST", `${API()}/local-encoder/upload`);
request.withCredentials = true;
const authorization = getAuthorizationHeaderValue();
const csrf = getCsrfToken();
if (authorization) request.setRequestHeader("Authorization", authorization);
if (csrf) request.setRequestHeader("X-CSRF-Token", csrf);
request.upload.onprogress = (event) => {
if (event.lengthComputable && event.total > 0) {
onProgress?.(Math.min(99, Math.round((event.loaded / event.total) * 100)));
}
};
request.onerror = () => reject(new Error("上传本地编码器失败:网络连接中断"));
request.onload = () => {
let payload: unknown = null;
try {
payload = JSON.parse(request.responseText || "null");
} catch {
payload = null;
}
if (request.status < 200 || request.status >= 300) {
const detail = payload && typeof payload === "object" && "detail" in payload
? (payload as { detail: unknown }).detail
: `请求失败 (${request.status})`;
reject(new Error(typeof detail === "string" ? detail : JSON.stringify(detail)));
return;
}
onProgress?.(100);
resolve(payload as LocalWikiEncoderStatus);
};
request.send(body);
});
}
export async function exportLlmWikiWikiExcel(
knowledgeBaseId: string,
knowledgeBaseName = "LLMWiki",
@@ -742,13 +903,23 @@ export async function getPublicLlmWikiKnowledgeBase(
);
}
export async function listPublicLlmWikiKnowledgeBases(): Promise<PublicLlmWikiKnowledgeBaseList> {
return readJson(
await apiFetch(`${PUBLIC_API()}/knowledge-bases`),
"读取公开知识库列表",
);
}
export async function listPublicLlmWikiWikiPages(
knowledgeBaseId: string,
page: number,
pageSize: number,
query = "",
): Promise<LlmWikiWikiPageList> {
const params = new URLSearchParams({ page: String(page), page_size: String(pageSize) });
const params = new URLSearchParams({
page: String(page),
page_size: String(pageSize),
});
if (query.trim()) params.set("query", query.trim());
return readJson(
await apiFetch(
@@ -827,9 +998,7 @@ export async function resetLlmWikiSettings(): Promise<LlmWikiRuntime> {
);
}
export async function testLlmWikiConnection(
apiBaseUrl: string,
): Promise<void> {
export async function testLlmWikiConnection(apiBaseUrl: string): Promise<void> {
await readJson(
await apiFetch(`${SETTINGS_API()}/test`, {
method: "POST",
@@ -854,10 +1023,7 @@ export function useLlmWikiRuntime() {
});
}
export function useLlmWikiKnowledgeBases(
scope: LlmWikiScope,
enabled = true,
) {
export function useLlmWikiKnowledgeBases(scope: LlmWikiScope, enabled = true) {
const { userId } = useLiveAccount();
return useQuery({
queryKey: knowledgeBasesKey(userId, scope),
@@ -873,7 +1039,13 @@ export function useAgentLlmWikiKnowledgeBases(
) {
const { userId } = useLiveAccount();
return useQuery({
queryKey: ["llmwiki", "agents", agentId, "knowledge-bases", userId ?? "anonymous"],
queryKey: [
"llmwiki",
"agents",
agentId,
"knowledge-bases",
userId ?? "anonymous",
],
queryFn: () => listAgentLlmWikiKnowledgeBases(agentId!),
enabled: enabled && !!agentId,
});
@@ -885,7 +1057,12 @@ export function useLlmWikiKnowledgeBase(
) {
const { userId } = useLiveAccount();
return useQuery({
queryKey: ["llmwiki", "knowledge-base", knowledgeBaseId, userId ?? "anonymous"],
queryKey: [
"llmwiki",
"knowledge-base",
knowledgeBaseId,
userId ?? "anonymous",
],
queryFn: () => getLlmWikiKnowledgeBase(knowledgeBaseId!),
enabled: enabled && !!knowledgeBaseId,
refetchInterval: 8_000,
@@ -912,7 +1089,13 @@ export function useLlmWikiDocument(
) {
const { userId } = useLiveAccount();
return useQuery({
queryKey: ["llmwiki", "document", knowledgeBaseId, documentId, userId ?? "anonymous"],
queryKey: [
"llmwiki",
"document",
knowledgeBaseId,
documentId,
userId ?? "anonymous",
],
queryFn: () => getLlmWikiDocument(knowledgeBaseId!, documentId!),
enabled: enabled && !!knowledgeBaseId && !!documentId,
});
@@ -927,13 +1110,25 @@ export function useLlmWikiDocumentChunks(
) {
const { userId } = useLiveAccount();
return useQuery({
queryKey: ["llmwiki", "document-chunks", knowledgeBaseId, documentId, page, pageSize, userId ?? "anonymous"],
queryFn: () => listLlmWikiDocumentChunks(knowledgeBaseId!, documentId!, page, pageSize),
queryKey: [
"llmwiki",
"document-chunks",
knowledgeBaseId,
documentId,
page,
pageSize,
userId ?? "anonymous",
],
queryFn: () =>
listLlmWikiDocumentChunks(knowledgeBaseId!, documentId!, page, pageSize),
enabled: enabled && !!knowledgeBaseId && !!documentId,
});
}
export function useLlmWikiGraph(knowledgeBaseId: string | null, enabled = true) {
export function useLlmWikiGraph(
knowledgeBaseId: string | null,
enabled = true,
) {
const { userId } = useLiveAccount();
return useQuery({
queryKey: ["llmwiki", "graph", knowledgeBaseId, userId ?? "anonymous"],
@@ -951,8 +1146,17 @@ export function useLlmWikiWikiPages(
) {
const { userId } = useLiveAccount();
return useQuery({
queryKey: ["llmwiki", "wiki-pages", knowledgeBaseId, page, pageSize, query, userId ?? "anonymous"],
queryFn: () => listLlmWikiWikiPages(knowledgeBaseId!, page, pageSize, query),
queryKey: [
"llmwiki",
"wiki-pages",
knowledgeBaseId,
page,
pageSize,
query,
userId ?? "anonymous",
],
queryFn: () =>
listLlmWikiWikiPages(knowledgeBaseId!, page, pageSize, query),
enabled: enabled && !!knowledgeBaseId,
refetchInterval: 10_000,
});
@@ -965,7 +1169,13 @@ export function useLlmWikiWikiPage(
) {
const { userId } = useLiveAccount();
return useQuery({
queryKey: ["llmwiki", "wiki-page", knowledgeBaseId, slug, userId ?? "anonymous"],
queryKey: [
"llmwiki",
"wiki-page",
knowledgeBaseId,
slug,
userId ?? "anonymous",
],
queryFn: () => getLlmWikiWikiPage(knowledgeBaseId!, slug!),
enabled: enabled && !!knowledgeBaseId && !!slug,
});
@@ -991,8 +1201,16 @@ export function usePublicLlmWikiWikiPages(
enabled = true,
) {
return useQuery({
queryKey: ["public-llmwiki", "wiki-pages", knowledgeBaseId, page, pageSize, query],
queryFn: () => listPublicLlmWikiWikiPages(knowledgeBaseId!, page, pageSize, query),
queryKey: [
"public-llmwiki",
"wiki-pages",
knowledgeBaseId,
page,
pageSize,
query,
],
queryFn: () =>
listPublicLlmWikiWikiPages(knowledgeBaseId!, page, pageSize, query),
enabled: enabled && !!knowledgeBaseId,
staleTime: 60_000,
});
@@ -1029,7 +1247,12 @@ export function useLlmWikiWikiIndexStatus(
) {
const { userId } = useLiveAccount();
return useQuery({
queryKey: ["llmwiki", "wiki-index-status", knowledgeBaseId, userId ?? "anonymous"],
queryKey: [
"llmwiki",
"wiki-index-status",
knowledgeBaseId,
userId ?? "anonymous",
],
queryFn: () => getLlmWikiWikiIndexStatus(knowledgeBaseId!),
enabled: enabled && !!knowledgeBaseId,
refetchInterval: (query) => {
@@ -1058,8 +1281,14 @@ export function useLlmWikiActions() {
onSuccess: invalidateKnowledge,
}),
update: useMutation({
mutationFn: ({ id, ...input }: { id: string; name?: string; description?: string }) =>
updateLlmWikiKnowledgeBase(id, input),
mutationFn: ({
id,
...input
}: {
id: string;
name?: string;
description?: string;
}) => updateLlmWikiKnowledgeBase(id, input),
onSuccess: invalidateKnowledge,
}),
remove: useMutation({
@@ -1077,8 +1306,15 @@ export function useLlmWikiActions() {
onSuccess: invalidateKnowledge,
}),
createManual: useMutation({
mutationFn: ({ id, title, content }: { id: string; title: string; content: string }) =>
createManualLlmWikiDocument(id, { title, content }),
mutationFn: ({
id,
title,
content,
}: {
id: string;
title: string;
content: string;
}) => createManualLlmWikiDocument(id, { title, content }),
onSuccess: invalidateKnowledge,
}),
removeDocument: useMutation({
@@ -1092,13 +1328,25 @@ export function useLlmWikiActions() {
onSuccess: invalidateKnowledge,
}),
createWikiPage: useMutation({
mutationFn: ({ id, input }: { id: string; input: LlmWikiWikiPageCreate }) =>
createLlmWikiWikiPage(id, input),
mutationFn: ({
id,
input,
}: {
id: string;
input: LlmWikiWikiPageCreate;
}) => createLlmWikiWikiPage(id, input),
onSuccess: invalidateKnowledge,
}),
updateWikiPage: useMutation({
mutationFn: ({ id, slug, input }: { id: string; slug: string; input: LlmWikiWikiPageUpdate }) =>
updateLlmWikiWikiPage(id, slug, input),
mutationFn: ({
id,
slug,
input,
}: {
id: string;
slug: string;
input: LlmWikiWikiPageUpdate;
}) => updateLlmWikiWikiPage(id, slug, input),
onSuccess: invalidateKnowledge,
}),
removeWikiPage: useMutation({
@@ -1128,8 +1376,14 @@ export function useLlmWikiSettingsActions() {
await queryClient.invalidateQueries({ queryKey: ["llmwiki"] });
};
return {
save: useMutation({ mutationFn: saveLlmWikiSettings, onSuccess: invalidate }),
reset: useMutation({ mutationFn: resetLlmWikiSettings, onSuccess: invalidate }),
save: useMutation({
mutationFn: saveLlmWikiSettings,
onSuccess: invalidate,
}),
reset: useMutation({
mutationFn: resetLlmWikiSettings,
onSuccess: invalidate,
}),
test: useMutation({ mutationFn: testLlmWikiConnection }),
};
}
@@ -0,0 +1,280 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { apiFetch } from "@/core/api/fetch-client";
import { getBackendBaseURL } from "@/core/config";
const BASE = () => `${getBackendBaseURL()}/api/skill-knowledge`;
export type SkillKnowledgeTargetType = "weknora" | "assistant";
export type SkillKnowledgeTargetMode =
| "wiki"
| "document"
| "faq"
| "conversation_archive"
| "readonly_sidecar"
| "assistant";
export interface SkillKnowledgeTarget {
target_type: SkillKnowledgeTargetType;
target_mode: SkillKnowledgeTargetMode;
target_id: string;
target_name: string;
remote_business_key?: string | null;
readonly?: boolean;
}
export interface SkillKnowledgeBinding {
id: string;
skill_name: string;
target_type: SkillKnowledgeTargetType;
target_mode: SkillKnowledgeTargetMode;
target_id: string;
target_name: string;
remote_business_key?: string | null;
remote_refs?: Array<{
provider: string;
remote_kind: string;
remote_id: string;
slug?: string;
}>;
status: string;
active: boolean;
auto_sync_enabled: boolean;
synced_digest?: string | null;
last_synced_at?: string | null;
last_error_message?: string | null;
}
export interface SkillKnowledgeSyncItem {
id: string;
job_id: string;
binding_id: string;
skill_name: string;
target_type: SkillKnowledgeTargetType;
target_mode: SkillKnowledgeTargetMode;
target_id: string;
status: string;
phase: string;
progress: number;
attempts: number;
last_error_message?: string | null;
}
export interface SkillKnowledgeJob {
id: string;
status: string;
total_count: number;
success_count: number;
partial_failed_count: number;
failed_count: number;
review_mode: "required" | "auto_high_confidence" | "off";
created_at: string;
completed_at?: string | null;
items?: SkillKnowledgeSyncItem[];
}
export interface SkillKnowledgeReview {
id: string;
job_id: string;
item_id: string;
binding_id: string;
object_type: "entity" | "relation" | "wiki" | "alignment";
object_key: string;
original: Record<string, unknown>;
edited?: Record<string, unknown> | null;
status: string;
reason?: string | null;
}
export interface SkillKnowledgeAutoSyncSettings {
watch_enabled: boolean;
schedule_enabled: boolean;
schedule_cron: string;
debounce_seconds: number;
write_remote_enabled: boolean;
review_mode: "required" | "auto_high_confidence" | "off";
}
async function readJson<T>(response: Response): Promise<T> {
if (!response.ok) {
const data = await response.json().catch(() => null);
const detail =
data && typeof data === "object" && "detail" in data
? JSON.stringify((data as { detail: unknown }).detail)
: `请求失败 (${response.status})`;
throw new Error(detail.replace(/^"|"$/g, ""));
}
return (await response.json()) as T;
}
export async function listSkillKnowledgeBindings(
skillNames?: string[],
mineOnly = false,
): Promise<SkillKnowledgeBinding[]> {
const params = new URLSearchParams();
for (const name of skillNames ?? []) params.append("skill_names", name);
if (mineOnly) params.set("mine_only", "true");
const response = await apiFetch(`${BASE()}/bindings?${params.toString()}`);
return (await readJson<{ bindings: SkillKnowledgeBinding[] }>(response)).bindings;
}
export async function createSkillKnowledgeJob(input: {
skill_names: string[];
targets: SkillKnowledgeTarget[];
force?: boolean;
review_mode?: "required" | "auto_high_confidence" | "off";
}): Promise<SkillKnowledgeJob> {
return readJson(
await apiFetch(`${BASE()}/sync-jobs`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
}
export async function listSkillKnowledgeJobs(mineOnly = false): Promise<SkillKnowledgeJob[]> {
const params = mineOnly ? "?mine_only=true" : "";
const response = await apiFetch(`${BASE()}/sync-jobs${params}`);
return (await readJson<{ jobs: SkillKnowledgeJob[] }>(response)).jobs;
}
export async function getSkillKnowledgeJob(id: string): Promise<SkillKnowledgeJob> {
return readJson(await apiFetch(`${BASE()}/sync-jobs/${encodeURIComponent(id)}`));
}
export async function listSkillKnowledgeReviews(): Promise<SkillKnowledgeReview[]> {
const response = await apiFetch(`${BASE()}/reviews?status=pending`);
return (await readJson<{ reviews: SkillKnowledgeReview[] }>(response)).reviews;
}
export async function reviewSkillKnowledge(input: {
review_ids: string[];
action: "approve" | "reject";
edits?: Record<string, Record<string, unknown>>;
reason?: string;
}) {
return readJson<{ updated: number; resumed_item_ids: string[] }>(
await apiFetch(`${BASE()}/reviews/actions`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
}
export async function retrySkillKnowledgeItem(itemId: string) {
return readJson<SkillKnowledgeSyncItem>(
await apiFetch(`${BASE()}/sync-items/${encodeURIComponent(itemId)}/retry`, {
method: "POST",
}),
);
}
export async function detachSkillKnowledgeBinding(
bindingId: string,
deleteContributions: boolean,
) {
return readJson<SkillKnowledgeBinding>(
await apiFetch(`${BASE()}/bindings/${encodeURIComponent(bindingId)}/detach`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ delete_contributions: deleteContributions }),
}),
);
}
export async function resyncSkillKnowledgeBinding(bindingId: string) {
return readJson<SkillKnowledgeJob>(
await apiFetch(`${BASE()}/bindings/${encodeURIComponent(bindingId)}/resync`, {
method: "POST",
}),
);
}
export async function setSkillKnowledgeBindingAutoSync(
bindingId: string,
enabled: boolean,
) {
return readJson<SkillKnowledgeBinding>(
await apiFetch(
`${BASE()}/bindings/${encodeURIComponent(bindingId)}/${enabled ? "enable" : "disable"}-auto-sync`,
{ method: "POST" },
),
);
}
export async function getSkillKnowledgeAutoSyncSettings() {
return readJson<SkillKnowledgeAutoSyncSettings>(
await apiFetch(`${BASE()}/auto-sync-settings`),
);
}
export async function updateSkillKnowledgeAutoSyncSettings(
input: Partial<SkillKnowledgeAutoSyncSettings>,
) {
return readJson<SkillKnowledgeAutoSyncSettings>(
await apiFetch(`${BASE()}/auto-sync-settings`, {
method: "PUT",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(input),
}),
);
}
const KEY = "skill-knowledge";
export function useSkillKnowledgeBindings(skillNames?: string[], mineOnly = false) {
return useQuery({
queryKey: [KEY, "bindings", mineOnly ? "mine" : "all", ...(skillNames ?? [])],
queryFn: () => listSkillKnowledgeBindings(skillNames, mineOnly),
refetchOnWindowFocus: false,
});
}
export function useSkillKnowledgeJobs(mineOnly = false) {
return useQuery({
queryKey: [KEY, "jobs", mineOnly ? "mine" : "all"],
queryFn: () => listSkillKnowledgeJobs(mineOnly),
refetchInterval: (query) =>
(query.state.data ?? []).some((job) => ["queued", "running"].includes(job.status))
? 1500
: false,
});
}
export function useSkillKnowledgeReviews() {
return useQuery({ queryKey: [KEY, "reviews"], queryFn: listSkillKnowledgeReviews });
}
export function useSkillKnowledgeAutoSyncSettings() {
return useQuery({
queryKey: [KEY, "auto-sync-settings"],
queryFn: getSkillKnowledgeAutoSyncSettings,
});
}
export function useSkillKnowledgeActions() {
const client = useQueryClient();
const invalidate = () => client.invalidateQueries({ queryKey: [KEY] });
return {
createJob: useMutation({ mutationFn: createSkillKnowledgeJob, onSuccess: invalidate }),
review: useMutation({ mutationFn: reviewSkillKnowledge, onSuccess: invalidate }),
retry: useMutation({ mutationFn: retrySkillKnowledgeItem, onSuccess: invalidate }),
resync: useMutation({ mutationFn: resyncSkillKnowledgeBinding, onSuccess: invalidate }),
setAutoSync: useMutation({
mutationFn: ({ id, enabled }: { id: string; enabled: boolean }) =>
setSkillKnowledgeBindingAutoSync(id, enabled),
onSuccess: invalidate,
}),
updateAutoSyncSettings: useMutation({
mutationFn: updateSkillKnowledgeAutoSyncSettings,
onSuccess: invalidate,
}),
detach: useMutation({
mutationFn: ({ id, deleteContributions }: { id: string; deleteContributions: boolean }) =>
detachSkillKnowledgeBinding(id, deleteContributions),
onSuccess: invalidate,
}),
};
}
@@ -1,6 +1,19 @@
import { AlertCircle, Edit3, FileText, Loader2, RefreshCw, Trash2 } from "lucide-react";
import {
AlertCircle,
Archive,
Edit3,
FileText,
Loader2,
RefreshCw,
Trash2,
} from "lucide-react";
import { useEffect, useRef, useState } from "react";
import { Navigate, useNavigate, useParams, useSearchParams } from "react-router-dom";
import {
Navigate,
useNavigate,
useParams,
useSearchParams,
} from "react-router-dom";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
@@ -14,6 +27,7 @@ import { Input } from "@/components/ui/input";
import { Textarea } from "@/components/ui/textarea";
import { MarkdownCodeEditor } from "@/components/wiki/MarkdownCodeEditor";
import { getBackendBaseURL } from "@/core/config";
import { getStoredAuth } from "@/core/auth";
import { buildWeKnoraWikiPagePath } from "@/core/llmwiki/source-links";
import {
createLlmWikiWeKnoraEmbedSession,
@@ -34,7 +48,10 @@ type DeerFlowThemePayload = {
function cssVar(name: string, fallback = ""): string {
if (typeof window === "undefined") return fallback;
return getComputedStyle(document.documentElement).getPropertyValue(name).trim() || fallback;
return (
getComputedStyle(document.documentElement).getPropertyValue(name).trim() ||
fallback
);
}
function collectThemePayload(): DeerFlowThemePayload {
@@ -46,15 +63,27 @@ function collectThemePayload(): DeerFlowThemePayload {
theme: isDark ? "dark" : "light",
vars: {
primary: cssVar("--primary", isDark ? "#2b81ff" : "#0062d9"),
primaryForeground: cssVar("--button-foreground", cssVar("--primary-foreground", "#ffffff")),
primaryForeground: cssVar(
"--button-foreground",
cssVar("--primary-foreground", "#ffffff"),
),
background: cssVar("--background", isDark ? "#0e172b" : "#ffffff"),
foreground: cssVar("--foreground", isDark ? "#eff2f5" : "#202a3a"),
card: cssVar("--card", isDark ? "#1a2539" : "#ffffff"),
cardForeground: cssVar("--card-foreground", cssVar("--foreground", isDark ? "#eff2f5" : "#202a3a")),
cardForeground: cssVar(
"--card-foreground",
cssVar("--foreground", isDark ? "#eff2f5" : "#202a3a"),
),
muted: cssVar("--muted", isDark ? "#232e40" : "#f3f5f8"),
mutedForeground: cssVar("--muted-foreground", isDark ? "#8e9aa4" : "#7f7f7f"),
mutedForeground: cssVar(
"--muted-foreground",
isDark ? "#8e9aa4" : "#7f7f7f",
),
border: cssVar("--border", isDark ? "rgba(255,255,255,0.12)" : "#dbdbdb"),
popover: cssVar("--popover", cssVar("--card", isDark ? "#1a2539" : "#ffffff")),
popover: cssVar(
"--popover",
cssVar("--card", isDark ? "#1a2539" : "#ffffff"),
),
destructive: cssVar("--destructive", "#d54941"),
},
};
@@ -68,7 +97,8 @@ function postThemeToFrame(frame: HTMLIFrameElement | null) {
try {
const currentUrl = frame.contentWindow.location.href;
if (!currentUrl || currentUrl === "about:blank") return;
if (new URL(currentUrl, window.location.href).origin !== targetOrigin) return;
if (new URL(currentUrl, window.location.href).origin !== targetOrigin)
return;
} catch {
// Reading location throws once the iframe has reached the cross-origin backend.
}
@@ -89,7 +119,9 @@ function buildFrameUrl(framePath: string) {
: framePath;
return new URL(path, backendOrigin);
}
return new URL(`${backendBase || backendOrigin}/${framePath.replace(/^\/+/, "")}`);
return new URL(
`${backendBase || backendOrigin}/${framePath.replace(/^\/+/, "")}`,
);
}
export function WeKnoraKnowledgeDetailPage() {
@@ -97,36 +129,57 @@ export function WeKnoraKnowledgeDetailPage() {
knowledge_base_id: strategyKnowledgeBaseId,
noteId: workspaceKnowledgeBaseId,
} = useParams();
const knowledgeBaseId = strategyKnowledgeBaseId ?? workspaceKnowledgeBaseId ?? "";
const knowledgeBaseId =
strategyKnowledgeBaseId ?? workspaceKnowledgeBaseId ?? "";
const [searchParams] = useSearchParams();
const navigate = useNavigate();
const frameRef = useRef<HTMLIFrameElement>(null);
const [frameSrc, setFrameSrc] = useState("");
const [frameVersion, setFrameVersion] = useState(0);
const [frameError, setFrameError] = useState<string | null>(null);
const isAdmin = getStoredAuth()?.system_role === "admin";
const runtime = useLlmWikiRuntime();
const details = useLlmWikiKnowledgeBase(knowledgeBaseId || null, !!knowledgeBaseId);
const details = useLlmWikiKnowledgeBase(
knowledgeBaseId || null,
!!knowledgeBaseId,
);
const detailsReady = !!details.data;
const requestedTab = searchParams.get("tab");
const detailsWikiEnabled =
details.data?.type === "wiki" ||
details.data?.wiki_index_enabled === true ||
details.data?.indexing_strategy?.wiki_enabled === true ||
details.data?.capabilities?.wiki_enabled === true ||
(details.data?.wiki_config &&
Object.keys(details.data.wiki_config).length > 0);
const preferredTab =
requestedTab === "documents" || requestedTab === "wiki" || requestedTab === "graph"
requestedTab === "documents" ||
requestedTab === "wiki" ||
requestedTab === "graph"
? requestedTab
: details.data?.indexing_strategy?.wiki_enabled
: detailsWikiEnabled
? "wiki"
: "documents";
const sourceChunkId = searchParams.get("sourceChunkId") || searchParams.get("chunkId") || "";
const sourceChunkId =
searchParams.get("sourceChunkId") || searchParams.get("chunkId") || "";
const documentId = searchParams.get("documentId") || "";
const wikiSlug = searchParams.get("wikiSlug") || "";
const nativeWikiPageUrl = knowledgeBaseId && wikiSlug && preferredTab === "wiki"
? buildWeKnoraWikiPagePath(knowledgeBaseId, wikiSlug)
: "";
const nativeWikiPageUrl =
knowledgeBaseId && wikiSlug && preferredTab === "wiki"
? buildWeKnoraWikiPagePath(knowledgeBaseId, wikiSlug)
: "";
useEffect(() => {
let cancelled = false;
setFrameSrc("");
setFrameError(null);
if (nativeWikiPageUrl || !knowledgeBaseId || runtime.data?.provider !== "weknora" || !detailsReady) {
if (
nativeWikiPageUrl ||
!knowledgeBaseId ||
runtime.data?.provider !== "weknora" ||
!detailsReady
) {
return () => {
cancelled = true;
};
@@ -136,30 +189,47 @@ export function WeKnoraKnowledgeDetailPage() {
const resolvedWikiSlug =
wikiSlug ||
(sourceChunkId
? (await getLlmWikiSourceContext(knowledgeBaseId, sourceChunkId).catch(
() => null,
))?.wiki_page?.slug
? (
await getLlmWikiSourceContext(
knowledgeBaseId,
sourceChunkId,
).catch(() => null)
)?.wiki_page?.slug
: "");
const session = await createLlmWikiWeKnoraEmbedSession(knowledgeBaseId, preferredTab);
if (cancelled) return;
const framePath = session.frame_path || "";
const nextUrl = buildFrameUrl(framePath);
nextUrl.searchParams.set("theme", collectThemePayload().theme);
if (sourceChunkId) nextUrl.searchParams.set("sourceChunkId", sourceChunkId);
if (documentId) nextUrl.searchParams.set("documentId", documentId);
if (resolvedWikiSlug) nextUrl.searchParams.set("wikiSlug", resolvedWikiSlug);
setFrameSrc(nextUrl.toString());
})()
.catch((error: unknown) => {
if (cancelled) return;
const message = error instanceof Error ? error.message : "创建知识库详情会话失败";
setFrameError(message);
});
const session = await createLlmWikiWeKnoraEmbedSession(
knowledgeBaseId,
preferredTab,
);
if (cancelled) return;
const framePath = session.frame_path || "";
const nextUrl = buildFrameUrl(framePath);
nextUrl.searchParams.set("theme", collectThemePayload().theme);
if (sourceChunkId)
nextUrl.searchParams.set("sourceChunkId", sourceChunkId);
if (documentId) nextUrl.searchParams.set("documentId", documentId);
if (resolvedWikiSlug)
nextUrl.searchParams.set("wikiSlug", resolvedWikiSlug);
setFrameSrc(nextUrl.toString());
})().catch((error: unknown) => {
if (cancelled) return;
const message =
error instanceof Error ? error.message : "创建知识库详情会话失败";
setFrameError(message);
});
return () => {
cancelled = true;
};
}, [detailsReady, documentId, knowledgeBaseId, nativeWikiPageUrl, preferredTab, runtime.data?.provider, sourceChunkId, wikiSlug]);
}, [
detailsReady,
documentId,
knowledgeBaseId,
nativeWikiPageUrl,
preferredTab,
runtime.data?.provider,
sourceChunkId,
wikiSlug,
]);
useEffect(() => {
if (!frameSrc) return;
@@ -192,7 +262,8 @@ export function WeKnoraKnowledgeDetailPage() {
);
}
if (runtime.data?.provider !== "weknora") return <Navigate to={LIST_PATH} replace />;
if (runtime.data?.provider !== "weknora")
return <Navigate to={LIST_PATH} replace />;
if (frameError) {
return (
@@ -201,7 +272,9 @@ export function WeKnoraKnowledgeDetailPage() {
<AlertCircle className="size-6" />
</div>
<div>
<div className="text-base font-semibold text-foreground">无法打开知识库详情</div>
<div className="text-base font-semibold text-foreground">
无法打开知识库详情
</div>
<p className="mt-2 text-sm text-muted-foreground">{frameError}</p>
</div>
<Button variant="outline" onClick={() => navigate(LIST_PATH)}>
@@ -218,7 +291,9 @@ export function WeKnoraKnowledgeDetailPage() {
<AlertCircle className="size-6" />
</div>
<div>
<div className="text-base font-semibold text-foreground">无法打开知识库详情</div>
<div className="text-base font-semibold text-foreground">
无法打开知识库详情
</div>
<p className="mt-2 text-sm text-muted-foreground">
当前账号没有访问权限,或知识服务资源已不存在。
</p>
@@ -230,10 +305,11 @@ export function WeKnoraKnowledgeDetailPage() {
);
}
const canWrite = details.data.can_write && !details.data.is_conversation_deposit;
const canWrite =
details.data.can_write && !details.data.is_conversation_deposit;
const canEditCurrentWikiArticle =
canWrite && !!details.data.indexing_strategy?.wiki_enabled && !!wikiSlug;
const canManageWiki = canWrite && !!details.data.indexing_strategy?.wiki_enabled;
canWrite && !!detailsWikiEnabled && !!wikiSlug;
const canManageWiki = canWrite && !!detailsWikiEnabled;
return (
<div className="flex h-full min-h-0 flex-col bg-background">
@@ -246,19 +322,40 @@ export function WeKnoraKnowledgeDetailPage() {
slug={wikiSlug}
onChanged={() => setFrameVersion((value) => value + 1)}
onDeleted={() => {
navigate(`/page/workspace/knowledge/${encodeURIComponent(knowledgeBaseId)}?tab=wiki`, {
replace: true,
});
navigate(
`/page/workspace/knowledge/${encodeURIComponent(knowledgeBaseId)}?tab=wiki`,
{
replace: true,
},
);
setFrameVersion((value) => value + 1);
}}
/>
)}
</div>
<div className="flex shrink-0 items-center gap-2">
{isAdmin && (
<Button
size="sm"
variant="outline"
onClick={() =>
navigate(
`/page/workspace/knowledge/${encodeURIComponent(knowledgeBaseId)}/packages`,
)
}
>
<Archive className="mr-2 size-4" />
数据包任务
</Button>
)}
<Button
size="sm"
variant="outline"
onClick={() => navigate(`/page/workspace/knowledge/${encodeURIComponent(knowledgeBaseId)}/wiki`)}
onClick={() =>
navigate(
`/page/workspace/knowledge/${encodeURIComponent(knowledgeBaseId)}/wiki`,
)
}
>
<FileText className="mr-2 size-4" />
Wiki管理
@@ -342,7 +439,12 @@ function CurrentWikiArticleActions({
const remove = async () => {
if (!slug) return;
if (!window.confirm(`确定删除文章「${detail.data?.title || slug}」吗?删除后不可恢复。`)) return;
if (
!window.confirm(
`确定删除文章「${detail.data?.title || slug}」吗?删除后不可恢复。`,
)
)
return;
try {
await actions.removeWikiPage.mutateAsync({ id: knowledgeBaseId, slug });
toast.success("文章已删除");
@@ -394,14 +496,18 @@ function CurrentWikiArticleActions({
<label className="text-sm font-medium">标题</label>
<Input
value={draft.title}
onChange={(event) => setDraft((prev) => ({ ...prev, title: event.target.value }))}
onChange={(event) =>
setDraft((prev) => ({ ...prev, title: event.target.value }))
}
/>
</div>
<div className="space-y-1.5">
<label className="text-sm font-medium">摘要</label>
<Textarea
value={draft.summary}
onChange={(event) => setDraft((prev) => ({ ...prev, summary: event.target.value }))}
onChange={(event) =>
setDraft((prev) => ({ ...prev, summary: event.target.value }))
}
rows={3}
/>
</div>
@@ -409,18 +515,28 @@ function CurrentWikiArticleActions({
<label className="text-sm font-medium">正文</label>
<MarkdownCodeEditor
value={draft.content}
onChange={(content) => setDraft((prev) => ({ ...prev, content }))}
onChange={(content) =>
setDraft((prev) => ({ ...prev, content }))
}
className="[&_.cm-editor]:min-h-[48vh]"
/>
</div>
</div>
<div className="mt-4 flex shrink-0 justify-end gap-2 border-t pt-3">
<Button variant="outline" onClick={() => setOpen(false)} disabled={actions.updateWikiPage.isPending}>
<Button
variant="outline"
onClick={() => setOpen(false)}
disabled={actions.updateWikiPage.isPending}
>
取消
</Button>
<Button
onClick={save}
disabled={actions.updateWikiPage.isPending || !draft.title.trim() || !draft.content.trim()}
disabled={
actions.updateWikiPage.isPending ||
!draft.title.trim() ||
!draft.content.trim()
}
>
{actions.updateWikiPage.isPending ? (
<Loader2 className="mr-2 size-4 animate-spin" />
@@ -1,8 +1,11 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import {
BookOpen,
BookOpenText,
ChevronRight,
Database,
Cpu,
Edit3,
FileText,
Globe2,
Loader2,
@@ -10,7 +13,9 @@ import {
RefreshCw,
Search,
Settings2,
Sparkles,
Trash2,
Upload,
} from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import { useNavigate } from "react-router-dom";
@@ -28,6 +33,7 @@ import {
DialogTitle,
} from "@/components/ui/dialog";
import { Input } from "@/components/ui/input";
import { Progress } from "@/components/ui/progress";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Textarea } from "@/components/ui/textarea";
import { getStoredAuth } from "@/core/auth";
@@ -37,7 +43,11 @@ import {
type LlmWikiSearchResponse,
type LlmWikiSearchResult,
type LlmWikiScope,
configureLocalWikiEncoder,
getLocalWikiEncoderStatus,
searchLlmWiki,
startLocalWikiEncoder,
uploadLocalWikiEncoder,
useLlmWikiActions,
useLlmWikiKnowledgeBases,
useLlmWikiSettings,
@@ -83,7 +93,6 @@ function formatSearchTime(value?: string | null) {
function resultPreview(result: LlmWikiSearchResult) {
const text = (
result.matched_sections?.map((section) => section.content).filter(Boolean).join("\n\n") ||
result.summary ||
result.content ||
""
);
@@ -109,6 +118,7 @@ export function WeKnoraKnowledgeWorkspace() {
const knowledge = useLlmWikiKnowledgeBases(scope, tab !== "settings");
const actions = useLlmWikiActions();
const [createOpen, setCreateOpen] = useState(false);
const [editingKnowledge, setEditingKnowledge] = useState<LlmWikiKnowledgeBase | null>(null);
const [searchText, setSearchText] = useState("");
const [searching, setSearching] = useState(false);
const [searchResponse, setSearchResponse] = useState<LlmWikiSearchResponse | null>(null);
@@ -119,12 +129,8 @@ export function WeKnoraKnowledgeWorkspace() {
setSelectedSearchResult(null);
}, [tab]);
const busy = actions.create.isPending || actions.remove.isPending || actions.publication.isPending;
const busy = actions.create.isPending || actions.update.isPending || actions.remove.isPending || actions.publication.isPending;
const openDetails = (item: LlmWikiKnowledgeBase) => {
if (item.remote_status === "missing") {
toast.error("该知识库已从知识服务删除,只能清理当前记录");
return;
}
navigate(`${LIST_PATH}/${encodeURIComponent(item.id)}`);
};
@@ -142,10 +148,7 @@ export function WeKnoraKnowledgeWorkspace() {
};
const removeKnowledgeBase = async (item: LlmWikiKnowledgeBase) => {
const warning = item.remote_status === "missing"
? `知识库“${item.name}”已不在知识服务中。确定清理 cmzs 中的这条失效记录吗?`
: `确定删除知识库“${item.name}”吗?知识服务中的文档和索引也会删除。`;
if (!window.confirm(warning)) return;
if (!window.confirm(`确定删除知识库“${item.name}”吗?知识服务中的文档和索引也会删除。`)) return;
try {
await actions.remove.mutateAsync(item.id);
toast.success("知识库已删除");
@@ -177,6 +180,11 @@ export function WeKnoraKnowledgeWorkspace() {
<p className="mt-1 text-sm text-muted-foreground">由知识服务负责解析与检索,用户、权限和发布范围由 cmzs 管理。</p>
</div>
<div className="flex items-center gap-2 rounded-full border bg-background/80 p-1 shadow-sm backdrop-blur">
{tab !== "settings" && (
<Button variant="ghost" size="sm" className="rounded-full" onClick={() => navigate("/page/workspace/skills?tab=knowledge")}>
<Sparkles className="mr-2 size-4" />从技能归纳
</Button>
)}
{tab !== "settings" && (
<Button variant="ghost" size="sm" className="rounded-full" onClick={() => knowledge.refetch()} disabled={knowledge.isFetching}>
<RefreshCw className={`mr-2 size-4 ${knowledge.isFetching ? "animate-spin" : ""}`} />刷新
@@ -207,8 +215,8 @@ export function WeKnoraKnowledgeWorkspace() {
)}
</div>
<TabsContent value="personal" className="mt-5"><KnowledgeGrid rows={knowledge.data ?? []} loading={knowledge.isLoading} error={knowledge.error} emptyText="还没有个人知识库,点击右上角创建。" busy={busy} onOpen={openDetails} onPublish={togglePublication} onDelete={removeKnowledgeBase} /></TabsContent>
<TabsContent value="public" className="mt-5"><KnowledgeGrid rows={knowledge.data ?? []} loading={knowledge.isLoading} error={knowledge.error} emptyText="公共 LLMWiki 暂无已发布知识库。" busy={busy} onOpen={openDetails} onPublish={togglePublication} onDelete={removeKnowledgeBase} /></TabsContent>
<TabsContent value="personal" className="mt-5"><KnowledgeGrid rows={knowledge.data ?? []} loading={knowledge.isLoading} error={knowledge.error} emptyText="还没有个人知识库,点击右上角创建。" busy={busy} onOpen={openDetails} onEdit={setEditingKnowledge} onPublish={togglePublication} onDelete={removeKnowledgeBase} /></TabsContent>
<TabsContent value="public" className="mt-5"><KnowledgeGrid rows={knowledge.data ?? []} loading={knowledge.isLoading} error={knowledge.error} emptyText="公共 LLMWiki 暂无已发布知识库。" busy={busy} onOpen={openDetails} onEdit={setEditingKnowledge} onPublish={togglePublication} onDelete={removeKnowledgeBase} /></TabsContent>
{isAdmin && <TabsContent value="settings" className="mt-5"><ConnectionSettings /></TabsContent>}
</Tabs>
@@ -301,6 +309,24 @@ export function WeKnoraKnowledgeWorkspace() {
}}
/>
{editingKnowledge && (
<EditKnowledgeDialog
key={editingKnowledge.id}
knowledgeBase={editingKnowledge}
pending={actions.update.isPending}
onClose={() => setEditingKnowledge(null)}
onSave={async (input) => {
try {
await actions.update.mutateAsync({ id: editingKnowledge.id, ...input });
setEditingKnowledge(null);
toast.success("知识库信息已保存");
} catch (error) {
toast.error(messageOf(error));
}
}}
/>
)}
<CreateKnowledgeDialog open={createOpen} pending={actions.create.isPending} onOpenChange={setCreateOpen} onCreate={async (input) => {
try {
const created = await actions.create.mutateAsync(input);
@@ -322,6 +348,7 @@ function KnowledgeGrid({
emptyText,
busy,
onOpen,
onEdit,
onPublish,
onDelete,
}: {
@@ -331,29 +358,27 @@ function KnowledgeGrid({
emptyText: string;
busy: boolean;
onOpen: (item: LlmWikiKnowledgeBase) => void;
onEdit: (item: LlmWikiKnowledgeBase) => void;
onPublish: (item: LlmWikiKnowledgeBase) => void;
onDelete: (item: LlmWikiKnowledgeBase) => void;
}) {
if (loading) return <div className="flex h-64 items-center justify-center text-muted-foreground"><Loader2 className="mr-2 size-5 animate-spin" />正在读取知识库</div>;
if (error) return <div className="rounded-lg border border-destructive/30 bg-destructive/5 p-6 text-sm text-destructive">{messageOf(error)}</div>;
if (!rows.length) return <div className="flex h-64 items-center justify-center rounded-xl border border-dashed text-sm text-muted-foreground">{emptyText}</div>;
const visibleRows = rows.filter((item) => item.remote_status !== "missing");
if (!visibleRows.length) return <div className="flex h-64 items-center justify-center rounded-xl border border-dashed text-sm text-muted-foreground">{emptyText}</div>;
return (
<div className="grid gap-4 md:grid-cols-2 xl:grid-cols-3">
{rows.map((item) => (
{visibleRows.map((item) => (
<Card
key={item.id}
className={`group overflow-hidden rounded-2xl transition-all hover:-translate-y-0.5 hover:shadow-lg ${
item.remote_status === "missing"
? "border-amber-500/40 bg-amber-500/[0.03]"
: "border-border/70 bg-card/95 hover:border-primary/30"
}`}
className="group overflow-hidden rounded-2xl border-border/70 bg-card/95 transition-all hover:-translate-y-0.5 hover:border-primary/30 hover:shadow-lg"
>
<div className={`h-1 ${item.publication_status === "published" ? "bg-primary" : "bg-muted"}`} />
<button type="button" disabled={item.remote_status === "missing"} className="w-full text-left outline-none transition-colors enabled:hover:bg-muted/25 focus-visible:ring-2 focus-visible:ring-primary focus-visible:ring-offset-2 disabled:cursor-default" onClick={() => onOpen(item)}>
<button type="button" className="w-full text-left outline-none transition-colors hover:bg-muted/25 focus-visible:ring-2 focus-visible:ring-primary focus-visible:ring-offset-2" onClick={() => onOpen(item)}>
<CardHeader className="pb-3 pt-5">
<div className="flex items-start justify-between gap-3">
<div className="min-w-0"><CardTitle className="truncate text-base">{item.name}</CardTitle><CardDescription className="mt-1 line-clamp-2 min-h-10">{item.description || "暂无描述"}</CardDescription></div>
<div className="flex shrink-0 flex-col items-end gap-1.5">{item.remote_status === "missing" ? <Badge variant="outline" className="border-amber-500/50 text-amber-700 dark:text-amber-300">远端已删除</Badge> : <Badge variant={item.publication_status === "published" ? "default" : "secondary"}>{publicationLabels[item.publication_status] ?? "仅自己"}</Badge>}{item.remote_status !== "missing" && item.indexing_strategy?.wiki_enabled && <Badge className="bg-violet-100 text-violet-700 hover:bg-violet-100 dark:bg-violet-950 dark:text-violet-300">Wiki</Badge>}</div>
<div className="flex shrink-0 flex-col items-end gap-1.5"><Badge variant={item.publication_status === "published" ? "default" : "secondary"}>{publicationLabels[item.publication_status] ?? "仅自己"}</Badge>{item.indexing_strategy?.wiki_enabled && <Badge className="bg-violet-100 text-violet-700 hover:bg-violet-100 dark:bg-violet-950 dark:text-violet-300">Wiki</Badge>}</div>
</div>
</CardHeader>
<div className="px-6 pb-4"><div className="grid grid-cols-3 gap-2 rounded-xl border bg-muted/35 p-3 text-center text-xs"><div><div className="text-base font-semibold text-foreground">{item.knowledge_count}</div><span className="text-muted-foreground">文档</span></div><div><div className="text-base font-semibold text-foreground">{item.chunk_count}</div><span className="text-muted-foreground">分块</span></div><div><div className="text-base font-semibold text-foreground">{item.processing_count}</div><span className="text-muted-foreground">处理中</span></div></div></div>
@@ -361,10 +386,11 @@ function KnowledgeGrid({
<CardContent className="border-t bg-muted/20 px-6 py-4">
<div className="flex items-center justify-between gap-3">
<div className="flex min-w-0 flex-wrap items-center gap-2">
{item.remote_status === "missing" ? <span className="text-xs text-muted-foreground">知识服务资源不存在,可清理本地记录</span> : <Button size="sm" variant="outline" className="rounded-full" onClick={() => onOpen(item)}><FileText className="mr-2 size-4" />查看详情</Button>}
{item.remote_status !== "missing" && item.is_owner && !item.is_conversation_deposit && <Button size="sm" variant="secondary" className="rounded-full" disabled={busy} onClick={() => void onPublish(item)}>{item.publication_status === "published" ? "取消公开" : "发布公共库"}</Button>}
<Button size="sm" variant="outline" className="rounded-full" onClick={() => onOpen(item)}><FileText className="mr-2 size-4" />查看详情</Button>
{item.can_write && !item.is_conversation_deposit && <Button size="sm" variant="outline" className="rounded-full" disabled={busy} onClick={() => onEdit(item)} aria-label={`编辑知识库 ${item.name}`}><Edit3 className="mr-2 size-4" />编辑</Button>}
{item.is_owner && !item.is_conversation_deposit && <Button size="sm" variant="secondary" className="rounded-full" disabled={busy} onClick={() => void onPublish(item)}>{item.publication_status === "published" ? "取消公开" : "发布公共库"}</Button>}
</div>
{item.can_write && !item.is_conversation_deposit && <Button size="icon" variant="ghost" className="size-8 shrink-0 rounded-full text-destructive hover:bg-destructive/10 hover:text-destructive" aria-label={item.remote_status === "missing" ? `清理失效记录 ${item.name}` : `删除知识库 ${item.name}`} title={item.remote_status === "missing" ? "清理失效记录" : "删除知识库"} disabled={busy} onClick={() => void onDelete(item)}><Trash2 className="size-4" /></Button>}
{item.can_write && !item.is_conversation_deposit && <Button size="icon" variant="ghost" className="size-8 shrink-0 rounded-full text-destructive hover:bg-destructive/10 hover:text-destructive" aria-label={`删除知识库 ${item.name}`} title="删除知识库" disabled={busy} onClick={() => void onDelete(item)}><Trash2 className="size-4" /></Button>}
</div>
</CardContent>
</Card>
@@ -373,6 +399,55 @@ function KnowledgeGrid({
);
}
function EditKnowledgeDialog({
knowledgeBase,
pending,
onClose,
onSave,
}: {
knowledgeBase: LlmWikiKnowledgeBase;
pending: boolean;
onClose: () => void;
onSave: (input: { name: string; description: string }) => Promise<void>;
}) {
const [name, setName] = useState(knowledgeBase.name);
const [description, setDescription] = useState(knowledgeBase.description);
return (
<Dialog open onOpenChange={(open) => { if (!open && !pending) onClose(); }}>
<DialogContent>
<form onSubmit={(event) => {
event.preventDefault();
if (!pending && name.trim()) {
void onSave({ name: name.trim(), description: description.trim() });
}
}}>
<DialogHeader>
<DialogTitle>编辑知识库</DialogTitle>
<DialogDescription>修改知识库名称和描述,保存后同步到知识服务。</DialogDescription>
</DialogHeader>
<div className="space-y-4 py-4">
<div className="space-y-2">
<label className="text-sm font-medium" htmlFor="llmwiki-edit-name">名称</label>
<Input id="llmwiki-edit-name" value={name} onChange={(event) => setName(event.target.value)} maxLength={255} required disabled={pending} />
</div>
<div className="space-y-2">
<label className="text-sm font-medium" htmlFor="llmwiki-edit-description">描述</label>
<Textarea id="llmwiki-edit-description" value={description} onChange={(event) => setDescription(event.target.value)} rows={4} maxLength={2048} disabled={pending} />
</div>
</div>
<DialogFooter>
<Button type="button" variant="outline" onClick={onClose} disabled={pending}>取消</Button>
<Button type="submit" disabled={pending || !name.trim()}>
{pending && <Loader2 className="mr-2 size-4 animate-spin" />}保存
</Button>
</DialogFooter>
</form>
</DialogContent>
</Dialog>
);
}
function CreateKnowledgeDialog({
open,
pending,
@@ -408,6 +483,122 @@ function CreateKnowledgeDialog({
);
}
export function LocalEncoderSettingsCard() {
const queryClient = useQueryClient();
const encoder = useQuery({
queryKey: ["llmwiki-local-encoder"],
queryFn: getLocalWikiEncoderStatus,
retry: false,
refetchInterval: (query) => query.state.data?.state === "starting" ? 1000 : false,
});
const [modelPath, setModelPath] = useState("");
const [files, setFiles] = useState<File[]>([]);
const [uploadProgress, setUploadProgress] = useState(0);
useEffect(() => {
if (encoder.data?.model_path) setModelPath(encoder.data.model_path);
}, [encoder.data?.model_path]);
const applyStatus = (status: Awaited<ReturnType<typeof getLocalWikiEncoderStatus>>) => {
queryClient.setQueryData(["llmwiki-local-encoder"], status);
setModelPath(status.model_path || "");
};
const configure = useMutation({
mutationFn: () => configureLocalWikiEncoder(modelPath.trim()),
onSuccess: (status) => { applyStatus(status); toast.success("编码器目录已保存"); },
onError: (error) => toast.error(messageOf(error)),
});
const upload = useMutation({
mutationFn: () => uploadLocalWikiEncoder(files, setUploadProgress),
onMutate: () => setUploadProgress(0),
onSuccess: (status) => {
applyStatus(status);
setFiles([]);
toast.success(`已保存 ${status.uploaded_files ?? 0} 个模型文件`);
},
onError: (error) => toast.error(messageOf(error)),
});
const start = useMutation({
mutationFn: startLocalWikiEncoder,
onSuccess: (status) => {
applyStatus(status);
toast.success(status.state === "ready" ? "本地编码器已就绪" : "本地编码器正在独立启动");
},
onError: (error) => toast.error(messageOf(error)),
});
const status = encoder.data;
const selectedBytes = files.reduce((sum, file) => sum + file.size, 0);
const formatSize = (value: number) => value >= 1024 ** 3
? `${(value / 1024 ** 3).toFixed(2)} GB`
: `${(value / 1024 ** 2).toFixed(1)} MB`;
return (
<Card>
<CardHeader>
<CardTitle className="flex items-center gap-2 text-base"><Cpu className="size-5" />DeerFlow 本地编码器</CardTitle>
<CardDescription>仅管理员可见。配置服务器上已有的 BGE-M3 目录,或直接上传完整离线模型目录;编码器作为独立进程启动,不影响 DeerFlow 其他服务。</CardDescription>
</CardHeader>
<CardContent className="space-y-5">
<div className="flex flex-wrap items-center gap-2 text-sm">
<Badge variant={status?.state === "failed" ? "destructive" : status?.state === "ready" ? "default" : "secondary"}>
{status?.state === "ready" ? "已就绪" : status?.state === "starting" ? "启动中" : status?.state === "failed" ? "启动失败" : "未启动"}
</Badge>
<span className="text-muted-foreground">{status ? `${status.model} · ${status.dimensions || "自动探测"} 维 · ${status.process_isolation ? "独立进程" : "主进程"}` : "正在读取状态"}</span>
{status?.model_files_ready && <Badge variant="outline">模型文件已就绪</Badge>}
</div>
{encoder.error && <p className="text-sm text-destructive">{messageOf(encoder.error)}</p>}
{status?.last_error && <p className="rounded-lg border border-destructive/30 bg-destructive/5 p-3 text-xs text-destructive">{status.last_error}</p>}
<div className="space-y-2">
<label className="text-sm font-medium" htmlFor="local-encoder-path">使用服务器已有目录</label>
<div className="flex flex-col gap-2 sm:flex-row">
<Input id="local-encoder-path" value={modelPath} onChange={(event) => setModelPath(event.target.value)} placeholder="例如 /opt/deerflow/models/bge-m3" />
<Button variant="outline" disabled={!modelPath.trim() || configure.isPending} onClick={() => configure.mutate()}>
{configure.isPending && <Loader2 className="mr-2 size-4 animate-spin" />}保存目录
</Button>
</div>
<p className="text-xs text-muted-foreground">目录内需包含 <code>onnx/model.onnx</code>(推荐)或根目录 <code>model.onnx</code>。</p>
</div>
<div className="space-y-2 rounded-xl border border-dashed p-4">
<div className="flex items-center gap-2 text-sm font-medium"><Upload className="size-4" />从当前电脑上传模型目录</div>
<Input
type="file"
multiple
{...{ webkitdirectory: "" }}
onChange={(event) => setFiles(Array.from(event.currentTarget.files ?? []))}
/>
{upload.isPending && (
<div className="space-y-1">
<Progress value={uploadProgress} />
<p className="text-right text-xs text-muted-foreground">上传进度 {uploadProgress}%</p>
</div>
)}
<div className="flex flex-wrap items-center justify-between gap-3">
<p className="text-xs text-muted-foreground">
{files.length ? `已选择 ${files.length} 个文件,共 ${formatSize(selectedBytes)}` : "请选择完整 BGE-M3 目录。大文件会落到运行目录,不写入源码仓库。"}
</p>
<Button disabled={!files.length || upload.isPending} onClick={() => upload.mutate()}>
{upload.isPending ? <Loader2 className="mr-2 size-4 animate-spin" /> : <Upload className="mr-2 size-4" />}
{upload.isPending ? "正在上传,请勿关闭页面" : "上传并设为当前模型"}
</Button>
</div>
</div>
<div className="flex flex-wrap gap-2">
<Button disabled={!status?.model_files_ready || start.isPending || status?.state === "starting" || status?.state === "ready"} onClick={() => start.mutate()}>
{(start.isPending || status?.state === "starting") && <Loader2 className="mr-2 size-4 animate-spin" />}
{status?.state === "ready" ? "编码器已启动" : status?.state === "failed" ? "重新启动" : "一键启动编码器"}
</Button>
<Button variant="ghost" disabled={encoder.isFetching} onClick={() => void encoder.refetch()}>
<RefreshCw className={`mr-2 size-4 ${encoder.isFetching ? "animate-spin" : ""}`} />刷新状态
</Button>
</div>
</CardContent>
</Card>
);
}
export function ConnectionSettings() {
const settings = useLlmWikiSettings(true);
const actions = useLlmWikiSettingsActions();
@@ -0,0 +1,221 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import {
Archive,
ArrowLeft,
Download,
HardDrive,
Loader2,
PackagePlus,
RefreshCw,
} from "lucide-react";
import { Navigate, useNavigate, useParams } from "react-router-dom";
import { toast } from "sonner";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
import { KnowledgeProductNav } from "@/components/workspace/knowledge-product-nav";
import { getStoredAuth } from "@/core/auth";
import {
createLlmWikiExportPackage,
listLlmWikiExportPackages,
} from "@/strategy-components/api/assistant-knowledge";
import { useLlmWikiKnowledgeBase, useLlmWikiRuntime } from "@/strategy-components/api/llmwiki";
import { WikiPackageTransfers } from "./WikiPackageTransfers";
const RUNNING = new Set(["queued", "running"]);
const STATUS_LABEL: Record<string, string> = {
queued: "排队中",
running: "整理中",
completed: "已完成",
failed: "失败",
};
function formatBytes(value?: number | null) {
let current = Number(value || 0);
if (!current) return "—";
const units = ["B", "KB", "MB", "GB", "TB"];
let unit = 0;
while (current >= 1024 && unit < units.length - 1) {
current /= 1024;
unit += 1;
}
return `${current >= 10 || unit === 0 ? current.toFixed(0) : current.toFixed(1)} ${units[unit]}`;
}
export default function WeKnoraPackageTransfersPage() {
const { noteId = "" } = useParams();
const navigate = useNavigate();
const queryClient = useQueryClient();
const isAdmin = getStoredAuth()?.system_role === "admin";
const runtime = useLlmWikiRuntime();
const knowledge = useLlmWikiKnowledgeBase(noteId || null, !!noteId);
const jobsKey = ["llmwiki-export-packages", noteId];
const jobs = useQuery({
queryKey: jobsKey,
queryFn: () => listLlmWikiExportPackages(noteId),
enabled: isAdmin && !!noteId && runtime.data?.provider === "weknora",
refetchInterval: (query) =>
query.state.data?.some((job) => RUNNING.has(job.status)) ? 3000 : false,
});
const createExport = useMutation({
mutationFn: () => createLlmWikiExportPackage(noteId),
onSuccess: () => {
void queryClient.invalidateQueries({ queryKey: jobsKey });
toast.success("已创建后台整理任务,完成后可在本页下载");
},
onError: (error) => toast.error(error.message),
});
if (!isAdmin) return <Navigate to="/page/workspace/knowledge" replace />;
return (
<div className="flex size-full min-h-0 flex-col bg-muted/20">
<KnowledgeProductNav />
<main className="min-h-0 flex-1 overflow-auto px-5 py-6 lg:px-8">
<div className="mx-auto max-w-6xl space-y-6">
<header className="flex flex-wrap items-center justify-between gap-4">
<div className="flex min-w-0 items-center gap-3">
<Button
variant="ghost"
size="icon"
onClick={() =>
navigate(
`/page/workspace/knowledge/${encodeURIComponent(noteId)}`,
)
}
>
<ArrowLeft className="size-5" />
</Button>
<div className="flex size-11 shrink-0 items-center justify-center rounded-xl bg-primary/10 text-primary">
<Archive className="size-5" />
</div>
<div className="min-w-0">
<h1 className="truncate text-xl font-semibold">
知识库数据包任务
</h1>
<p className="truncate text-sm text-muted-foreground">
{knowledge.data?.name || "正在读取知识库"} ·
大文件在后台整理,不占用详情页面
</p>
</div>
</div>
<Button
onClick={() => createExport.mutate()}
disabled={createExport.isPending || !knowledge.data}
>
{createExport.isPending ? (
<Loader2 className="mr-2 size-4 animate-spin" />
) : (
<PackagePlus className="mr-2 size-4" />
)}
新建导出任务
</Button>
</header>
<Card>
<CardHeader className="flex-row items-center justify-between space-y-0 border-b">
<div>
<CardTitle className="text-base">
向量化数据包导出记录
</CardTitle>
<p className="mt-1 text-sm text-muted-foreground">
每次生成独立记录;即使文件达到数
GB,也只在后台落盘完成后提供下载。
</p>
</div>
<Button
variant="ghost"
size="sm"
onClick={() => void jobs.refetch()}
disabled={jobs.isFetching}
>
<RefreshCw
className={`mr-2 size-4 ${jobs.isFetching ? "animate-spin" : ""}`}
/>
刷新
</Button>
</CardHeader>
<CardContent className="p-0">
{jobs.data?.map((job) => {
const counts = job.counts || {};
const ready =
job.status === "completed" &&
job.file_exists &&
job.package_download_url;
return (
<div
key={job.id}
className="grid gap-4 border-b p-5 last:border-b-0 md:grid-cols-[auto_1fr_auto] md:items-center"
>
<div
className={`flex size-10 items-center justify-center rounded-full ${job.status === "completed" ? "bg-emerald-100 text-emerald-700" : job.status === "failed" ? "bg-destructive/10 text-destructive" : "bg-amber-100 text-amber-700"}`}
>
{RUNNING.has(job.status) ? (
<Loader2 className="size-4 animate-spin" />
) : (
<HardDrive className="size-4" />
)}
</div>
<div className="min-w-0">
<div className="flex flex-wrap items-center gap-2">
<span className="font-medium">
任务 {job.id.slice(0, 8)}
</span>
<Badge variant="outline">
{STATUS_LABEL[job.status] || job.status}
</Badge>
<span className="text-xs text-muted-foreground">
{job.created_at
? new Date(job.created_at).toLocaleString()
: "—"}
</span>
</div>
<div className="mt-2 flex flex-wrap gap-x-5 gap-y-1 text-sm text-muted-foreground">
<span>
Wiki {counts.wiki_page ?? counts.wiki_pages ?? 0}
</span>
<span>向量 {counts.vector ?? counts.vectors ?? 0}</span>
<span>文件 {formatBytes(job.file_size_bytes)}</span>
<span>阶段 {job.phase}</span>
</div>
{job.error && (
<p className="mt-2 text-sm text-destructive">
{job.error}
</p>
)}
</div>
{ready ? (
<Button asChild variant="outline">
<a href={job.package_download_url}>
<Download className="mr-2 size-4" />
下载数据包
</a>
</Button>
) : (
<Button variant="outline" disabled>
{job.status === "failed" ? "整理失败" : "等待完成"}
</Button>
)}
</div>
);
})}
{!jobs.isLoading && !jobs.data?.length && (
<div className="px-6 py-14 text-center text-sm text-muted-foreground">
暂无导出记录。点击右上角“新建导出任务”开始整理。
</div>
)}
{jobs.isLoading && (
<div className="flex items-center justify-center px-6 py-14 text-sm text-muted-foreground">
<Loader2 className="mr-2 size-4 animate-spin" />
加载导出记录
</div>
)}
</CardContent>
</Card>
<WikiPackageTransfers id={noteId} />
</div>
</main>
</div>
);
}
@@ -52,8 +52,11 @@ import {
usePublicLlmWikiWikiPages,
} from "@/strategy-components/api/llmwiki";
import { WikiProtectedImage } from "./WikiProtectedImage";
import { WikiLibraryReader } from "./WikiLibraryReader";
import { useAssistantKnowledgeBases, useAssistantKnowledgeSources } from "@/strategy-components/api/assistant-knowledge";
const LIST_PATH = "/page/workspace/knowledge";
const GENERATED_PUBLIC_INDEX_SLUG = "__index__";
function cleanText(value?: string | null) {
return (value || "").replace(/\r\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim();
@@ -120,6 +123,10 @@ export function WeKnoraWikiPageView({
const navigate = useNavigate();
const publicBackground = usePublicBackground();
const requestedWikiSlug = searchParams.get("wikiSlug")?.trim() || "";
const assistantBases = useAssistantKnowledgeBases(!publicEmbed);
const assistantSources = useAssistantKnowledgeSources(null, !publicEmbed);
const globalBase = assistantBases.data?.find(base => base.is_global);
const isDeposited = assistantSources.data?.some(source => source.source_key === noteId && source.status === "active");
const privateKnowledgeBase = useLlmWikiKnowledgeBase(
noteId || null,
!!noteId && !publicEmbed,
@@ -144,13 +151,40 @@ export function WeKnoraWikiPageView({
!!noteId && publicEmbed,
);
const pages = publicEmbed ? publicPages : privatePages;
const pagesList = useMemo(() => pages.data?.pages || [], [pages.data?.pages]);
const indexPage = useMemo(
() => (publicEmbed ? pagesList.find(isWikiIndexPage) : undefined),
[pagesList, publicEmbed],
const remotePagesList = useMemo(() => pages.data?.pages || [], [pages.data?.pages]);
const explicitIndexPage = useMemo(
() => (publicEmbed ? remotePagesList.find(isWikiIndexPage) : undefined),
[publicEmbed, remotePagesList],
);
const generatedIndexPage = useMemo<LlmWikiWikiPage | undefined>(
() => publicEmbed && remotePagesList.length > 0 && !explicitIndexPage
? {
id: "__generated_public_index__",
slug: GENERATED_PUBLIC_INDEX_SLUG,
title: "索引",
page_type: "index",
status: "generated",
content: "# 索引\n\n知识库页面目录",
summary: "",
aliases: [],
source_refs: [],
in_links: [],
out_links: [],
version: 0,
}
: undefined,
[explicitIndexPage, publicEmbed, remotePagesList.length],
);
const indexPage = explicitIndexPage || generatedIndexPage;
const pagesList = useMemo(
() => generatedIndexPage ? [generatedIndexPage, ...remotePagesList] : remotePagesList,
[generatedIndexPage, remotePagesList],
);
const landingPage = useMemo(() => selectDefaultWikiPage(pagesList), [pagesList]);
const wikiSlug = requestedWikiSlug || (publicEmbed ? landingPage?.slug || "" : "");
const generatedIndexActive = generatedIndexPage?.slug === wikiSlug;
const generatedIndexRequested =
publicEmbed && wikiSlug === GENERATED_PUBLIC_INDEX_SLUG && !explicitIndexPage;
const privatePage = useLlmWikiWikiPage(
noteId || null,
wikiSlug || null,
@@ -159,7 +193,11 @@ export function WeKnoraWikiPageView({
const publicPage = usePublicLlmWikiWikiPage(
noteId || null,
wikiSlug || null,
!!noteId && !!wikiSlug && publicEmbed,
!!noteId &&
!!wikiSlug &&
publicEmbed &&
!generatedIndexActive &&
!generatedIndexRequested,
);
const page = publicEmbed ? publicPage : privatePage;
const publicWikiIndex = usePublicLlmWikiWikiIndex(
@@ -170,7 +208,9 @@ export function WeKnoraWikiPageView({
const [directoryQuery, setDirectoryQuery] = useState("");
const [collapsedDirectories, setCollapsedDirectories] = useState<Set<string>>(() => new Set());
const routedPage = (location.state as { wikiPage?: LlmWikiWikiPage } | null)?.wikiPage;
const pageData = page.data || (routedPage?.slug === wikiSlug ? routedPage : undefined);
const pageData = generatedIndexActive
? generatedIndexPage
: page.data || (routedPage?.slug === wikiSlug ? routedPage : undefined);
const filteredPages = useMemo(
() => filterWikiPagesWithAncestors(pagesList, directoryQuery),
[directoryQuery, pagesList],
@@ -345,6 +385,24 @@ export function WeKnoraWikiPageView({
</div>
);
}
if (publicEmbed && pages.error) {
return (
<div className="flex h-full flex-col items-center justify-center gap-4 px-6 text-center">
<div className="flex size-12 items-center justify-center rounded-full bg-destructive/10 text-destructive">
<BookOpenText className="size-6" />
</div>
<div>
<div className="font-medium text-foreground">无法读取 Wiki 目录</div>
<div className="mt-1 text-sm text-muted-foreground">
{pages.error instanceof Error ? pages.error.message : "公开 Wiki 服务暂时不可用"}
</div>
</div>
<Button onClick={() => void pages.refetch()}>
<RefreshCw className="mr-2 size-4" />重试
</Button>
</div>
);
}
return (
<div className="flex h-full flex-col items-center justify-center gap-4 text-sm text-muted-foreground">
{publicEmbed ? "该知识库暂无可展示的 Wiki 页面" : "当前链接缺少 Wiki 页面标识"}
@@ -434,6 +492,7 @@ export function WeKnoraWikiPageView({
</div>
</div>
<div className="flex items-center gap-2">
{globalBase && isDeposited && <Button variant="outline" size="sm" onClick={() => navigate(`/page/workspace/knowledge/assistant?${new URLSearchParams({ base_id: globalBase.id, wikiSlug: pageData.slug })}`)}>在知识梳理中查看</Button>}
<Button variant="outline" size="sm" onClick={() => void download("md")} disabled={downloading !== null}>
{downloading === "md" ? <Loader2 className="size-4 animate-spin" /> : <Download className="size-4" />}
下载 MD
@@ -446,7 +505,7 @@ export function WeKnoraWikiPageView({
</header>
)}
<div className="flex min-h-0 flex-1 overflow-hidden">
{!publicEmbed ? <WikiLibraryReader pages={pagesList} activePage={pageData} mappingId={noteId} onNavigate={slug => navigate(buildWeKnoraWikiPagePath(noteId, slug))} pageHref={slug => buildAppHashUrl(buildWeKnoraWikiPagePath(noteId, slug))} /> : <div className="flex min-h-0 flex-1 overflow-hidden">
<aside className="flex w-64 shrink-0 flex-col border-r bg-background/90 lg:w-72">
<div className="shrink-0 border-b px-3 py-3">
<div className="flex items-center justify-between gap-3 px-1">
@@ -531,7 +590,7 @@ export function WeKnoraWikiPageView({
</section>
</article>
</main>
</div>
</div>}
</div>
);
}
@@ -0,0 +1,471 @@
import {
BookOpenText,
ChevronDown,
ChevronRight,
FileText,
Folder,
Link2,
Network,
Search,
X,
} from "lucide-react";
import { useEffect, useMemo, useRef, useState } from "react";
import { Badge } from "@/components/ui/badge";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { MarkdownContent } from "@/components/workspace/messages/markdown-content";
import {
buildWikiTree,
filterWikiPagesWithAncestors,
type WikiTreeNode,
} from "@/core/llmwiki/wiki-tree";
import {
renderableWikiMarkdown,
rewriteProtectedWikiImageSources,
} from "@/core/llmwiki/wiki-page-markdown";
import { reasoningPlugins } from "@/core/streamdown";
import type { LlmWikiWikiPage } from "@/strategy-components/api/llmwiki";
import { WikiProtectedImage } from "./WikiProtectedImage";
export type LibraryWikiPage = LlmWikiWikiPage & {
tags?: string[];
origins?: { mapping_id?: string; source_name: string; slug: string }[];
};
const TYPES: Record<string, string> = {
index: "索引",
summary: "摘要",
entity: "实体",
concept: "概念",
synthesis: "综合分析",
comparison: "对比分析",
article: "文章",
source: "资料",
relation: "关系",
};
export function WikiLibraryReader({
pages,
activePage,
onNavigate,
pageHref,
mappingId,
initiallyCollapseTree = false,
}: {
pages: LibraryWikiPage[];
activePage: LibraryWikiPage | null;
onNavigate: (slug: string) => void;
pageHref: (slug: string) => string;
mappingId?: string;
initiallyCollapseTree?: boolean;
}) {
const [query, setQuery] = useState("");
const [kind, setKind] = useState("");
const [tag, setTag] = useState("");
const [collapsed, setCollapsed] = useState<Set<string>>(new Set());
const [graph, setGraph] = useState(false);
const [directoryOpen, setDirectoryOpen] = useState(false);
const initialTreeCollapseApplied = useRef(false);
const navigate = (slug: string) => {
onNavigate(slug);
setDirectoryOpen(false);
};
const groups = useMemo(
() => [...new Set(pages.map((p) => p.page_type))],
[pages],
);
const tags = useMemo(
() =>
[
...new Set(
pages.flatMap((p) => [
...(p.tags || p.page_metadata?.tags || []),
...(p.aliases || []),
]),
),
].sort(),
[pages],
);
const tree = useMemo(
() =>
buildWikiTree(
filterWikiPagesWithAncestors(
pages.filter(
(p) =>
(!kind || p.page_type === kind) &&
(!tag ||
[
...(p.tags || p.page_metadata?.tags || []),
...(p.aliases || []),
].includes(tag)),
),
query,
),
),
[pages, kind, tag, query],
);
const bySlug = useMemo(() => new Map(pages.map((p) => [p.slug, p])), [pages]);
useEffect(() => {
const ancestors: string[] = [];
const branches: string[] = [];
const collectBranches = (nodes: WikiTreeNode[]) => {
for (const node of nodes) {
if (node.children.length) branches.push(node.key);
collectBranches(node.children);
}
};
const visit = (nodes: WikiTreeNode[], parents: string[]): boolean =>
nodes.some((node) => {
if (node.page?.slug === activePage?.slug) {
ancestors.push(...parents);
return true;
}
return visit(node.children, [...parents, node.key]);
});
collectBranches(tree.roots);
visit(tree.roots, []);
if (
initiallyCollapseTree &&
!initialTreeCollapseApplied.current &&
tree.roots.length
) {
initialTreeCollapseApplied.current = true;
setCollapsed(
new Set(branches.filter((key) => !ancestors.includes(key))),
);
return;
}
setCollapsed(
(current) =>
new Set([...current].filter((key) => !ancestors.includes(key))),
);
}, [activePage?.slug, initiallyCollapseTree, tree]);
const links = useMemo(() => {
const outgoing = new Set(activePage?.out_links || []);
for (const match of (activePage?.content || "").matchAll(
/\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g,
))
outgoing.add(match[1].trim());
const incoming = new Set(activePage?.in_links || []);
for (const p of pages)
if (
p.slug !== activePage?.slug &&
((p.out_links || []).includes(activePage?.slug || "") ||
p.content?.includes(`[[${activePage?.slug}`))
)
incoming.add(p.slug);
return {
outgoing: [...outgoing].filter((s) => bySlug.has(s)),
incoming: [...incoming].filter((s) => bySlug.has(s)),
};
}, [activePage, pages, bySlug]);
const renderNode = (node: WikiTreeNode, depth = 0): React.ReactNode => {
const active = activePage?.slug === node.page?.slug;
const open = !!query || !collapsed.has(node.key);
const toggle = () =>
setCollapsed((current) => {
const next = new Set(current);
if (next.has(node.key)) next.delete(node.key);
else next.add(node.key);
return next;
});
return (
<div key={node.key}>
<div
style={{ paddingLeft: 8 + depth * 14 }}
className={`flex items-center gap-1 rounded-lg pr-2 text-sm ${active ? "bg-emerald-100 text-emerald-800 dark:bg-emerald-950 dark:text-emerald-200" : "text-muted-foreground hover:bg-muted"}`}
>
{node.children.length ? (
<button
aria-label={open ? "收起分类" : "展开分类"}
onClick={toggle}
className="p-1"
>
{open ? (
<ChevronDown className="size-4" />
) : (
<ChevronRight className="size-4" />
)}
</button>
) : (
<span className="w-6" />
)}
<button
onClick={() => (node.page ? navigate(node.page.slug) : toggle())}
className="flex min-w-0 flex-1 items-center gap-2 py-2.5 text-left"
aria-current={active ? "page" : undefined}
>
{node.page ? (
<FileText className="size-4 shrink-0" />
) : (
<Folder className="size-4 shrink-0 text-amber-500" />
)}
<span className="truncate">{TYPES[node.label] || node.label}</span>
</button>
</div>
{open && node.children.map((child) => renderNode(child, depth + 1))}
</div>
);
};
const renderLinks = (values: string[]) =>
values.map((slug) => (
<button
key={slug}
onClick={() => navigate(slug)}
className="block w-full rounded-lg px-2 py-2 text-left text-sm text-primary hover:bg-muted"
>
<Link2 className="mr-2 inline size-3.5" />
{bySlug.get(slug)?.title || slug}
</button>
));
return (
<div className="relative flex min-h-0 flex-1 overflow-hidden bg-muted/20">
{directoryOpen && (
<button
aria-label="关闭 Wiki 目录"
className="absolute inset-0 z-20 bg-black/30 md:hidden"
onClick={() => setDirectoryOpen(false)}
/>
)}
<aside
className={`${directoryOpen ? "flex" : "hidden"} absolute inset-y-0 left-0 z-30 w-64 shrink-0 flex-col border-r bg-background md:static md:z-auto md:flex lg:w-72`}
>
<div className="space-y-3 border-b p-4">
<div className="flex items-center gap-2 font-semibold">
<BookOpenText className="size-4 text-emerald-600" />
Wiki 目录
<span className="ml-auto text-xs font-normal text-muted-foreground">
{pages.length} 篇
</span>
<button
className="md:hidden"
aria-label="收起 Wiki 目录"
onClick={() => setDirectoryOpen(false)}
>
<X className="size-4" />
</button>
</div>
<div className="relative">
<Search className="absolute left-2 top-3 size-4 text-muted-foreground" />
<Input
value={query}
onChange={(e) => setQuery(e.target.value)}
placeholder="搜索标题、分类或标签"
className="pl-8"
/>
</div>
<select
aria-label="Wiki 分类"
className="h-9 w-full rounded-md border bg-background px-2 text-sm"
value={kind}
onChange={(e) => setKind(e.target.value)}
>
<option value="">全部类型</option>
{groups.map((k) => (
<option key={k} value={k}>
{TYPES[k] || k}({pages.filter((p) => p.page_type === k).length}
)
</option>
))}
</select>
{tags.length > 0 && (
<select
aria-label="Wiki 标签"
className="h-9 w-full rounded-md border bg-background px-2 text-sm"
value={tag}
onChange={(e) => setTag(e.target.value)}
>
<option value="">全部标签</option>
{tags.map((t) => (
<option key={t}>{t}</option>
))}
</select>
)}
</div>
<nav
className="min-h-0 flex-1 overflow-auto p-2"
aria-label="Wiki 目录"
>
{tree.roots.map((node) => renderNode(node))}
{!tree.roots.length && (
<p className="p-4 text-sm text-muted-foreground">暂无匹配页面</p>
)}
</nav>
</aside>
<main className="min-w-0 flex-1 overflow-auto p-3 md:p-5 lg:p-8">
<Button
variant="outline"
size="sm"
className="mb-3 md:hidden"
onClick={() => setDirectoryOpen(true)}
>
<BookOpenText className="mr-2 size-4" />
目录与分类({pages.length})
</Button>
{activePage ? (
<article className="mx-auto max-w-5xl rounded-2xl border bg-background shadow-sm">
<header className="space-y-3 border-b p-4 md:px-7 md:py-6">
<div className="text-xs text-muted-foreground">
{(activePage.category_path || []).join(" / ") || "知识库 Wiki"}
</div>
<div className="flex items-start justify-between gap-4">
<h1 className="text-2xl font-semibold tracking-tight">
{activePage.title}
</h1>
<Badge variant="outline">
{TYPES[activePage.page_type] || activePage.page_type}
</Badge>
</div>
<div className="flex flex-wrap gap-2">
{[
...new Set([
...(activePage.tags ||
activePage.page_metadata?.tags ||
[]),
...(activePage.aliases || []),
]),
].map((t) => (
<button
key={t}
onClick={() => {
setTag(t);
setDirectoryOpen(true);
}}
>
<Badge variant="secondary">{t}</Badge>
</button>
))}
</div>
{activePage.summary && (
<p className="text-sm leading-7 text-muted-foreground">
{activePage.summary}
</p>
)}
<div className="flex flex-wrap gap-3 text-xs text-muted-foreground">
{activePage.updated_at && (
<span>
更新于 {new Date(activePage.updated_at).toLocaleString()}
</span>
)}
<span>
{links.outgoing.length} 个引用 · {links.incoming.length}{" "}
个反向链接
</span>
</div>
</header>
<section className="px-4 py-6 md:px-7 md:py-8 lg:px-10">
<MarkdownContent
content={rewriteProtectedWikiImageSources(
renderableWikiMarkdown(
activePage.title,
activePage.content,
mappingId || "assistant",
(_id, slug) => pageHref(slug),
pages,
).replace(/^# [^\n]*(?:\n|$)/, ""),
)}
isLoading={false}
remarkPlugins={reasoningPlugins.remarkPlugins}
rehypePlugins={reasoningPlugins.rehypePlugins}
components={
mappingId
? {
img: (props) => (
<WikiProtectedImage
{...props}
knowledgeBaseId={mappingId}
/>
),
}
: undefined
}
linkClassName={(href) =>
pages.some((p) => pageHref(p.slug) === href)
? "wiki-content-link !text-primary border-b border-dashed"
: undefined
}
onLinkClick={(event, href) => {
const target = pages.find(
(p) =>
pageHref(p.slug) === href ||
href === p.slug ||
href === `/${p.slug}`,
);
if (target) {
event.preventDefault();
navigate(target.slug);
}
}}
/>
</section>
<section className="border-t px-4 py-5 md:px-7">
<Button
variant="ghost"
size="sm"
onClick={() => setGraph(!graph)}
>
<Network className="mr-2 size-4" />
{graph ? "收起关系图谱" : "查看关系图谱"}
</Button>
{graph && (
<div className="my-4 grid grid-cols-3 items-center gap-3 rounded-xl bg-muted/40 p-4">
<div>
{renderLinks(links.incoming)}
{!links.incoming.length && (
<span className="text-xs text-muted-foreground">
暂无反向链接
</span>
)}
</div>
<div className="rounded-xl border border-primary/30 bg-primary/10 p-4 text-center text-sm font-medium">
→ {activePage.title} →
</div>
<div>
{renderLinks(links.outgoing)}
{!links.outgoing.length && (
<span className="text-xs text-muted-foreground">
暂无引用
</span>
)}
</div>
</div>
)}
<div className="grid gap-5 sm:grid-cols-2">
<div>
<h3 className="my-2 text-sm font-medium">引用页面</h3>
{renderLinks(links.outgoing)}
</div>
<div>
<h3 className="my-2 text-sm font-medium">被哪些页面引用</h3>
{renderLinks(links.incoming)}
</div>
</div>
{(activePage.origins || [])
.filter(
(o, index, all) =>
o.mapping_id &&
all.findIndex(
(value) =>
value.mapping_id === o.mapping_id &&
value.slug === o.slug,
) === index,
)
.map((o) => (
<a
key={`${o.mapping_id}:${o.slug}`}
className="mt-3 block text-xs text-primary hover:underline"
href={`#/page/workspace/knowledge/${encodeURIComponent(o.mapping_id!)}/wiki/view?wikiSlug=${encodeURIComponent(o.slug)}`}
>
查看普通知识库来源 · {o.source_name}
</a>
))}
</section>
</article>
) : (
<div className="py-20 text-center text-sm text-muted-foreground">
选择一个 Wiki 页面开始阅读。完成向量化的数据会自动沉淀到知识梳理。
</div>
)}
</main>
</div>
);
}
@@ -0,0 +1,126 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { Download, Loader2, Upload } from "lucide-react";
import { useRef } from "react";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
import { Badge } from "@/components/ui/badge";
import {
createAssistantExportPackage,
importOrdinaryWikiPackage,
listWikiTransfers,
} from "@/strategy-components/api/assistant-knowledge";
export function WikiPackageTransfers({
id,
assistant = false,
}: {
id: string;
assistant?: boolean;
}) {
const input = useRef<HTMLInputElement>(null);
const client = useQueryClient();
const key = ["wiki-transfers", assistant, id];
const jobs = useQuery({
queryKey: key,
queryFn: () => listWikiTransfers(id, assistant),
refetchInterval: (q) =>
q.state.data?.some((j) => ["queued", "running"].includes(j.status))
? 3000
: false,
});
const action = useMutation({
mutationFn: (file?: File) =>
assistant
? createAssistantExportPackage(id)
: importOrdinaryWikiPackage(id, file!),
onSuccess: () => {
void client.invalidateQueries({ queryKey: key });
toast.success(
assistant
? "已创建导出记录,整理完成后可下载"
: "文件已上传,后台正在校验并导入原有 Wiki 和向量",
);
},
onError: (e) => toast.error(e.message),
});
return (
<div className="space-y-3 rounded-xl border bg-card p-4">
<div className="flex flex-wrap items-center justify-between gap-3">
<div>
<h3 className="text-sm font-semibold">
{assistant ? "导出 Wiki 向量数据包" : "导入 Wiki 向量数据包"}
</h3>
<p className="mt-1 text-xs text-muted-foreground">
{assistant
? "保留正文、分类、标签、链接和原始向量;后台整理完成后下载。"
: "选择助手知识库导出的文件,导入当前空库。模型一致时直接复用向量。"}
</p>
</div>
<input
type="file"
accept=".jsonl,.ndjson"
ref={input}
className="hidden"
onChange={(e) => {
const f = e.target.files?.[0];
if (f) action.mutate(f);
e.target.value = "";
}}
/>
<Button
variant="outline"
size="sm"
disabled={action.isPending}
onClick={() =>
assistant ? action.mutate(undefined) : input.current?.click()
}
>
{action.isPending ? (
<Loader2 className="mr-2 size-4 animate-spin" />
) : assistant ? (
<Download className="mr-2 size-4" />
) : (
<Upload className="mr-2 size-4" />
)}
{assistant ? "生成导出包" : "上传并导入"}
</Button>
</div>
{jobs.error && (
<p className="text-sm text-destructive">{jobs.error.message}</p>
)}
{jobs.data?.map((job) => (
<div
key={job.id}
className="flex flex-wrap items-center gap-3 border-t pt-3 text-xs"
>
<Badge variant="outline">
{{
queued: "排队中",
running: "处理中",
failed: "失败",
completed: "已完成",
}[job.status] || job.status}
</Badge>
<span>
{job.created_at && new Date(job.created_at).toLocaleString()}
</span>
<span>
{job.counts?.wiki_page || 0} Wiki · {job.counts?.vector || 0} 向量
</span>
{job.file_size_bytes ? (
<span>{(job.file_size_bytes / 1024 / 1024).toFixed(2)} MB</span>
) : null}
{job.error && <span className="text-destructive">{job.error}</span>}
{assistant &&
job.status === "completed" &&
job.file_exists &&
job.package_download_url && (
<Button asChild size="sm" variant="outline" className="ml-auto">
<a href={job.package_download_url}>下载</a>
</Button>
)}
</div>
))}
</div>
);
}
+123 -10
View File
@@ -396,7 +396,7 @@ FastAPI application on port 8001 with health checks at `GET|HEAD /health`, `/api
| **Uploads** (`/api/threads/{id}/uploads`) | `POST /` - upload files (auto-converts PDF/PPT/Excel/Word); `GET /list` - list; `DELETE /{filename}` - delete |
| **Threads** (`/api/threads/{id}`) | `DELETE /` - remove DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
| **Threads search** (`POST /threads/search`, LangGraph-compat) | Conversation listing. **System threads are excluded by default**: rows whose `metadata.system` is truthy (scheduler runs, roundtable workers) never crowd the user's list; a caller whose `metadata` filter explicitly references `thread_type`/`system` (e.g. studio notebooks) opts out of the exclusion. New `query` field = case-insensitive title (`display_name`) substring search, used by the chats page's server-side pagination. `POST /threads/count` takes the same request body (limit/offset ignored) and returns `{total}` under identical filtering/exclusion semantics ? powers the chats page's ??/??? display without fetching rows (`ThreadMetaStore.count`). `ThreadMetaStore.search` gained `exclude_system`/`query` params; JSON-blob filters batch-scan until the requested `limit`/`offset` window is full (so pagination over a heavily-filtered set never under-fills), with a `thread_id` tie-breaker on the `updated_at DESC` ordering for stable OFFSET paging. Tests: `tests/test_thread_meta_search.py` |
| **Artifacts** (`/api/threads/{id}/artifacts`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types |
| **Artifacts** (`/api/threads/{id}/artifacts`, `/api/artifact-library`) | `GET /{path}` - serve artifacts; active content types (`text/html`, `application/xhtml+xml`, `image/svg+xml`) are always forced as download attachments to reduce XSS risk; `?download=true` still forces download for other file types. `GET /api/artifact-library` scans every current-user normal thread's authoritative `outputs/` directory (including files omitted from `present_files`), excludes uploads and image/audio/video media, joins the owning conversation title/agent route metadata, and exposes search/filter/pagination with a bounded 15-second per-user cache plus `refresh=true` bypass. Frontend route `/page/workspace/library` links each item back to the source chat with `?artifact=<path>` so the chat opens the existing artifact panel directly. |
| **Word export** (`POST /api/writing/export/docx`) | Shared Markdown → DOCX generator used by conversation, artifact, scheduled-task, Deep Research, and AI-writing download surfaces. `app/gateway/word_export.py` builds OOXML with no Python conversion dependency: A4 portrait, 3.7/3.5/2.8/2.6 cm margins, real multilevel heading numbering (`一、` / `(一)` / `1.`), two-character first-line indent on heading levels 1–3, formal body/table styles, page numbers beginning at 1 with odd pages left-aligned and even pages right-aligned, and embedded `app/gateway/assets/fonts/方正小标宋简体.ttf`. Before numbering it removes complete manual heading prefixes (including `2.1` / `2.1.1`) and drops Markdown horizontal rules from Word output. The ordinary-Q&A Lead Agent can emit the matching Markdown heading forms and avoid unordered bullet lists when the admin enables `prompt_prefix.ordinary_qa_markdown_format_enabled` in `system_settings.json`; the switch defaults off, while writing setup, writing mode, notebook, scheduled runs, and SOUL-driven custom agents keep their independent prompts. The font loader validates its internal family, SHA-256, and OpenType embedding permission (`OS/2.fsType`) before export. Tests: `tests/test_word_export.py`, `tests/test_es_query_routing_prompt.py`. |
| **Deep Research** (`/api/deep-research`) | Durable, user-owned research sessions/jobs with replayable SSE events and hidden-thread artifacts. `basic`/`quick` use the compact runner; `detailed` creates bounded subtopics, searches/writes sections concurrently, and assembles a report; `deep` recursively searches follow-up questions under a deterministic breadth/depth query budget. `multi_agent` is a LangGraph 1.x editor/researcher/writer/reviewer graph: its initial plan emits `awaiting_input`, the recorder stores the interrupt and releases the lease, and `POST /jobs/{id}/resume` rebuilds the graph from the persisted approved/revised plan. A bounded `custom_outline` is frozen into the session; headings/list entries deterministically supply the detailed and multi-agent plan, and the shared writer receives it only as a guarded content constraint. Completed reports support owner-scoped follow-up (`GET /sessions/{id}/messages`, `POST /sessions/{id}/chat`): context is only the current report, selected sources and recent messages; source-id citations are persisted, while `allow_new_research=true` is explicitly rejected. Owners may change that evidence set only after the job stops via `PATCH /sessions/{id}/sources/{source_id}`; this applies to future follow-ups rather than rewriting a completed report. Optional report illustrations are gated by `config.yaml -> deep_research.image_generation`: the Harness `OpenAICompatibleImageProvider` requires base64 output, the shared runner step writes only bounded `research-image-01..04.(png|jpg|webp)` artifacts and Markdown uses private `deep-research://` links. `GET /sessions/{id}/artifacts/{name}` checks session ownership before serving an image; an unavailable provider yields a warning but never fails the text report. Runners live in `packages/harness/deerflow/agents/deep_research/runners/`, may never import `app.*`, and use only the `AdapterBundle`. |
| **Gateway auth** | Owner-scoped Deep Research routes await the platform's asynchronous `get_current_user` resolver before they read or write persistence, ensuring each session and job operation receives a concrete user ID. Deep Research artifacts are always resolved from a whitelisted filename via `/mnt/user-data/outputs/<name>` before mapping to a host path, preserving the sandbox guard on native Windows and Linux hosts. |
@@ -1126,6 +1126,8 @@ Both can be modified at runtime via Gateway API endpoints or `DeerFlowClient` me
### Conditional WeKnora LLMWiki
Ordinary knowledge-base cards expose an owner/admin-only name/description edit dialog in `WeKnoraKnowledgeWorkspace.tsx`, reusing `useLlmWikiActions().update` and `PATCH /api/llmwiki/knowledge-bases/{id}`. The route trims names, rejects blank or reserved conversation-deposit names, writes to WeKnora before updating local metadata, and retains existing write authorization. A successful remote list is authoritative for every list scope, including `personal`: absent remote IDs are hidden without deleting mappings/references. Upstream failures remain errors. The grid also filters cached `remote_status=missing` rows and no longer renders stale-record cleanup cards. Regression coverage: `tests/test_llmwiki_knowledge_management.py`.
`llmwiki.weknora.api_base_url` is the provider switch: non-empty enables WeKnora, empty preserves the legacy LLMWiki UI. WeKnora is independently deployed and owns PostgreSQL/Redis/object storage/vector storage/parsers/models; DeerFlow config contains only `api_base_url` and optional `web_base_url`. DeerFlow deliberately sends no WeKnora credential, so the WeKnora endpoint must be restricted to a trusted network and allow DeerFlow traffic itself or through its reverse proxy. There is intentionally no `default_embedding_model_id`: `WeKnoraClient` queries `/api/v1/models` and uses WeKnora's active Embedding model when creating a KB; Wiki creation additionally resolves an active KnowledgeQA model inside WeKnora for synthesis. The native BFF is `app/gateway/routers/llmwiki.py`; ownership/publication mappings live in `deerflow.persistence.llmwiki`; the browser only sees DeerFlow mapping IDs. Knowledge-base owners publish immediately (`private` -> `published`) without a review queue, and can unpublish ordinary bases. The conversation-deposit KB is system-owned, always published, shared by all DeerFlow users, excluded from My Knowledge, and immutable through user/admin detail operations; recognition requires both the reserved name and `system` owner. Existing same-name remote mappings are adopted in place as the system deposit, preserving mapping ids and document links. Its user-facing description, generated Markdown source and tags use `cmzs` branding; list/detail reads synchronize stale remote descriptions, and search results, source previews and embedded-detail JSON recursively replace legacy brand text in historical conversation-deposit payloads. Deposit requests must reference a thread accessible to the current account. Personal knowledge-base list/search views are scoped to the current DeerFlow account even for admins; frontend LLMWiki caches include the live account id, and admin write power does not make other users' private KBs appear as owned. Only `system_role === "admin"` sees the WeKnora connection-config tab. Knowledge cards navigate to a dedicated WeKnora-style detail route. Wiki-enabled bases render and default to `Wiki -> documents -> graph`; standard bases omit Wiki and render `documents -> graph`. The detail page supports sequential multi-file upload with an aggregate result for writable ordinary bases, hides it for public read-only/system-deposit bases, and uses a generic loading label while the embed session starts. It also supports URL and published manual-Markdown imports, browses real Wiki pages/content, proxies authenticated original-file previews, pages through `/chunks/{knowledge_id}`, and renders the Wiki page graph from `/knowledgebase/{kb_id}/wiki/graph`; document details use a right sheet rather than the removed KB modal. The credential-free proxy uses one WeKnora service identity, so per-user separation is enforced by DeerFlow mappings while all remote KBs physically share that identity's WeKnora workspace. WeKnora iframe detail pages first call `/api/llmwiki/knowledge-bases/{id}/weknora/session` through the normal DeerFlow API auth path, then load `/platform/knowledge-bases/{weknora_id}` with a signed `deerflow_weknora_embed` cookie for `/platform`, `/assets`, `/locales`, root assets, and API calls. Injected fetch/XHR rewriting sends all iframe `/api/v1/*`, including colliding `/api/v1/auth/*`, through `/api/llmwiki/weknora-embed/api/v1/*`; Gateway auth/CSRF bypass only applies to those signed iframe proxy requests, the proxy still revalidates the mapped KB, and cookie security honors `X-Forwarded-Proto` for HTTPS reverse proxies. `llmwiki_search` is conditionally registered in `tools.py`, is read-only, and rechecks the current DeerFlow user, agent `llmwiki_knowledge_base_ids`, and thread-context selection on every call. Source previews prefer the cleaned Wiki/LLMWiki page text when a retrieved chunk maps to a Wiki page, with raw split chunks as the fallback. Agent bindings use an inline WeKnora-style searchable/grouped multi-select in the standard agent create/detail UI; new chats use the same interaction pattern as a compact composer dropdown, and selections are written to thread metadata/context and restored on reopen. The DeerFlow frontend is native React and the WeKnora frontend is not modified or embedded. Full deployment and behavior notes: `docs/LLMWIKI_WEKNORA_INTEGRATION_ZH.md`. Tests: `tests/test_llmwiki_weknora.py`.
The embedded detail proxy hides WeKnora's outer `.main > .aside_box` application sidebar and expands the route outlet, but deliberately keeps the Wiki page's internal directory/index sidebar. Do not add an HTML `sandbox` attribute to this trusted, permission-filtered detail iframe: a sandboxed ancestor causes Chromium's built-in Blob PDF viewer to render the browser-blocked page. The Gateway follows upstream redirects only for `/knowledge/{id}/preview` and `/knowledge-bases/{id}/files`, keeping PDF bytes and protected Wiki images inside the signed proxy instead of sending the browser to WeKnora object storage.
@@ -1565,10 +1567,12 @@ detail. Never call `/knowledge-search` from internal search, RAG, or the
`deerflow.integrations.weknora.local_index` and
`deerflow.persistence.llmwiki_index`; preserve the harness → app import
firewall. Gateway composition injects the store/search/sync services through
`local_index.runtime`. `enabled` turns on local vector search and manual
vectorization; `auto_sync` is the separate opt-in for the lease-protected
scheduler. The scheduler sleeps for one full configured interval before its
first scan, so application startup must never trigger immediate vectorization.
`local_index.runtime`. `enabled` turns on local vector search and `auto_sync`
defaults to the lease-protected background scheduler. The scheduler performs
an immediate incremental scan at startup, then rescans every configured
interval so new WeKnora libraries and asynchronously generated Wiki pages are
picked up without a manual vectorization click. Unchanged content must reuse
existing vectors.
Vectors are normalized little-endian float32 BLOBs. A page mirror and all of
its vectors are replaced in one transaction only after strict batched
@@ -1628,8 +1632,10 @@ same DOM value, so a fetched-and-revoked object URL is not acceptable.
Third-party pages embed published Wiki content through the chrome-free route
`/#/embed/knowledge/{mapping_id}` (optional exact-page `wikiSlug`). It defaults
to a recognized Wiki index page (title `索引`/`index`, page type `index`, or an
`index`/`home`/`readme` slug), promotes that page to the first public-directory
entry, then falls back to an overview/root page when no index exists. It retains
`index`/`home`/`readme` slug) and promotes that page to the first public-directory
entry. When upstream pages exist without an explicit index shell, the frontend
injects a virtual `__index__` page and builds its catalogue from the page list;
an all-empty dynamic index response must also fall back to that page list. It retains
directory navigation and supports the shared public-embed `theme` / `bg` /
`text` appearance inputs. WeKnora dynamically renders index page-type counts,
nested category headings, page summaries, and `data-slug` links even though the
@@ -1645,9 +1651,14 @@ the loaded page title, slug, or alias and rewrites them to exact embedded
`wikiSlug` routes. Preserve WeKnora's `wiki-content-link` interaction: primary
brand color, medium weight, dashed underline becoming solid on hover, focus
outline, and in-reader SPA navigation rather than a page reload. `GET
/api/public/llmwiki/knowledge-bases` is the
anonymous discovery endpoint and returns only each published ordinary mapping's
`id` and `name` (plus the collection total). All anonymous reader calls use that
/api/public/llmwiki/knowledge-bases` is the anonymous discovery endpoint. It
validates published ordinary mappings against WeKnora, drops stale mappings,
and returns safe comprehensive metadata from the WeKnora list/detail/page APIs:
document/chunk/processing/share counts, non-secret configuration and timestamps,
Wiki page type/status counts and explicit/generated index entry, plus aggregate
local-vector state/counts/timestamps. Never expose remote WeKnora ids,
tenant/creator identity, storage credentials, API credentials, raw vectors, or
per-page indexing errors. All anonymous reader calls use that
public namespace and the `/knowledge-bases/{id}` Wiki-page children. Those
endpoints must resolve with a synthetic anonymous user and `is_admin=False`,
explicitly require `publication_status=published`, and hard-deny the system
@@ -1726,3 +1737,105 @@ before MarkItDown extraction, avoiding concurrent profile locks. The image
build and `OFFLINE_INSTALL.sh --office-check` both run
`scripts/office_runtime_selfcheck.py`; that smoke test creates and re-parses
minimal DOCX/XLSX/PPTX files and does not need the application database.
## Skill/assistant knowledge invariants
Skill distillation is implemented in `deerflow.skill_knowledge` and
`deerflow.persistence.skill_knowledge`; native assistant bases are in
`deerflow.assistant_knowledge` and
`deerflow.persistence.assistant_knowledge`. Gateway composition, remote adapters,
the lease dispatcher, and APIs remain in `app.gateway` so the harness-to-app
import firewall is preserved.
Never execute a skill or read/upload/summarize its code and script files.
Knowledge fingerprints exclude those files and Markdown code fences. All
resyncs must stage and validate the knowledge digest before activation; the old
snapshot remains authoritative on failure. A binding may have only one active
item (`active_dedupe_key` plus atomic lease claims). Skill distillation targets
only writable WeKnora Wiki knowledge bases in this product version; reject direct
assistant-base targets, non-Wiki/FAQ/conversation/readonly targets, and direct
Neo4j writes. The create/list/resync/retry/detach flow is available to ordinary
users for their own visible skills, writable knowledge bases, jobs and bindings;
administrator-only auto-sync/review APIs remain privileged. Detach/delete operates on source contributions, not canonical rows,
and shared entities/relations are retired only when their final contribution is
gone.
Assistant knowledge has one global full-library base (`知识梳理总库`) and any
number of admin-created custom bases. Ordinary users have read/search access and
administrators own all mutations. Assistant bases only accept WeKnora export
packages or imports triggered from a WeKnora mapping. Do not add direct skill
imports or arbitrary local-file imports to assistant knowledge. Importing into a
custom assistant base must also mirror the package into the global base for
cross-source entity alignment. Admins may rename any assistant base and delete
custom bases. Deletion removes the custom base's Wiki pages, vectors, entities,
relations, import audit rows, and package files; the global base is never
deletable because it owns full-library aggregation.
### Wiki package and local encoder invariants (2026-09)
- `knowledge_transfer.sync_to_global` is called after successful explicit local
Wiki vectorization; it skips conversation/system mappings. Do not enable a new
watcher or automatic remote-write scheduler as a side effect.
- `wiki_packages` exposes file-backed assistant exports and restores into empty
ordinary Wiki libraries. Validate vectors before any remote mutation; recreate
folder IDs from category paths using provider APIs (a raw category_path is
discarded by WeKnora without a matching folder_id). Preserve page metadata and
rebuild backlinks after restoring all pages.
- `PackageArchive` is a replayable, disk-backed JSONL staging index. Always close
it in finally. Runtime imports must not use the legacy in-memory read_package.
- Preserve normalized float32 base64 bytes exactly. Same dimension alone is not
model compatibility. Assistant vector retrieval is fingerprint-gated cosine
search, and any keyword fallback must be labeled as such.
Preserve a valid zero section_index. Export/search vectors belonging to each
current source snapshot (`source.last_import_job_id`), not merely the page's
current revision: one aligned page may intentionally contain current vectors
from several sources.
- The `local` embedding provider uses FastEmbed with official BAAI/bge-m3 ONNX,
1024 dimensions, bounded CPU batches and a per-client async lock. Run inference
off the event loop. Existing page vectors are reusable; zero-vector nonempty
pages need backfill, not a forced rebuild of every page.
FastEmbed is a standard backend dependency. Encoder configuration, directory
upload and warmup live only in the administrator global settings dialog.
A runtime `system_settings.json` path override wins over config.yaml; uploaded
files are streamed into `.deer-flow/models/bge-m3`, never the source tree.
Encoder warmup is explicitly started from the settings UI/API and must never block application startup
or turn a model-load failure into a DeerFlow service failure. Local provider
is offline-only: require local_model_path or DEERFLOW_BGE_M3_MODEL_PATH and
never let the application download model weights. Keep
`local_process_isolation=true` by default: inference runs in the bundled JSONL
worker subprocess, and shutdown must terminate that worker without waiting on
model teardown.
- Every successful mapping snapshot is mirrored to the global assistant base,
including empty snapshots so deleted source pages are retired. Re-import is
source-replacement semantics: absent Wiki/chunk/entity/relation pages must not
remain visible merely because they existed in an older source snapshot.
- Provider graph nodes bind to their original wiki_slug. They must not create
duplicate synthetic entity/relation articles. Store original Wiki metadata
separately from assistant workflow status. Count Wiki pages separately from
raw document chunks.
- The global base aligns entities by normalized display name and aliases across
source records, and aligns content Wiki pages by normalized title. Do not use
source slug as the global canonical identity. Merge unique knowledge blocks
while retaining source contributions/origins. Uploaded packages must rebind
the temporary upload source to the stable WeKnora knowledge-base id in the
package, so re-upload is source-replacement rather than append. Retrieval
candidates exclude summary/index/directory pages; keyword matching, vector
snippets, citations and prompts must use knowledge body/sections, never page
summary as a substitute.
- Migration `20260903_01` adds nullable page metadata and merges the pre-existing
knowledge/workflow Alembic heads. Regression coverage is in
`tests/test_wiki_package_roundtrip.py`, `test_llmwiki_wiki_sync.py` and
`test_assistant_knowledge_repository.py`.
## openGauss / GaussDB persistence invariant
`database.backend=gauss` is an application-ORM backend, parallel to `mysql`.
It normalizes `database.gauss_url` to `opengauss+asyncpg://` and requires the
`gauss` optional dependency (`asyncpg` plus `opengauss-sqlalchemy`). Never route
a Gauss URL through SQLAlchemy's generic PostgreSQL dialect: the wire protocol
may connect, but dialect initialization and schema DDL can fail against the
Gauss server/version behavior. Keep the LangGraph checkpointer on the separately
configured SQLite backend; the PostgreSQL checkpointer schema is not part of the
Gauss compatibility contract. Treat both `postgresql` and `opengauss` dialect
names as timezone-aware PostgreSQL-family engines in portable types and
dialect-specific session settings.
+2 -2
View File
@@ -13,8 +13,8 @@ FROM python:3.12-slim-bookworm AS builder
ARG NODE_MAJOR=22
ARG APT_MIRROR
ARG UV_INDEX_URL
# Optional extras to install (e.g. "postgres" for PostgreSQL support)
# Usage: docker build --build-arg UV_EXTRAS=postgres ...
# Optional extras to install (e.g. "postgres" or "gauss")
# Usage: docker build --build-arg UV_EXTRAS=gauss ...
ARG UV_EXTRAS
# Optionally override apt mirror for restricted networks (e.g. APT_MIRROR=mirrors.aliyun.com)
+147 -11
View File
@@ -162,6 +162,7 @@ FastAPI application providing REST endpoints for frontend integration:
| `POST /api/threads/{id}/uploads` | Upload files (auto-converts PDF/PPT/Excel/Word to Markdown, rejects directory paths) |
| `GET /api/threads/{id}/uploads/list` | List uploaded files |
| `DELETE /api/threads/{id}` | Delete DeerFlow-managed local thread data after LangGraph thread deletion; unexpected failures are logged server-side and return a generic 500 detail |
| `GET /api/artifact-library` | List the current user's generated conversation files across normal threads, with filename/conversation search, file-kind filtering, pagination, and an optional cache-bypassing refresh. The filesystem `outputs/` directories remain authoritative; uploads and image/audio/video files are excluded. |
| `GET /api/threads/{id}/artifacts/{path}` | Serve generated artifacts |
| `GET/POST /api/deep-research/*` | Durable Deep Research sessions and jobs. `quick`/`basic` run the compact report flow; `detailed` plans subtopics and writes independent sections; `deep` recursively investigates bounded follow-up branches. All modes reuse DeerFlow's configured material provider, LLM factory, hidden-thread artifacts, and SSE event log. Deep Research session/job/event/source/message writes serialise their rows in the writer transaction, rather than doing a post-commit ORM refresh or readback; this prevents MySQL read/write-splitting replica lag from failing “start writing”, dispatcher claim, report persistence, or follow-up history. Whole-document artifact/report rewrites likewise create their reversible version snapshots from writer-local ORM data and never refresh after commit, so replica lag cannot restore an otherwise successful rewrite. A report is written to the session and confirmed before its Job becomes `completed`; an unconfirmed write is retried and never exposed as a false completion. Retrying an already-created queued job re-nudges the dispatcher, recovering an interrupted HTTP response without creating a second report. If progress-event persistence, durable replay, or live fan-out is briefly unavailable, the worker/stream logs the degradation and continues or reconnects rather than failing the whole job. Whole-report rewrite accepts an optional `reportOutline` Markdown template; the durable job persists it and applies it as the chapter structure constraint. Optional report illustrations use a separately configured OpenAI-compatible image endpoint and are served only through the owning session's authenticated artifact route. |
| `/api/enterprise-research/*` | Temporarily disabled. Its router and persistence initialization are not registered, so these endpoints return 404 and no enterprise-research tables are created or migrated during startup. |
@@ -903,19 +904,24 @@ without deleting the audit row. DeerFlow stores only the artifact reference
never the Markdown body and never a task-system delivery. Continue to preview
the file through the existing thread-artifact endpoint.
## 普通知识库管理
在「普通知识库」的「我的知识库」或「公共 LLMWiki」中,知识库所有者和管理员可以点击卡片上的「编辑」,修改名称和描述(描述可清空)。保存后同步更新知识服务和本地记录,并刷新列表;系统「对话沉淀」库不提供编辑入口。
远端已经删除的知识库会在下次列表刷新时自动隐藏,包括个人、公共列表和知识库选择器;页面每 15 秒自动刷新,也可以手动点击「刷新」。远端服务请求失败时展示错误,不把连接故障当成删除,不自动删除本地映射或已有引用。
## DeerFlow-local LLMWiki Wiki vector index
LLMWiki can mirror processed WeKnora Wiki pages into DeerFlow and search only
processed Wiki content. A knowledge base uses exact NumPy cosine similarity
only after its whole Wiki has completed one coherent vectorization pass; until
then internal search calls WeKnora's Wiki-page search/list API and reads the
matched Wiki pages. It never falls back to raw documents or chunks. The local
vector feature is disabled by default. Configure the three
`llmwiki.local_wiki_index.embedding` values (`model`, `base_url`, and
`api_key`), then change only `enabled` to `true` in `config.yaml`. Manual
vectorization is the default. Set `auto_sync: true` only when periodic
background synchronization is wanted; the scheduler waits one full interval
after startup before its first scan, so boot does not trigger a rebuild.
matched Wiki pages. It never falls back to raw documents or chunks. Configure
`llmwiki.local_wiki_index.embedding`, then enable the local index in
`config.yaml`. `auto_sync` defaults to true: the scheduler performs one
immediate incremental scan at startup and then scans every configured interval
(30 seconds by default). Wiki changes are content-hash checked, so unchanged
pages reuse their vectors instead of being encoded again.
`database_batch_size` caps each vector insert statement (default `50`) while
the page replacement remains atomic, preventing a large Wiki from flooding
the SQL connection in one statement.
@@ -992,8 +998,10 @@ report-structure column for databases stamped by either old variant.
Published knowledge bases also have a login-free, chrome-free reader at
`/#/embed/knowledge/{mapping_id}` for direct third-party iframe use. With no
`wikiSlug`, it selects the Wiki **索引** page first (recognized by title, page
type, or `index`/`home`/`readme` slug), pins that page to the top of the public
directory, and otherwise falls back to an overview/root page;
type, or `index`/`home`/`readme` slug) and pins that page to the top of the
public directory. If WeKnora has pages but no explicit index page, the reader
creates a virtual `__index__` landing page from the public directory; empty
dynamic-index groups also fall back to those page summaries;
`?wikiSlug=...` deep-links one page. The reader supports
`theme=light|dark`, `bg`, and `text` embed appearance parameters. Index-article
catalogue sections (page-type counts, nested categories, page summaries) are
@@ -1009,8 +1017,14 @@ only from the read-only `/api/public/llmwiki/knowledge-bases/{id}/...` namespace
which rechecks `publication_status=published` on every request and denies the
system conversation-deposit knowledge base. Private or unpublished bases are
never exposed by this route. Anonymous consumers can discover the available
public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`; each
entry deliberately contains only the DeerFlow mapping `id` and display `name`.
public Wiki libraries through `GET /api/public/llmwiki/knowledge-bases`. The
endpoint validates each published mapping against WeKnora, omits stale remote
mappings, and returns safe comprehensive metadata: description/type,
document/chunk/processing/share counts, non-secret configuration and
timestamps, Wiki page/type/status counts plus explicit/generated index mode and
default embed route, and aggregate local-vector state/counts/timestamps. It
never returns the WeKnora id, tenant/creator identity, storage credentials, API
credentials, raw vectors, or per-page indexing errors.
Protected `resource://` and supported object-storage images are fetched through
the current mapping instead of being exposed as upstream URLs. Authenticated
readers use `GET /api/llmwiki/knowledge-bases/{id}/files?file_path=...`; public
@@ -1081,3 +1095,125 @@ uploads are parsed directly; legacy DOC/XLS/PPT and ODT/ODS/ODP files are
normalized through an isolated LibreOffice profile before Markdown extraction.
Run `./OFFLINE_INSTALL.sh --office-check` from the offline bundle to generate
and re-parse Word, Excel, and PowerPoint samples without requiring a database.
## Skill knowledge distillation and assistant knowledge
The Gateway now exposes an administrator-only skill distillation pipeline at
`/api/skill-knowledge`. Each request can send up to 200 skills to one writable
WeKnora Wiki knowledge base; multi-base distillation is done by repeated
operations so the local binding remains `skill_name + target_id`. The scanner
never reads or uploads code/script files, strips Markdown code fences, blocks
credential-like files and content, safely inspects ZIP members, and converts
supported Office/PDF/spreadsheet files off the event loop. Jobs use database
claims and renewable leases, stage snapshots before activation, retain the
prior active version on failure, support per-item retry, and keep per-binding
contribution records for safe detach/delete. Direct assistant-base targets,
non-Wiki/FAQ/conversation/readonly targets, and direct Neo4j writes are rejected
for this product version.
`/api/assistant-knowledge` manages DeerFlow-native assistant knowledge bases.
There is one global full-library base named `知识梳理总库`, plus admin-created
custom assistant bases. Reads and search are available to authenticated users;
global initialization, custom-base creation, WeKnora export/import, source
reimport, and source deletion are admin-only. Assistant bases only accept
WeKnora export packages or imports triggered from a WeKnora mapping; direct
skill import and arbitrary local-file import return 410. Importing into a custom
assistant base also mirrors the same package into the global base, where
canonical entities and relations are aligned across all imported sources.
The database schema is introduced by Alembic revision `20260901_01`.
### Wiki 流转与本地小批量编码
所有登录用户都可在「技能 → 归纳到知识库」中多选自己可见的技能,并归纳到一个
自己可写的 WeKnora Wiki 知识库;用户只能看到和维护自己发起的归纳记录,管理员
接口仍可查看全量记录。技能归纳上传清理后的知识正文,等待 WeKnora 完成 Wiki 分析,再建立本地
Wiki 向量并通过落盘导出包自动沉淀到「知识梳理」。内置技能没有数据库所有权记录
也可以归纳。若解析结束却没有 Wiki,会提示检查生成模型和资料;文档解析完成
不等于 Wiki 生成成功,例如生成服务余额不足仍可能表现为文档已完成。
管理员可以在普通知识库直接生成向量数据包,每次操作有独立后台记录和下载入口。
助手知识库支持上传包、异步导入和导出;新建空的普通 Wiki 库可上传助手导出包。管理员可编辑助手知识库名称与描述,也可删除自建助手知识库及其 Wiki、向量、实体、关系、导入记录和落盘数据包;承担全库沉淀的「知识梳理总库」允许编辑但禁止删除。
包保留正文、别名/标签、目录分类、双向链接、实体关系及 float32 向量。
回流时通过 WeKnora API 重建目录/页面,向量进入 DeerFlow 本地 Wiki 索引,
**不直接写 WeKnora 的底层向量或图数据库**。普通与助手 Wiki 使用同一个阅读组件。
小批量 CPU 编码可配置如下,无需额外部署 HTTP 编码服务:
```yaml
llmwiki:
local_wiki_index:
enabled: true
auto_sync: true
sync_interval_seconds: 30
embedding:
provider: local
model: bge-embedding-m3
dimensions: 1024
batch_size: 2
local_threads: 2
local_process_isolation: true
# 离线部署时可指定已下载的 BAAI/bge-m3 模型目录:
# local_model_path: /opt/models/bge-m3
```
本地编码依赖随标准后端安装。将完整 BGE-M3 模型目录提前放到离线服务器,并通过
`local_model_path` 或 `DEERFLOW_BGE_M3_MODEL_PATH` 指向它。管理员也可在
「设置和更多 → 设置 → 本地编码器」中选择服务器目录,或从浏览器上传完整模型目录;
页面配置持久化到运行目录的 `system_settings.json`,上传文件落到
`.deer-flow/models/bge-m3`,均不进入源码仓库。系统强制 `local_files_only`,
**不会访问互联网或下载模型**。管理员可在同一设置页点击「一键启动编码器」异步加载;该操作不挂在应用启动流程,加载失败不会
中断 DeerFlow 主服务。默认由独立低优先级子进程承载模型内存,正常停止 DeerFlow
时会一并回收该子进程;编码进程异常退出只会将当前向量任务标记失败。模型目录应包含
`onnx/model.onnx`(官方权重约 2.3 GB)及对应 tokenizer/config 文件。权重不进入业务源码仓库。
浏览器目录上传采用磁盘暂存和 4 MB 分块复制,页面显示上传进度;经 Nginx 等反向
代理部署时仍需把请求体上限和读写超时配置到可承载完整模型目录。
相同模型指纹、相同内容的页面复用现有向量;短正文/摘要至少保留一个非空分段。
数据包导入不重新编码,模型指纹或维度不一致时明确拒绝,不能只按维度混用模型。
问答编码查询本身仍需可用的同一模型。WeKnora 的原始文档向量不能当作生成后
Wiki 正文的向量使用,两种文本需要分别建立各自的首次索引。
Wiki 页面新增、编辑、删除及文档上传/重解析都会立即触发一次增量检查;WeKnora
后台稍后才完成的 Wiki 抽取由 30 秒周期补扫接管。每次成功向量化后自动把该普通
知识库的最新完整快照同步到「知识梳理总库」,包括服务重启前已有库和新建库;来源
页面被删除时也会在总库中退役,不保留失效 Wiki。
「知识梳理总库」不是按来源简单堆叠:导入时会规范化全/半角、空白、标点及常见实体
类型,并利用实体别名对齐同一人物、组织、产品或概念;不同来源中同标题的实体 Wiki
只保留一个规范页面,正文按去重后的知识块合并,来源与实体/关系贡献仍分别留痕。
重复上传同一个 WeKnora 导出包时,以包内稳定知识库 ID 识别同一来源并执行快照替换,
不会再产生一批新的片段副本。已有历史重复数据会在总库首次读取、检索、导出或下次
同步时自动归并。助手知识检索排除目录、索引和摘要页;关键词只查标题与知识正文,
向量检索返回实际命中的正文片段,问答上下文和检索卡片均不再用摘要代替知识。
大包采用流式 JSONL 落盘、同盘临时 SQLite 分阶段读取及异步导入,避免一次性将
整个包加载进内存。部署需预留包、暂存索引和业务数据库空间,并配置代理上传大小
与超时。当前不宣称断点续传或 GB 级压力验收已完成。
## openGauss / GaussDB application database
The application ORM can use a PG-compatible openGauss/GaussDB database through
the official `opengauss-sqlalchemy` async dialect. Install the optional driver
set before enabling it:
```bash
uv sync --extra gauss
```
Configure the business database independently from the LangGraph checkpointer:
```yaml
database:
backend: gauss
gauss_url: $GAUSS_DATABASE_URL
checkpointer:
type: sqlite
connection_string: .deer-flow/data/deerflow.db
```
`GAUSS_DATABASE_URL` should normally be
`opengauss+asyncpg://user:password@host:26000/deerflow`. Plain
`opengauss://`, `gaussdb://`, and PostgreSQL-style URLs are accepted and
normalized to the openGauss dialect. Do not configure the LangGraph
PostgreSQL checkpointer against GaussDB: its migrations are PostgreSQL-specific.
For a production image, build with `--build-arg UV_EXTRAS=gauss`.
@@ -32,6 +32,7 @@ from app.gateway.routers import (
agents,
ai_writing,
artifacts,
assistant_knowledge,
assistants_compat,
auth,
browser_context,
@@ -88,6 +89,7 @@ from app.gateway.routers import (
sensitive_words,
sentiment_agent,
skill_addresses,
skill_knowledge,
skills,
suggestions,
system_settings,
@@ -1581,6 +1583,13 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
# Skills API is mounted at /api/skills
app.include_router(skills.router)
# Admin skill distillation matrix + global assistant knowledge base.
app.include_router(skill_knowledge.router)
app.include_router(assistant_knowledge.router)
app.include_router(assistant_knowledge.export_router)
from app.gateway.routers import wiki_packages
app.include_router(wiki_packages.router)
# Artifacts API is mounted at /api/threads/{thread_id}/artifacts
app.include_router(artifacts.router)
@@ -354,6 +354,33 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.llmwiki_store = make_llmwiki_store(sf)
register_llmwiki_store(app.state.llmwiki_store)
# Skill distillation and the single global assistant knowledge base.
# Both repositories live in ``deerflow.*``; only this composition layer
# wires them to Gateway executors and remote adapters.
from deerflow.persistence.assistant_knowledge import make_assistant_knowledge_store
from deerflow.persistence.skill_knowledge import make_skill_knowledge_store
app.state.assistant_knowledge_store = make_assistant_knowledge_store(sf)
app.state.skill_knowledge_store = make_skill_knowledge_store(sf)
if app.state.skill_knowledge_store is not None and app.state.assistant_knowledge_store is not None:
from app.gateway.skill_knowledge_job_dispatcher import SkillKnowledgeJobDispatcher
from app.gateway.skill_knowledge_job_executor import SkillKnowledgeJobExecutor
app.state.skill_knowledge_executor = SkillKnowledgeJobExecutor(app)
app.state.skill_knowledge_dispatcher = SkillKnowledgeJobDispatcher(
app.state.skill_knowledge_store,
app.state.skill_knowledge_executor,
)
app.state.skill_knowledge_dispatcher.start()
from app.gateway.skill_knowledge_auto_sync import SkillKnowledgeAutoSyncService
app.state.skill_knowledge_auto_sync = SkillKnowledgeAutoSyncService(app)
app.state.skill_knowledge_auto_sync.start()
else:
app.state.skill_knowledge_executor = None
app.state.skill_knowledge_dispatcher = None
app.state.skill_knowledge_auto_sync = None
# DeerFlow-local Wiki mirror/vector index. The store is always wired so
# migrations and status inspection work while feature flags are off;
# network/search services only exist when explicitly enabled.
@@ -374,13 +401,32 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
resolved_llmwiki = get_resolved_llmwiki_runtime(config)
weknora_client = build_weknora_client(resolved_llmwiki) if resolved_llmwiki.weknora_enabled else None
if local_index_config.enabled:
embedding_config = local_index_config.embedding
# The administrator may select or upload the offline encoder from the
# UI. This runtime-owned value deliberately overrides config.yaml and
# survives application restarts without modifying release assets.
if embedding_config.provider == "local":
from deerflow.config.system_settings import load_system_settings
runtime_model_path = load_system_settings().local_wiki_encoder.model_path.strip()
if runtime_model_path:
embedding_config = embedding_config.model_copy(
update={"local_model_path": runtime_model_path}
)
embedding_configured = bool(
embedding_config.enabled
and (embedding_config.base_url or embedding_config.provider == "local")
and embedding_config.model.strip()
)
if embedding_configured:
from deerflow.integrations.weknora.local_index.embedding import StrictWikiEmbeddingClient
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(embedding_config)
if local_index_config.enabled and app.state.llmwiki_embedding is not None:
from deerflow.integrations.weknora.local_index.search import WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from deerflow.integrations.weknora.local_index.vector_cache import WikiVectorCache
app.state.llmwiki_embedding = StrictWikiEmbeddingClient(local_index_config.embedding)
app.state.llmwiki_vector_cache = WikiVectorCache(
app.state.llmwiki_index_store,
max_knowledge_bases=local_index_config.cache_max_knowledge_bases,
@@ -399,6 +445,12 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
app.state.llmwiki_vector_cache,
weknora_client,
)
from app.gateway.knowledge_transfer import sync_to_global
async def on_wiki_vectorized(mapping):
return await sync_to_global(app, mapping)
app.state.llmwiki_sync_service.on_completed = on_wiki_vectorized
if weknora_client is not None or app.state.llmwiki_vector_search is not None:
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService
@@ -676,6 +728,12 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
from deerflow.workflows.runtime.sql_runner import dispose_engines
await dispose_engines()
skill_knowledge_auto_sync = getattr(app.state, "skill_knowledge_auto_sync", None)
if skill_knowledge_auto_sync is not None:
await skill_knowledge_auto_sync.close()
skill_knowledge_dispatcher = getattr(app.state, "skill_knowledge_dispatcher", None)
if skill_knowledge_dispatcher is not None:
await skill_knowledge_dispatcher.close()
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
if queue is not None:
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
@@ -0,0 +1,28 @@
"""Wiki package flow orchestration; all remote writes use WeKnora APIs."""
from __future__ import annotations
from starlette.requests import Request
async def sync_to_global(app, mapping):
from app.gateway.routers.assistant_knowledge import _export_weknora_then_import_for_job
if mapping.get("is_conversation_deposit") or mapping.get("owner_user_id") == "system":
return {"status": "excluded"}
store = app.state.assistant_knowledge_store
revision, fingerprint, _ = await app.state.llmwiki_index_store.get_index_revision(str(mapping["id"]))
base = await store.initialize(created_by=None)
for source in await store.list_sources(base_id=base["id"]):
meta = source.get("metadata") or {}
if source.get("source_key") == mapping["id"] and meta.get("index_revision") == revision:
job = await store.get_job(source.get("last_import_job_id")) if source.get("last_import_job_id") else None
if job and job.get("status") == "completed":
return {"status": "unchanged", "job_id": job["id"], "base_id": base["id"]}
source = {"provider": "weknora", "type": "weknora", "knowledge_base_id": mapping["id"], "knowledge_base_name": mapping["name"], "index_revision": revision, "embedding_fingerprint": fingerprint}
job = await store.create_queued_import_job(source_type="weknora", source_key=mapping["id"], source_name=mapping["name"], trigger="vectorization_completed", created_by=None, metadata=source, mapping_id=mapping["id"], base_id=base["id"])
await _export_weknora_then_import_for_job(request=Request({"type": "http", "app": app}), job_id=job["id"], mapping=mapping, remote_id=mapping["weknora_id"], source=source, actor=None, index_fresh=True)
result = await store.get_job(job["id"])
if not result or result.get("status") != "completed":
raise RuntimeError(f"向量化已完成,但知识梳理同步失败:{(result or {}).get('last_error_message') or job['id']}")
return {"status": "completed", "job_id": job["id"], "base_id": base["id"], "counts": result.get("counts")}
@@ -49,9 +49,14 @@ async def _sync_once(app: FastAPI) -> None:
async def local_wiki_index_scheduler_loop(app: FastAPI) -> None:
interval = app.state.config.llmwiki.local_wiki_index.sync_interval_seconds
logger.info(
"Local Wiki auto-sync scheduler started; first scan delayed by %s seconds",
"Local Wiki auto-sync scheduler started; first scan is immediate, interval=%s seconds",
interval,
)
while True:
try:
await _sync_once(app)
except asyncio.CancelledError:
raise
except Exception:
logger.exception("Local Wiki auto-sync scheduler tick failed (non-fatal)")
await asyncio.sleep(interval)
await _sync_once(app)
@@ -40,6 +40,7 @@ _MAX_SOURCE_CHARS = 1400
@dataclass(frozen=True)
class LlmWikiRagContext:
knowledge_base_ids: list[str]
assistant_knowledge_base_ids: list[str]
source_count: int
prompt: str
sources: list[dict[str, Any]]
@@ -50,6 +51,10 @@ def _store(request: Request) -> LlmWikiStore | None:
return getattr(request.app.state, "llmwiki_store", None)
def _assistant_store(request: Request) -> Any | None:
return getattr(request.app.state, "assistant_knowledge_store", None)
async def _actor(request: Request) -> tuple[str, bool]:
user = await get_optional_user_from_request(request)
if user is None:
@@ -76,6 +81,13 @@ def _body_selected_ids(body: Any) -> list[str]:
return []
def _body_selected_assistant_ids(body: Any) -> list[str]:
for source in (getattr(body, "context", None), getattr(body, "metadata", None)):
if isinstance(source, Mapping) and "assistant_knowledge_base_ids" in source:
return _as_id_list(source.get("assistant_knowledge_base_ids"))
return []
def _assistant_agent_id(body: Any) -> str | None:
assistant_id = validate_agent_id(getattr(body, "assistant_id", None))
if assistant_id:
@@ -106,8 +118,17 @@ def resolve_requested_llmwiki_knowledge_base_ids(body: Any) -> list[str]:
return _body_selected_ids(body)
def resolve_requested_assistant_knowledge_base_ids(body: Any) -> list[str]:
"""Return assistant knowledge bases explicitly selected for the run."""
return _body_selected_assistant_ids(body)
def llmwiki_selection_requested(body: Any) -> bool:
return bool(resolve_requested_llmwiki_knowledge_base_ids(body))
return bool(
resolve_requested_llmwiki_knowledge_base_ids(body)
or resolve_requested_assistant_knowledge_base_ids(body)
)
def _latest_human_query(graph_input: dict[str, Any]) -> str:
@@ -149,13 +170,14 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
if result.get("kind") == "wiki_page":
sections = list(result.get("matched_sections") or [])
content = "\n\n".join(str(section.get("content") or "") for section in sections if section.get("content"))
content = content or str(result.get("content") or result.get("content_markdown") or "")
raw = {**result, "source": mapping.get("name") or "LLMWiki"}
return {
"index": index,
"id": result.get("wiki_page_id") or result.get("id"),
"title": result.get("title") or mapping.get("name") or f"Wiki 页面 {index}",
"type": "llmwiki_wiki",
"snippet": _truncate(content or str(result.get("summary") or ""), 360),
"snippet": _truncate(content, 360),
"content": content,
"source": mapping.get("name") or "LLMWiki",
"score": float(result.get("score") or 0),
@@ -191,18 +213,52 @@ def _source_from_result(result: dict[str, Any], mapping: dict[str, Any], index:
}
def _source_from_assistant_result(result: dict[str, Any], index: int) -> dict[str, Any]:
title = str(result.get("title") or f"助手知识库片段 {index}")
matched_section = result.get("matched_section") if isinstance(result.get("matched_section"), Mapping) else {}
content = str(
matched_section.get("content")
or result.get("content_markdown")
or result.get("content")
or result.get("summary")
or ""
)
# The retrieval citation must expose the matched knowledge itself. A page
# summary is display metadata and can omit the fact that answered the query.
snippet = content
raw = {
**result,
"source": "助手知识库",
"url": "/page/workspace/knowledge/assistant",
}
return {
"index": index,
"id": result.get("id") or result.get("slug") or f"assistant:{index}",
"title": title,
"type": "assistant_knowledge",
"snippet": _truncate(snippet, 360),
"content": content,
"source": "助手知识库",
"score": float(result.get("score") or 0),
"url": "/page/workspace/knowledge/assistant",
"skillName": "assistant_knowledge_search",
"mode": "citation",
"raw": raw,
}
def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
if not sources:
return "当前用户选择了 LLMWiki 知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
return "当前用户选择了知识库,但 DeerFlow 未检索到可用片段。请直接说明没有检索到可用背景资料,不要编造知识库内容或引用编号。"
lines = [
"当前用户选择了 LLMWiki 知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
"当前用户选择了知识库。DeerFlow 已在模型回答前自动检索到以下背景资料。",
"请优先依据这些资料回答;如果资料不足,请明确说明不足之处,不要编造。",
"引用规则:正文关键结论必须使用 [1]、[2] 这样的编号引用下方资料;只能引用存在的编号;不要在正文末尾重复输出完整参考文献列表,前端会展示引用卡片。",
"",
f"用户问题:{query}",
"",
f'<llmwiki_wiki_reference_batch source_count="{len(sources)}">',
f'<knowledge_reference_batch source_count="{len(sources)}">',
]
for source in sources:
meta = " / ".join(
@@ -213,12 +269,12 @@ def _build_prompt(query: str, sources: list[dict[str, Any]]) -> str:
)
if item
)
lines.append(f"[{source['index']}] {source.get('title') or 'LLMWiki 片段'}")
lines.append(f"[{source['index']}] {source.get('title') or '知识库片段'}")
if meta:
lines.append(f"来源:{meta}")
lines.append(_truncate(str(source.get("content") or source.get("snippet") or ""), _MAX_SOURCE_CHARS))
lines.append("")
lines.append("</llmwiki_wiki_reference_batch>")
lines.append("</knowledge_reference_batch>")
return "\n".join(lines).strip()
@@ -229,52 +285,102 @@ async def build_llmwiki_rag_context(
graph_input: dict[str, Any],
) -> LlmWikiRagContext:
selected_ids = resolve_requested_llmwiki_knowledge_base_ids(body)
if not selected_ids:
return LlmWikiRagContext([], 0, "", [], "")
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
local_config = request.app.state.config.llmwiki.local_wiki_index
if not (local_config.enabled and local_config.internal_search_enabled) and not runtime.weknora_enabled:
return LlmWikiRagContext(selected_ids, 0, "", [], "")
store = _store(request)
if store is None:
return LlmWikiRagContext(selected_ids, 0, "", [], "")
assistant_selected_ids = resolve_requested_assistant_knowledge_base_ids(body)
if not selected_ids and not assistant_selected_ids:
return LlmWikiRagContext([], [], 0, "", [], "")
query = _latest_human_query(graph_input)
if not query:
return LlmWikiRagContext(selected_ids, 0, "", [], "")
return LlmWikiRagContext(selected_ids, assistant_selected_ids, 0, "", [], "")
user_id, is_admin = await _actor(request)
rows: list[dict[str, Any]] = []
for mapping_id in selected_ids:
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if row is None:
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
rows.append(row)
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
if search_service is None:
return LlmWikiRagContext(
selected_ids,
0,
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
[],
query,
)
try:
payload = await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
except WikiIndexError as exc:
logger.warning("Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s", sanitize_log_param(user_id), len(rows), exc.code)
return LlmWikiRagContext(selected_ids, 0, f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。", [], query)
mapping_by_id = {str(row["id"]): row for row in rows}
sources: list[dict[str, Any]] = []
for result in payload.get("results") or []:
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
if mapping is None:
continue
sources.append(_source_from_result(result, mapping, len(sources) + 1))
if len(sources) >= _MAX_SOURCES:
break
return LlmWikiRagContext(selected_ids, len(sources), _build_prompt(query, sources), sources, query)
runtime = get_resolved_llmwiki_runtime(getattr(request.app.state, "config", None))
local_config = request.app.state.config.llmwiki.local_wiki_index
user_id, is_admin = await _actor(request)
if selected_ids:
llmwiki_available = (local_config.enabled and local_config.internal_search_enabled) or runtime.weknora_enabled
store = _store(request)
if not llmwiki_available or store is None:
if not assistant_selected_ids:
return LlmWikiRagContext(selected_ids, assistant_selected_ids, 0, "", [], query)
else:
rows: list[dict[str, Any]] = []
for mapping_id in selected_ids:
row = await store.get_authorized(mapping_id, user_id, write=False, is_admin=is_admin)
if row is None:
raise HTTPException(status_code=404, detail=f"Knowledge base {mapping_id} not found")
rows.append(row)
search_service = getattr(request.app.state, "llmwiki_retrieval_service", None)
if search_service is None:
if not assistant_selected_ids:
return LlmWikiRagContext(
selected_ids,
assistant_selected_ids,
0,
"Wiki 检索暂不可用。请明确说明系统检索不可用,不要表述为资料不存在。",
[],
query,
)
try:
payload = (
await search_service.search(query, rows, top_k_pages=_MAX_SOURCES)
if search_service is not None
else {"results": []}
)
except WikiIndexError as exc:
logger.warning(
"Wiki-only pre-retrieval failed user=%s kb_count=%s code=%s",
sanitize_log_param(user_id),
len(rows),
exc.code,
)
if not assistant_selected_ids:
return LlmWikiRagContext(
selected_ids,
assistant_selected_ids,
0,
f"Wiki 检索暂不可用({exc.code})。请明确说明系统检索状态,不要声称知识库没有资料。",
[],
query,
)
payload = {"results": []}
mapping_by_id = {str(row["id"]): row for row in rows}
for result in payload.get("results") or []:
mapping = mapping_by_id.get(str(result.get("knowledge_base_id") or ""))
if mapping is None:
continue
source = _source_from_result(result, mapping, len(sources) + 1)
if not str(source.get("content") or "").strip():
continue
sources.append(source)
if len(sources) >= _MAX_SOURCES:
break
if assistant_selected_ids and len(sources) < _MAX_SOURCES:
assistant_store = _assistant_store(request)
if assistant_store is not None:
rows = await assistant_store.search_pages(
base_ids=assistant_selected_ids,
query=query,
limit=_MAX_SOURCES - len(sources),
include_chunks=True,
embedding_client=getattr(request.app.state, "llmwiki_embedding", None),
)
for row in rows:
sources.append(_source_from_assistant_result(row, len(sources) + 1))
if len(sources) >= _MAX_SOURCES:
break
return LlmWikiRagContext(
selected_ids,
assistant_selected_ids,
len(sources),
_build_prompt(query, sources),
sources,
query,
)
def attach_llmwiki_rag_to_graph_input(
@@ -294,6 +400,7 @@ def attach_llmwiki_rag_to_graph_input(
existing = str(extra.get("prompt_prefix") or "").strip()
extra["prompt_prefix"] = f"{existing}\n\n{rag.prompt}".strip() if existing else rag.prompt
extra["llmwiki_knowledge_base_ids"] = rag.knowledge_base_ids
extra["assistant_knowledge_base_ids"] = rag.assistant_knowledge_base_ids
extra["llmwiki_reference_query"] = rag.query
if rag.sources:
extra["llmwiki_reference_sources"] = rag.sources
@@ -461,7 +461,7 @@ async def _set_analytics_read_timeout(session) -> None:
- MariaDB: ``max_statement_time`` (seconds)
- MySQL <= 5.6: no per-statement timeout variable — skip it entirely
(5.6 raises "Unknown system variable" otherwise)
- PostgreSQL: ``statement_timeout`` (milliseconds)
- PostgreSQL/openGauss: ``statement_timeout`` (milliseconds)
"""
conn = await session.connection()
dialect = conn.dialect
@@ -474,7 +474,7 @@ async def _set_analytics_read_timeout(session) -> None:
await _try_set_session_value(session, f"SET SESSION max_execution_time = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
# else: MySQL <= 5.6 has no statement-timeout variable — skip silently.
await _try_set_session_value(session, f"SET SESSION innodb_lock_wait_timeout = {_LEADERBOARD_LOCK_WAIT_SECONDS}")
elif name == "postgresql":
elif name in {"postgresql", "opengauss"}:
await _try_set_session_value(session, f"SET LOCAL statement_timeout = {_LEADERBOARD_QUERY_TIMEOUT_MS}")
@@ -1,329 +1,625 @@
import logging
import mimetypes
import zipfile
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import quote
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import FileResponse, PlainTextResponse, Response
from pydantic import BaseModel
from app.gateway.authz import require_permission
from app.gateway.path_utils import aresolve_thread_virtual_path
from deerflow.config.paths import get_paths
from deerflow.runtime.thread_paths import aresolve_path_user_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api", tags=["artifacts"])
ACTIVE_CONTENT_MIME_TYPES = {
"text/html",
"application/xhtml+xml",
"image/svg+xml",
}
EDITABLE_TEXT_SUFFIXES = {
".md",
".markdown",
".txt",
".json",
".yaml",
".yml",
".csv",
".xml",
".log",
}
_VIRTUAL_USER_DATA_PREFIX = "mnt/user-data"
def coerce_artifact_virtual_path(path: str) -> str:
"""Normalize an artifacts URL path to a sandbox virtual path.
Streaming write previews may request a bare filename such as
``未命名文件.md`` before the model has emitted a real
``/mnt/user-data/outputs/...`` path. Mapping a single path segment into
the thread outputs directory turns that into a normal 404 (file not written
yet) instead of HTTP 400 ``Path must start with /mnt/user-data``.
"""
stripped = path.strip().lstrip("/").replace("\\", "/")
if not stripped:
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
if stripped == _VIRTUAL_USER_DATA_PREFIX or stripped.startswith(_VIRTUAL_USER_DATA_PREFIX + "/"):
return stripped
if "/" in stripped or stripped in {".", ".."}:
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
return f"{_VIRTUAL_USER_DATA_PREFIX}/outputs/{stripped}"
class ArtifactUpdateRequest(BaseModel):
content: str
def _build_content_disposition(disposition_type: str, filename: str) -> str:
"""Build an RFC 5987 encoded Content-Disposition header value."""
return f"{disposition_type}; filename*=UTF-8''{quote(filename)}"
def _build_attachment_headers(filename: str, extra_headers: dict[str, str] | None = None) -> dict[str, str]:
headers = {"Content-Disposition": _build_content_disposition("attachment", filename)}
if extra_headers:
headers.update(extra_headers)
return headers
def is_text_file_by_content(path: Path, sample_size: int = 8192) -> bool:
"""Check if file is text by examining content for null bytes."""
try:
with open(path, "rb") as f:
chunk = f.read(sample_size)
# Text files shouldn't contain null bytes
return b"\x00" not in chunk
except Exception:
return False
def _extract_file_from_skill_archive(zip_path: Path, internal_path: str) -> bytes | None:
"""Extract a file from a .skill ZIP archive.
Args:
zip_path: Path to the .skill file (ZIP archive).
internal_path: Path to the file inside the archive (e.g., "SKILL.md").
Returns:
The file content as bytes, or None if not found.
"""
if not zipfile.is_zipfile(zip_path):
return None
try:
with zipfile.ZipFile(zip_path, "r") as zip_ref:
# List all files in the archive
namelist = zip_ref.namelist()
# Try direct path first
if internal_path in namelist:
return zip_ref.read(internal_path)
# Try with any top-level directory prefix (e.g., "skill-name/SKILL.md")
for name in namelist:
if name.endswith("/" + internal_path) or name == internal_path:
return zip_ref.read(name)
# Not found
return None
except (zipfile.BadZipFile, KeyError):
return None
def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
"""Walk an outputs directory and return artifact listing entries.
Each entry has ``path`` (virtual ``mnt/user-data/outputs/...``), ``name``,
``size_bytes``, ``mime_type``, and ``modified_at`` (ISO 8601 UTC). Symlinks
that escape the outputs root are dropped to keep the listing scoped to the
thread. The result is sorted by ``modified_at`` descending (newest first).
"""
if not outputs_dir.exists() or not outputs_dir.is_dir():
return []
outputs_root = outputs_dir.resolve()
files: list[dict] = []
for entry in outputs_root.rglob("*"):
if not entry.is_file():
continue
try:
resolved = entry.resolve()
resolved.relative_to(outputs_root)
except (OSError, ValueError):
continue
relative = resolved.relative_to(outputs_root).as_posix()
mime_type, _ = mimetypes.guess_type(resolved.name)
stat = resolved.stat()
files.append(
{
"path": f"mnt/user-data/outputs/{relative}",
"name": resolved.name,
"size_bytes": stat.st_size,
"mime_type": mime_type,
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat(),
}
)
files.sort(key=lambda f: f["modified_at"], reverse=True)
return files
@router.get(
"/threads/{thread_id}/artifacts",
summary="List Artifact Files",
description="List all files under the thread's outputs directory (/mnt/user-data/outputs/). Returns an empty list if the thread has no outputs.",
)
@require_permission("threads", "read", owner_check=True)
async def list_artifacts(thread_id: str, request: Request) -> dict:
"""列出 thread 在 ``outputs/`` 下生成的产物文件。
圆桌 Step 3(``HighFidelityReport``)对每个**子席位** thread_id 调用本接口,
展示各智能体写入沙箱 ``mnt/user-data/outputs/`` 的文件;协调智能体 thread
通常无独立交付文件。
遍历路径::
.deer-flow/users/{user_id}/threads/{thread_id}/user-data/outputs/
返回 ``{ "thread_id", "files": [{ path, name, size_bytes, mime_type, modified_at }] }``,
目录不存在时 ``files`` 为空数组(非 404)。
"""
outputs_dir = get_paths().sandbox_outputs_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
return {"thread_id": thread_id, "files": collect_outputs_listing(outputs_dir)}
@router.get(
"/threads/{thread_id}/artifacts/{path:path}",
summary="Get Artifact File",
description="Retrieve an artifact file generated by the AI agent. Text and binary files can be viewed inline, while active web content is always downloaded.",
)
@require_permission("threads", "read", owner_check=True)
async def get_artifact(thread_id: str, path: str, request: Request, download: bool = False) -> Response:
"""Get an artifact file by its path.
The endpoint automatically detects file types and returns appropriate content types.
Use the `download` query parameter to force file download for non-active content.
Args:
thread_id: The thread ID.
path: The artifact path with virtual prefix (e.g., mnt/user-data/outputs/file.txt).
request: FastAPI request object (automatically injected).
Returns:
The file content as a FileResponse with appropriate content type:
- Active content (HTML/XHTML/SVG): Served as download attachment
- Text files: Plain text with proper MIME type
- Binary files: Inline display with download option
Raises:
HTTPException:
- 400 if path is invalid or not a file
- 403 if access denied (path traversal detected)
- 404 if file not found
Query Parameters:
download (bool): If true, forces attachment download for file types that are
otherwise returned inline or as plain text. Active HTML/XHTML/SVG content
is always downloaded regardless of this flag.
Example:
- Get text file inline: `/api/threads/abc123/artifacts/mnt/user-data/outputs/notes.txt`
- Download file: `/api/threads/abc123/artifacts/mnt/user-data/outputs/data.csv?download=true`
- Active web content such as `.html`, `.xhtml`, and `.svg` artifacts is always downloaded
"""
path = coerce_artifact_virtual_path(path)
# Check if this is a request for a file inside a .skill archive (e.g., xxx.skill/SKILL.md)
if ".skill/" in path:
# Split the path at ".skill/" to get the ZIP file path and internal path
skill_marker = ".skill/"
marker_pos = path.find(skill_marker)
skill_file_path = path[: marker_pos + len(".skill")] # e.g., "mnt/user-data/outputs/my-skill.skill"
internal_path = path[marker_pos + len(skill_marker) :] # e.g., "SKILL.md"
actual_skill_path = await aresolve_thread_virtual_path(thread_id, skill_file_path)
if not actual_skill_path.exists():
raise HTTPException(status_code=404, detail=f"Skill file not found: {skill_file_path}")
if not actual_skill_path.is_file():
raise HTTPException(status_code=400, detail=f"Path is not a file: {skill_file_path}")
# Extract the file from the .skill archive
content = _extract_file_from_skill_archive(actual_skill_path, internal_path)
if content is None:
raise HTTPException(status_code=404, detail=f"File '{internal_path}' not found in skill archive")
# Determine MIME type based on the internal file
mime_type, _ = mimetypes.guess_type(internal_path)
# Add cache headers to avoid repeated ZIP extraction (cache for 5 minutes)
cache_headers = {"Cache-Control": "private, max-age=300"}
download_name = Path(internal_path).name or actual_skill_path.stem
if download or mime_type in ACTIVE_CONTENT_MIME_TYPES:
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=_build_attachment_headers(download_name, cache_headers))
if mime_type and mime_type.startswith("text/"):
return PlainTextResponse(content=content.decode("utf-8"), media_type=mime_type, headers=cache_headers)
# Default to plain text for unknown types that look like text
try:
return PlainTextResponse(content=content.decode("utf-8"), media_type="text/plain", headers=cache_headers)
except UnicodeDecodeError:
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=cache_headers)
actual_path = await aresolve_thread_virtual_path(thread_id, path)
logger.info(f"Resolving artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
if not actual_path.exists():
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
if not actual_path.is_file():
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
mime_type, _ = mimetypes.guess_type(actual_path)
if download:
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
# Always force download for active content types to prevent script execution
# in the application origin when users open generated artifacts.
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
if mime_type and mime_type.startswith("text/"):
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
if is_text_file_by_content(actual_path):
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
return Response(content=actual_path.read_bytes(), media_type=mime_type, headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)})
@router.patch(
"/threads/{thread_id}/artifacts/{path:path}",
summary="Update Artifact File",
description="Update a text artifact file in the current thread workspace.",
)
@require_permission("threads", "write", owner_check=True)
async def update_artifact(
thread_id: str,
path: str,
request: Request,
body: ArtifactUpdateRequest,
) -> Response:
"""Update a text artifact.
This endpoint is intentionally limited to normal text artifacts. Files inside
.skill archives and active web content remain read-only from the artifact UI.
"""
if ".skill/" in path:
raise HTTPException(status_code=400, detail="Skill archive contents are read-only.")
path = coerce_artifact_virtual_path(path)
actual_path = await aresolve_thread_virtual_path(thread_id, path)
logger.info(f"Updating artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
if not actual_path.exists():
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
if not actual_path.is_file():
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
mime_type, _ = mimetypes.guess_type(actual_path)
suffix = actual_path.suffix.lower()
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
raise HTTPException(status_code=400, detail="Active web artifacts are read-only.")
if suffix not in EDITABLE_TEXT_SUFFIXES and not (mime_type and mime_type.startswith("text/")):
raise HTTPException(status_code=400, detail="Only text artifacts can be edited.")
actual_path.write_text(body.content, encoding="utf-8")
return PlainTextResponse(content=body.content, media_type=mime_type or "text/plain")
import asyncio
import logging
import mimetypes
import time
import zipfile
from collections import OrderedDict
from datetime import UTC, datetime
from pathlib import Path
from typing import Literal
from urllib.parse import quote
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import FileResponse, PlainTextResponse, Response
from pydantic import BaseModel
from app.gateway.authz import require_permission
from app.gateway.path_utils import aresolve_thread_virtual_path
from deerflow.config.paths import get_paths
from deerflow.runtime.thread_paths import aresolve_path_user_id
from deerflow.runtime.user_context import get_effective_user_id
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/api", tags=["artifacts"])
ACTIVE_CONTENT_MIME_TYPES = {
"text/html",
"application/xhtml+xml",
"image/svg+xml",
}
EDITABLE_TEXT_SUFFIXES = {
".md",
".markdown",
".txt",
".json",
".yaml",
".yml",
".csv",
".xml",
".log",
}
_VIRTUAL_USER_DATA_PREFIX = "mnt/user-data"
ArtifactLibraryKind = Literal["all", "document", "spreadsheet", "text_data", "archive", "other"]
ArtifactLibrarySource = Literal["all", "general_qa", "agent"]
_MEDIA_MIME_PREFIXES = ("image/", "audio/", "video/")
_MEDIA_SUFFIXES = {
".aac",
".aiff",
".ape",
".avi",
".bmp",
".flac",
".gif",
".heic",
".ico",
".jpeg",
".jpg",
".m4a",
".m4v",
".mkv",
".mov",
".mp3",
".mp4",
".ogg",
".png",
".svg",
".tiff",
".wav",
".webm",
".webp",
".wma",
}
_DOCUMENT_SUFFIXES = {
".doc",
".docx",
".epub",
".odt",
".pdf",
".ppt",
".pptx",
".rtf",
}
_SPREADSHEET_SUFFIXES = {".csv", ".ods", ".tsv", ".xls", ".xlsx"}
_TEXT_DATA_SUFFIXES = {
".bash",
".c",
".conf",
".cpp",
".css",
".go",
".h",
".html",
".ini",
".java",
".js",
".json",
".jsonl",
".log",
".markdown",
".md",
".mmd",
".py",
".rs",
".sh",
".sql",
".toml",
".ts",
".tsx",
".txt",
".xml",
".yaml",
".yml",
}
_ARCHIVE_SUFFIXES = {".7z", ".gz", ".rar", ".skill", ".tar", ".tgz", ".zip"}
_ARTIFACT_LIBRARY_CACHE_TTL_SECONDS = 15.0
_ARTIFACT_LIBRARY_CACHE_MAX_USERS = 128
_ARTIFACT_LIBRARY_SCAN_CONCURRENCY = 8
_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE = 500
_artifact_library_cache: OrderedDict[str, tuple[float, list[dict]]] = OrderedDict()
def coerce_artifact_virtual_path(path: str) -> str:
"""Normalize an artifacts URL path to a sandbox virtual path.
Streaming write previews may request a bare filename such as
``未命名文件.md`` before the model has emitted a real
``/mnt/user-data/outputs/...`` path. Mapping a single path segment into
the thread outputs directory turns that into a normal 404 (file not written
yet) instead of HTTP 400 ``Path must start with /mnt/user-data``.
"""
stripped = path.strip().lstrip("/").replace("\\", "/")
if not stripped:
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
if stripped == _VIRTUAL_USER_DATA_PREFIX or stripped.startswith(_VIRTUAL_USER_DATA_PREFIX + "/"):
return stripped
if "/" in stripped or stripped in {".", ".."}:
raise HTTPException(status_code=400, detail="Path must start with /mnt/user-data")
return f"{_VIRTUAL_USER_DATA_PREFIX}/outputs/{stripped}"
class ArtifactUpdateRequest(BaseModel):
content: str
class ArtifactLibraryItem(BaseModel):
thread_id: str
thread_title: str
agent_id: str | None = None
agent_name: str | None = None
thread_type: str | None = None
source_type: Literal["general_qa", "agent"]
path: str
name: str
size_bytes: int
mime_type: str | None = None
modified_at: str
file_kind: str
class ArtifactLibraryResponse(BaseModel):
total: int
offset: int
limit: int
items: list[ArtifactLibraryItem]
def _build_content_disposition(disposition_type: str, filename: str) -> str:
"""Build an RFC 5987 encoded Content-Disposition header value."""
return f"{disposition_type}; filename*=UTF-8''{quote(filename)}"
def _build_attachment_headers(filename: str, extra_headers: dict[str, str] | None = None) -> dict[str, str]:
headers = {"Content-Disposition": _build_content_disposition("attachment", filename)}
if extra_headers:
headers.update(extra_headers)
return headers
def is_text_file_by_content(path: Path, sample_size: int = 8192) -> bool:
"""Check if file is text by examining content for null bytes."""
try:
with open(path, "rb") as f:
chunk = f.read(sample_size)
# Text files shouldn't contain null bytes
return b"\x00" not in chunk
except Exception:
return False
def _extract_file_from_skill_archive(zip_path: Path, internal_path: str) -> bytes | None:
"""Extract a file from a .skill ZIP archive.
Args:
zip_path: Path to the .skill file (ZIP archive).
internal_path: Path to the file inside the archive (e.g., "SKILL.md").
Returns:
The file content as bytes, or None if not found.
"""
if not zipfile.is_zipfile(zip_path):
return None
try:
with zipfile.ZipFile(zip_path, "r") as zip_ref:
# List all files in the archive
namelist = zip_ref.namelist()
# Try direct path first
if internal_path in namelist:
return zip_ref.read(internal_path)
# Try with any top-level directory prefix (e.g., "skill-name/SKILL.md")
for name in namelist:
if name.endswith("/" + internal_path) or name == internal_path:
return zip_ref.read(name)
# Not found
return None
except (zipfile.BadZipFile, KeyError):
return None
def collect_outputs_listing(outputs_dir: Path) -> list[dict]:
"""Walk an outputs directory and return artifact listing entries.
Each entry has ``path`` (virtual ``mnt/user-data/outputs/...``), ``name``,
``size_bytes``, ``mime_type``, and ``modified_at`` (ISO 8601 UTC). Symlinks
that escape the outputs root are dropped to keep the listing scoped to the
thread. The result is sorted by ``modified_at`` descending (newest first).
"""
if not outputs_dir.exists() or not outputs_dir.is_dir():
return []
outputs_root = outputs_dir.resolve()
files: list[dict] = []
for entry in outputs_root.rglob("*"):
if not entry.is_file():
continue
try:
resolved = entry.resolve()
resolved.relative_to(outputs_root)
except (OSError, ValueError):
continue
try:
relative = resolved.relative_to(outputs_root).as_posix()
mime_type, _ = mimetypes.guess_type(resolved.name)
stat = resolved.stat()
except OSError:
# A run may atomically replace or remove a file while this directory
# is being scanned. Treat that transient entry as absent instead of
# failing the complete thread/library listing.
continue
files.append(
{
"path": f"mnt/user-data/outputs/{relative}",
"name": resolved.name,
"size_bytes": stat.st_size,
"mime_type": mime_type,
"modified_at": datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat(),
}
)
files.sort(key=lambda f: f["modified_at"], reverse=True)
return files
def artifact_library_kind(name: str) -> str:
"""Classify one non-media artifact for the library filters."""
suffix = Path(name).suffix.lower()
if suffix in _DOCUMENT_SUFFIXES:
return "document"
if suffix in _SPREADSHEET_SUFFIXES:
return "spreadsheet"
if suffix in _TEXT_DATA_SUFFIXES:
return "text_data"
if suffix in _ARCHIVE_SUFFIXES:
return "archive"
return "other"
def is_artifact_library_file(file: dict) -> bool:
"""Return whether a generated file belongs in the document-only library."""
mime_type = str(file.get("mime_type") or "").lower()
if mime_type.startswith(_MEDIA_MIME_PREFIXES):
return False
return Path(str(file.get("name") or file.get("path") or "")).suffix.lower() not in _MEDIA_SUFFIXES
async def _search_artifact_library_threads(thread_store) -> list[dict]:
"""Load every normal conversation owned by the current request user."""
rows: list[dict] = []
seen: set[str] = set()
offset = 0
while True:
batch = await thread_store.search(
limit=_ARTIFACT_LIBRARY_THREAD_BATCH_SIZE,
offset=offset,
exclude_system=True,
)
if not batch:
break
added = 0
for row in batch:
thread_id = str(row.get("thread_id") or "")
if not thread_id or thread_id in seen:
continue
seen.add(thread_id)
rows.append(row)
added += 1
if len(batch) < _ARTIFACT_LIBRARY_THREAD_BATCH_SIZE or added == 0:
break
offset += len(batch)
return rows
async def _scan_artifact_library(thread_store) -> list[dict]:
"""Scan generated outputs for all current-user conversation threads."""
rows = await _search_artifact_library_threads(thread_store)
paths = get_paths()
semaphore = asyncio.Semaphore(_ARTIFACT_LIBRARY_SCAN_CONCURRENCY)
async def scan_thread(row: dict) -> list[dict]:
thread_id = str(row.get("thread_id") or "")
if not thread_id:
return []
async with semaphore:
try:
path_user_id = await aresolve_path_user_id(thread_id)
outputs_dir = paths.sandbox_outputs_dir(thread_id, user_id=path_user_id)
files = await asyncio.to_thread(collect_outputs_listing, outputs_dir)
except Exception:
logger.warning("Failed to scan artifacts for library thread %s", thread_id, exc_info=True)
return []
metadata = row.get("metadata") if isinstance(row.get("metadata"), dict) else {}
agent_id = metadata.get("agent_id")
agent_name = metadata.get("agent_name")
thread_type = metadata.get("thread_type")
normalized_agent_id = agent_id.strip() if isinstance(agent_id, str) else ""
normalized_agent_name = agent_name.strip() if isinstance(agent_name, str) else ""
title = str(row.get("display_name") or "新对话")
items: list[dict] = []
for file in files:
if not is_artifact_library_file(file):
continue
items.append(
{
"thread_id": thread_id,
"thread_title": title,
"agent_id": normalized_agent_id or None,
"agent_name": normalized_agent_name or None,
"thread_type": thread_type if isinstance(thread_type, str) and thread_type else None,
"source_type": "agent" if normalized_agent_id else "general_qa",
**file,
"file_kind": artifact_library_kind(str(file.get("name") or "")),
}
)
return items
nested = await asyncio.gather(*(scan_thread(row) for row in rows))
items = [item for group in nested for item in group]
items.sort(
key=lambda item: (
str(item.get("modified_at") or ""),
str(item.get("thread_id") or ""),
str(item.get("path") or ""),
),
reverse=True,
)
return items
async def _cached_artifact_library(thread_store, *, refresh: bool) -> list[dict]:
user_id = str(get_effective_user_id() or "default")
now = time.monotonic()
cached = _artifact_library_cache.get(user_id)
if not refresh and cached and cached[0] > now:
_artifact_library_cache.move_to_end(user_id)
return cached[1]
items = await _scan_artifact_library(thread_store)
_artifact_library_cache[user_id] = (now + _ARTIFACT_LIBRARY_CACHE_TTL_SECONDS, items)
_artifact_library_cache.move_to_end(user_id)
while len(_artifact_library_cache) > _ARTIFACT_LIBRARY_CACHE_MAX_USERS:
_artifact_library_cache.popitem(last=False)
return items
@router.get(
"/artifact-library",
response_model=ArtifactLibraryResponse,
summary="List Generated Files Across Conversations",
description="Aggregate non-media files generated in the current user's normal conversations.",
)
@require_permission("threads", "read")
async def list_artifact_library(
request: Request,
query: str | None = Query(default=None, max_length=200),
file_kind: ArtifactLibraryKind = Query(default="all"),
source_type: ArtifactLibrarySource = Query(default="all"),
offset: int = Query(default=0, ge=0),
limit: int = Query(default=50, ge=1, le=200),
refresh: bool = Query(default=False),
) -> ArtifactLibraryResponse:
"""Return a flat, newest-first library of generated conversation files.
The filesystem remains the source of truth: every owned, non-system thread's
``outputs`` directory is scanned, including files written without an
explicit ``present_files`` call. Uploads and image/audio/video outputs are
intentionally outside this collection.
"""
from app.gateway.deps import get_agent_store, get_thread_store
items = await _cached_artifact_library(get_thread_store(request), refresh=refresh)
agent_ids = list(dict.fromkeys(str(item.get("agent_id") or "") for item in items if item.get("agent_id")))
agent_names: dict[str, str] = {}
if agent_ids:
try:
agents = await get_agent_store(request).list_by_ids(
agent_ids,
str(get_effective_user_id() or "default"),
)
agent_names = {str(agent.get("id") or ""): str(agent.get("name") or "").strip() for agent in agents if agent.get("id") and agent.get("name")}
except Exception:
logger.warning("Failed to resolve artifact library agent names", exc_info=True)
items_with_sources = [
{
**item,
"agent_name": (agent_names.get(str(item.get("agent_id") or "")) or item.get("agent_name") or item.get("agent_id")),
}
for item in items
]
normalized_query = (query or "").strip().casefold()
filtered = [
item
for item in items_with_sources
if (file_kind == "all" or item["file_kind"] == file_kind)
and (source_type == "all" or item["source_type"] == source_type)
and (not normalized_query or normalized_query in str(item.get("name") or "").casefold() or normalized_query in str(item.get("thread_title") or "").casefold() or normalized_query in str(item.get("agent_name") or "").casefold())
]
page = filtered[offset : offset + limit]
return ArtifactLibraryResponse(
total=len(filtered),
offset=offset,
limit=limit,
items=[ArtifactLibraryItem.model_validate(item) for item in page],
)
@router.get(
"/threads/{thread_id}/artifacts",
summary="List Artifact Files",
description="List all files under the thread's outputs directory (/mnt/user-data/outputs/). Returns an empty list if the thread has no outputs.",
)
@require_permission("threads", "read", owner_check=True)
async def list_artifacts(thread_id: str, request: Request) -> dict:
"""列出 thread 在 ``outputs/`` 下生成的产物文件。
圆桌 Step 3(``HighFidelityReport``)对每个**子席位** thread_id 调用本接口,
展示各智能体写入沙箱 ``mnt/user-data/outputs/`` 的文件;协调智能体 thread
通常无独立交付文件。
遍历路径::
.deer-flow/users/{user_id}/threads/{thread_id}/user-data/outputs/
返回 ``{ "thread_id", "files": [{ path, name, size_bytes, mime_type, modified_at }] }``,
目录不存在时 ``files`` 为空数组(非 404)。
"""
outputs_dir = get_paths().sandbox_outputs_dir(thread_id, user_id=await aresolve_path_user_id(thread_id))
return {"thread_id": thread_id, "files": collect_outputs_listing(outputs_dir)}
@router.get(
"/threads/{thread_id}/artifacts/{path:path}",
summary="Get Artifact File",
description="Retrieve an artifact file generated by the AI agent. Text and binary files can be viewed inline, while active web content is always downloaded.",
)
@require_permission("threads", "read", owner_check=True)
async def get_artifact(thread_id: str, path: str, request: Request, download: bool = False) -> Response:
"""Get an artifact file by its path.
The endpoint automatically detects file types and returns appropriate content types.
Use the `download` query parameter to force file download for non-active content.
Args:
thread_id: The thread ID.
path: The artifact path with virtual prefix (e.g., mnt/user-data/outputs/file.txt).
request: FastAPI request object (automatically injected).
Returns:
The file content as a FileResponse with appropriate content type:
- Active content (HTML/XHTML/SVG): Served as download attachment
- Text files: Plain text with proper MIME type
- Binary files: Inline display with download option
Raises:
HTTPException:
- 400 if path is invalid or not a file
- 403 if access denied (path traversal detected)
- 404 if file not found
Query Parameters:
download (bool): If true, forces attachment download for file types that are
otherwise returned inline or as plain text. Active HTML/XHTML/SVG content
is always downloaded regardless of this flag.
Example:
- Get text file inline: `/api/threads/abc123/artifacts/mnt/user-data/outputs/notes.txt`
- Download file: `/api/threads/abc123/artifacts/mnt/user-data/outputs/data.csv?download=true`
- Active web content such as `.html`, `.xhtml`, and `.svg` artifacts is always downloaded
"""
path = coerce_artifact_virtual_path(path)
# Check if this is a request for a file inside a .skill archive (e.g., xxx.skill/SKILL.md)
if ".skill/" in path:
# Split the path at ".skill/" to get the ZIP file path and internal path
skill_marker = ".skill/"
marker_pos = path.find(skill_marker)
skill_file_path = path[: marker_pos + len(".skill")] # e.g., "mnt/user-data/outputs/my-skill.skill"
internal_path = path[marker_pos + len(skill_marker) :] # e.g., "SKILL.md"
actual_skill_path = await aresolve_thread_virtual_path(thread_id, skill_file_path)
if not actual_skill_path.exists():
raise HTTPException(status_code=404, detail=f"Skill file not found: {skill_file_path}")
if not actual_skill_path.is_file():
raise HTTPException(status_code=400, detail=f"Path is not a file: {skill_file_path}")
# Extract the file from the .skill archive
content = _extract_file_from_skill_archive(actual_skill_path, internal_path)
if content is None:
raise HTTPException(status_code=404, detail=f"File '{internal_path}' not found in skill archive")
# Determine MIME type based on the internal file
mime_type, _ = mimetypes.guess_type(internal_path)
# Add cache headers to avoid repeated ZIP extraction (cache for 5 minutes)
cache_headers = {"Cache-Control": "private, max-age=300"}
download_name = Path(internal_path).name or actual_skill_path.stem
if download or mime_type in ACTIVE_CONTENT_MIME_TYPES:
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=_build_attachment_headers(download_name, cache_headers))
if mime_type and mime_type.startswith("text/"):
return PlainTextResponse(content=content.decode("utf-8"), media_type=mime_type, headers=cache_headers)
# Default to plain text for unknown types that look like text
try:
return PlainTextResponse(content=content.decode("utf-8"), media_type="text/plain", headers=cache_headers)
except UnicodeDecodeError:
return Response(content=content, media_type=mime_type or "application/octet-stream", headers=cache_headers)
actual_path = await aresolve_thread_virtual_path(thread_id, path)
logger.info(f"Resolving artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
if not actual_path.exists():
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
if not actual_path.is_file():
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
mime_type, _ = mimetypes.guess_type(actual_path)
if download:
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
# Always force download for active content types to prevent script execution
# in the application origin when users open generated artifacts.
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
return FileResponse(path=actual_path, filename=actual_path.name, media_type=mime_type, headers=_build_attachment_headers(actual_path.name))
if mime_type and mime_type.startswith("text/"):
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
if is_text_file_by_content(actual_path):
return PlainTextResponse(content=actual_path.read_text(encoding="utf-8"), media_type=mime_type)
return Response(content=actual_path.read_bytes(), media_type=mime_type, headers={"Content-Disposition": _build_content_disposition("inline", actual_path.name)})
@router.patch(
"/threads/{thread_id}/artifacts/{path:path}",
summary="Update Artifact File",
description="Update a text artifact file in the current thread workspace.",
)
@require_permission("threads", "write", owner_check=True)
async def update_artifact(
thread_id: str,
path: str,
request: Request,
body: ArtifactUpdateRequest,
) -> Response:
"""Update a text artifact.
This endpoint is intentionally limited to normal text artifacts. Files inside
.skill archives and active web content remain read-only from the artifact UI.
"""
if ".skill/" in path:
raise HTTPException(status_code=400, detail="Skill archive contents are read-only.")
path = coerce_artifact_virtual_path(path)
actual_path = await aresolve_thread_virtual_path(thread_id, path)
logger.info(f"Updating artifact path: thread_id={thread_id}, requested_path={path}, actual_path={actual_path}")
if not actual_path.exists():
raise HTTPException(status_code=404, detail=f"Artifact not found: {path}")
if not actual_path.is_file():
raise HTTPException(status_code=400, detail=f"Path is not a file: {path}")
mime_type, _ = mimetypes.guess_type(actual_path)
suffix = actual_path.suffix.lower()
if mime_type in ACTIVE_CONTENT_MIME_TYPES:
raise HTTPException(status_code=400, detail="Active web artifacts are read-only.")
if suffix not in EDITABLE_TEXT_SUFFIXES and not (mime_type and mime_type.startswith("text/")):
raise HTTPException(status_code=400, detail="Only text artifacts can be edited.")
actual_path.write_text(body.content, encoding="utf-8")
return PlainTextResponse(content=body.content, media_type=mime_type or "text/plain")
File diff suppressed because it is too large. Load diff
@@ -55,6 +55,7 @@ from deerflow.config.agents_config import load_agent_config
from deerflow.config.llmwiki_config import LlmWikiRuntimeOverride, normalize_service_url
from deerflow.config.system_settings import load_system_settings, save_system_settings
from deerflow.integrations.weknora.client import WeKnoraClient, WeKnoraError
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.persistence.llmwiki import LlmWikiStore
@@ -254,7 +255,10 @@ def _schedule_local_wiki_sync(request: Request, mapping: dict[str, Any]) -> None
if service is not None and config.enabled and config.auto_sync and mapping.get("wiki_index_enabled", True):
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=True))
# Event-driven refresh is incremental. Content hash + embedding
# fingerprint decide which pages need encoding; unchanged Wiki pages
# must retain and reuse their existing vectors.
spawn_local_wiki_index_job(request.app, service.sync_mapping(mapping, force=False))
async def _actor(request: Request) -> tuple[str, bool]:
@@ -496,15 +500,7 @@ def _should_follow_weknora_binary_redirect(path: str) -> bool:
"""Keep protected preview/image bytes inside the DeerFlow proxy."""
segments = _path_segments(path)
return (
len(segments) == 5
and segments[:3] == ["api", "v1", "knowledge"]
and segments[4] == "preview"
) or (
len(segments) == 5
and segments[:3] == ["api", "v1", "knowledge-bases"]
and segments[4] == "files"
)
return (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge"] and segments[4] == "preview") or (len(segments) == 5 and segments[:3] == ["api", "v1", "knowledge-bases"] and segments[4] == "files")
def _forward_request_headers(request: Request) -> dict[str, str]:
@@ -1833,6 +1829,9 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
"knowledge_count": int(source.get("knowledge_count") or 0),
"chunk_count": int(source.get("chunk_count") or 0),
"processing_count": int(source.get("processing_count") or 0),
"share_count": int(source.get("share_count") or 0),
"is_processing": bool(source.get("is_processing") or int(source.get("processing_count") or 0) > 0),
"is_temporary": bool(source.get("is_temporary", False)),
"embedding_model_id": source.get("embedding_model_id"),
"vector_store_id": source.get("vector_store_id"),
"vector_store_name": source.get("vector_store_name"),
@@ -1840,16 +1839,23 @@ def _mapping_view(row: dict[str, Any], remote: dict[str, Any] | None, *, actor_u
"vector_store_engine_type": source.get("vector_store_engine_type"),
"vector_store_status": source.get("vector_store_status"),
"chunking_config": source.get("chunking_config") if isinstance(source.get("chunking_config"), dict) else {},
"image_processing_config": (source.get("image_processing_config") if isinstance(source.get("image_processing_config"), dict) else {}),
"indexing_strategy": source.get("indexing_strategy") if isinstance(source.get("indexing_strategy"), dict) else {},
"capabilities": source.get("capabilities") if isinstance(source.get("capabilities"), dict) else {},
"extract_config": source.get("extract_config") if isinstance(source.get("extract_config"), dict) else {},
"faq_config": source.get("faq_config") if isinstance(source.get("faq_config"), dict) else {},
"wiki_config": source.get("wiki_config") if isinstance(source.get("wiki_config"), dict) else {},
"question_generation_config": (source.get("question_generation_config") if isinstance(source.get("question_generation_config"), dict) else {}),
"auto_tag_config": source.get("auto_tag_config") if isinstance(source.get("auto_tag_config"), dict) else {},
"vlm_config": source.get("vlm_config") if isinstance(source.get("vlm_config"), dict) else {},
"asr_config": source.get("asr_config") if isinstance(source.get("asr_config"), dict) else {},
"summary_model_id": source.get("summary_model_id"),
"storage_provider": (source.get("storage_provider_config", {}).get("provider") if isinstance(source.get("storage_provider_config"), dict) else None),
"created_at": row.get("created_at"),
"updated_at": row.get("updated_at"),
"published_at": row.get("published_at"),
"remote_created_at": source.get("created_at"),
"remote_updated_at": source.get("updated_at"),
}
@@ -1877,11 +1883,7 @@ async def _public_mapping(request: Request, mapping_id: str) -> dict[str, Any]:
write=False,
is_admin=False,
)
if (
row is None
or row.get("publication_status") != "published"
or _is_conversation_deposit_mapping(row)
):
if row is None or row.get("publication_status") != "published" or _is_conversation_deposit_mapping(row):
raise HTTPException(status_code=404, detail="Published knowledge base not found")
return row
@@ -2042,6 +2044,7 @@ def _wiki_page_view(page: dict[str, Any]) -> dict[str, Any] | None:
if not slug and not title and not content and not summary:
return None
return {
"page_metadata": page.get("page_metadata") if isinstance(page.get("page_metadata"), dict) else {},
"id": str(page.get("id") or ""),
"slug": slug,
"title": title,
@@ -2537,24 +2540,260 @@ async def test_llmwiki_connection(request: Request, body: LlmWikiConnectionTest)
return {"success": True, "provider": "weknora", "health": health}
_PUBLIC_WIKI_SUMMARY_CONCURRENCY = 8
_PUBLIC_WIKI_PAGE_SIZE = 500
_PUBLIC_WIKI_PAGE_LIMIT = 100
_GENERATED_PUBLIC_WIKI_INDEX_SLUG = "__index__"
def _is_public_wiki_index_page(page: dict[str, Any]) -> bool:
page_type = str(page.get("page_type") or "").strip().lower()
title = str(page.get("title") or "").strip().lower()
slug_name = str(page.get("slug") or "").strip("/").rsplit("/", 1)[-1].lower()
return page_type == "index" or title in {"索引", "index"} or slug_name in {"index", "home", "readme"}
def _public_wiki_default_page(pages: list[dict[str, Any]]) -> dict[str, Any] | None:
return (
next((page for page in pages if _is_public_wiki_index_page(page)), None)
or next((page for page in pages if str(page.get("page_type") or "").lower() == "overview"), None)
or next(
(page for page in pages if not str(page.get("parent_slug") or "").strip() and int(page.get("depth") or 0) == 0),
None,
)
or (pages[0] if pages else None)
)
def _count_public_wiki_values(pages: list[dict[str, Any]], key: str, fallback: str) -> dict[str, int]:
counts: dict[str, int] = {}
for page in pages:
value = str(page.get(key) or fallback).strip().lower() or fallback
counts[value] = counts.get(value, 0) + 1
return counts
def _public_wiki_time_bound(pages: list[dict[str, Any]], key: str, *, latest: bool) -> str | None:
values = [str(page.get(key) or "").strip() for page in pages]
values = [value for value in values if value]
if not values:
return None
return max(values) if latest else min(values)
async def _all_public_wiki_page_views(
client: WeKnoraClient,
remote_id: str,
) -> tuple[list[dict[str, Any]], int, bool]:
pages_by_slug: dict[str, dict[str, Any]] = {}
remote_total = 0
complete = False
for page_number in range(1, _PUBLIC_WIKI_PAGE_LIMIT + 1):
result = await client.list_wiki_pages(
remote_id,
page=page_number,
page_size=_PUBLIC_WIKI_PAGE_SIZE,
query="",
)
raw_pages = result.get("pages") if isinstance(result, dict) else []
page_items = raw_pages if isinstance(raw_pages, list) else []
for page_item in page_items:
if not isinstance(page_item, dict):
continue
page = _wiki_page_view(page_item)
if page is None:
continue
slug = str(page.get("slug") or "").strip()
pages_by_slug[slug or f"__page_{len(pages_by_slug)}"] = page
reported_total = int(result.get("total") or 0)
remote_total = max(remote_total, reported_total, len(pages_by_slug))
total_pages = max(1, int(result.get("total_pages") or 1))
if not page_items or page_number >= total_pages or (reported_total > 0 and len(pages_by_slug) >= remote_total):
complete = True
break
return list(pages_by_slug.values()), max(remote_total, len(pages_by_slug)), complete
def _public_wiki_summary(
mapping_id: str,
pages: list[dict[str, Any]],
*,
total: int,
complete: bool,
) -> dict[str, Any]:
explicit_index = next((page for page in pages if _is_public_wiki_index_page(page)), None)
default_page = _public_wiki_default_page(pages)
if explicit_index is not None:
index_mode = "explicit"
index_slug: str | None = str(explicit_index.get("slug") or "") or None
elif pages:
index_mode = "generated"
index_slug = _GENERATED_PUBLIC_WIKI_INDEX_SLUG
else:
index_mode = "none"
index_slug = None
default_slug = index_slug or (str(default_page.get("slug") or "") if default_page else None)
route = f"/embed/knowledge/{quote(mapping_id, safe='')}"
if default_slug:
route = f"{route}?{urlencode({'wikiSlug': default_slug})}"
return {
"status": "ready" if pages else "empty",
"page_count": total,
"loaded_page_count": len(pages),
"stats_complete": complete,
"page_type_counts": _count_public_wiki_values(pages, "page_type", "page"),
"page_status_counts": _count_public_wiki_values(pages, "status", "unknown"),
"index_mode": index_mode,
"has_explicit_index": explicit_index is not None,
"index_slug": index_slug,
"default_slug": default_slug,
"created_at": _public_wiki_time_bound(pages, "created_at", latest=False),
"updated_at": _public_wiki_time_bound(pages, "updated_at", latest=True),
"embed_route": route,
}
def _unavailable_public_wiki_summary(mapping_id: str) -> dict[str, Any]:
return {
"status": "unavailable",
"page_count": 0,
"loaded_page_count": 0,
"stats_complete": False,
"page_type_counts": {},
"page_status_counts": {},
"index_mode": "unknown",
"has_explicit_index": False,
"index_slug": None,
"default_slug": None,
"created_at": None,
"updated_at": None,
"embed_route": f"/embed/knowledge/{quote(mapping_id, safe='')}",
}
def _public_local_index_summary(
request: Request,
mapping: dict[str, Any],
state: dict[str, Any] | None,
) -> dict[str, Any]:
app_state = request.app.state
local_config = getattr(
getattr(getattr(app_state, "config", None), "llmwiki", None),
"local_wiki_index",
None,
)
enabled = bool(getattr(local_config, "enabled", False))
current = dict(state or {})
state_name = str(current.get("state") or "not_synced")
if state_name == "idle" and int(current.get("index_revision") or 0) == 0 and not current.get("last_completed_at") and not current.get("last_success_at"):
state_name = "not_synced"
embedding = getattr(app_state, "llmwiki_embedding", None)
fingerprint = embedding.fingerprint if embedding is not None else None
fully_vectorized = bool(mapping.get("wiki_index_enabled", True)) and is_fully_vectorized(
current,
fingerprint,
)
return {
"enabled": enabled,
"wiki_index_enabled": bool(mapping.get("wiki_index_enabled", True)),
"state": state_name,
"fully_vectorized": fully_vectorized,
"retrieval_mode": "vector" if fully_vectorized else "wiki_api",
"remote_page_count": int(current.get("remote_page_count") or 0),
"local_page_count": int(current.get("local_page_count") or 0),
"ready_page_count": int(current.get("ready_page_count") or 0),
"failed_page_count": int(current.get("failed_page_count") or 0),
"vector_count": int(current.get("vector_count") or 0),
"last_started_at": current.get("last_started_at"),
"last_completed_at": current.get("last_completed_at"),
"last_success_at": current.get("last_success_at"),
"updated_at": current.get("updated_at"),
}
@router.get("/api/public/llmwiki/knowledge-bases")
async def list_public_knowledge_bases(request: Request) -> dict[str, Any]:
"""List the minimal identity of every knowledge base published as a public Wiki."""
"""List live, published Wiki libraries with safe metadata and index health."""
rows = await _store(request).list_visible(
"__public_wiki__",
scope="public",
is_admin=False,
)
knowledge_bases = [
{
"id": str(row["id"]),
"name": str(row.get("name") or ""),
}
for row in rows
if row.get("publication_status") == "published"
and not _is_conversation_deposit_mapping(row)
rows = [
row
for row in await _store(request).list_visible(
"__public_wiki__",
scope="public",
is_admin=False,
)
if row.get("publication_status") == "published" and not _is_conversation_deposit_mapping(row)
]
client = _client_or_503()
try:
remote_items = await client.list_knowledge_bases()
except WeKnoraError as exc:
_raise_upstream(exc)
remote_by_id = {str(item.get("id")): item for item in remote_items if isinstance(item, dict) and item.get("id")}
# A successful upstream list is authoritative: stale DeerFlow mappings are
# omitted so discovery never advertises a Wiki that its detail route cannot open.
rows = [row for row in rows if str(row.get("weknora_id") or "") in remote_by_id]
mapping_ids = [str(row["id"]) for row in rows]
index_states: dict[str, dict[str, Any]] = {}
index_store = getattr(request.app.state, "llmwiki_index_store", None)
if index_store is not None and mapping_ids:
try:
index_states = {str(state.get("knowledge_base_mapping_id") or ""): state for state in await index_store.list_index_status(mapping_ids) if isinstance(state, dict)}
except Exception:
logger.warning("Failed to read public Wiki local-index summaries", exc_info=True)
semaphore = asyncio.Semaphore(_PUBLIC_WIKI_SUMMARY_CONCURRENCY)
async def build_summary(row: dict[str, Any]) -> dict[str, Any]:
mapping_id = str(row["id"])
remote_id = str(row["weknora_id"])
remote = remote_by_id[remote_id]
async with semaphore:
try:
remote = {**remote, **await client.get_knowledge_base(remote_id)}
except WeKnoraError as exc:
if exc.status_code == 404:
return {}
logger.warning(
"Failed to load public Wiki detail metadata: mapping_id=%s",
mapping_id,
exc_info=True,
)
try:
pages, page_total, complete = await _all_public_wiki_page_views(client, remote_id)
wiki = _public_wiki_summary(
mapping_id,
pages,
total=page_total,
complete=complete,
)
except WeKnoraError as exc:
logger.warning(
"Failed to load public Wiki page summary: mapping_id=%s",
mapping_id,
exc_info=True,
)
wiki = (
_public_wiki_summary(mapping_id, [], total=0, complete=True)
if exc.status_code == 404
else _unavailable_public_wiki_summary(mapping_id)
)
return {
**_mapping_view(
row,
remote,
actor_user_id="__public_wiki__",
is_admin=False,
),
"wiki": wiki,
"local_index": _public_local_index_summary(
request,
row,
index_states.get(mapping_id),
),
}
knowledge_bases = [summary for summary in await asyncio.gather(*(build_summary(row) for row in rows)) if summary]
return {"knowledge_bases": knowledge_bases, "total": len(knowledge_bases)}
@@ -2596,13 +2835,7 @@ async def list_public_wiki_pages(
except WeKnoraError as exc:
_raise_upstream(exc)
raw_pages = result.get("pages") if isinstance(result, dict) else []
pages = [
view
for page_item in (raw_pages if isinstance(raw_pages, list) else [])
if isinstance(page_item, dict)
for view in [_wiki_page_view(page_item)]
if view is not None
]
pages = [view for page_item in (raw_pages if isinstance(raw_pages, list) else []) if isinstance(page_item, dict) for view in [_wiki_page_view(page_item)] if view is not None]
return {**result, "pages": pages}
@@ -2650,11 +2883,7 @@ def _public_wiki_index_item(value: Any) -> dict[str, Any] | None:
"slug": slug,
"title": str(value.get("title") or slug),
"summary": str(value.get("summary") or ""),
"category_path": (
[str(item) for item in value.get("category_path", []) if str(item).strip()]
if isinstance(value.get("category_path"), list)
else []
),
"category_path": ([str(item) for item in value.get("category_path", []) if str(item).strip()] if isinstance(value.get("category_path"), list) else []),
"wiki_path": str(value.get("wiki_path") or ""),
"depth": int(value.get("depth") or 0),
}
@@ -2680,20 +2909,12 @@ async def _all_public_wiki_index(client: WeKnoraClient, remote_id: str) -> dict[
version = max(version, int(payload.get("version") or 0))
raw_groups = payload.get("groups") if isinstance(payload.get("groups"), list) else []
group = next(
(
value
for value in raw_groups
if isinstance(value, dict) and value.get("type") == page_type
),
(value for value in raw_groups if isinstance(value, dict) and value.get("type") == page_type),
{},
)
total = max(total, int(group.get("total") or 0))
raw_items = group.get("items") if isinstance(group.get("items"), list) else []
items.extend(
item
for value in raw_items
if (item := _public_wiki_index_item(value)) is not None
)
items.extend(item for value in raw_items if (item := _public_wiki_index_item(value)) is not None)
next_cursor = str(group.get("next_cursor") or "")
if not next_cursor or next_cursor in seen_cursors:
break
@@ -2770,12 +2991,11 @@ async def list_knowledge_bases(
knowledge_bases: list[dict[str, Any]] = []
for row in rows:
remote_item = remote.get(str(row["weknora_id"]))
row, remote_item = await _sync_conversation_deposit_view(_store(request), client, row, remote_item)
# Owners need to see stale personal mappings so they can clean them up.
# Public/selector views must never expose a resource that no longer exists
# in WeKnora.
if remote_item is None and scope != "personal":
# A successful remote list is authoritative for every scope. Keep the
# mapping for existing references, but never display a deleted base.
if remote_item is None:
continue
row, remote_item = await _sync_conversation_deposit_view(_store(request), client, row, remote_item)
knowledge_bases.append(_mapping_view(row, remote_item, actor_user_id=user_id, is_admin=is_admin))
return {"knowledge_bases": knowledge_bases}
@@ -2841,6 +3061,7 @@ async def create_knowledge_base(request: Request, body: KnowledgeBaseCreate) ->
except Exception:
logger.exception("Failed to compensate orphan WeKnora knowledge base")
raise HTTPException(status_code=500, detail="Failed to register the knowledge base") from None
_schedule_local_wiki_sync(request, row)
return _mapping_view(row, remote, actor_user_id=user_id, is_admin=is_admin)
@@ -2860,13 +3081,19 @@ async def get_knowledge_base(request: Request, mapping_id: str) -> dict[str, Any
async def update_knowledge_base(request: Request, mapping_id: str, body: KnowledgeBaseUpdate) -> dict[str, Any]:
row, user_id, is_admin = await _authorized_mapping(request, mapping_id, write=True)
changes = body.model_dump(exclude_none=True)
if "name" in changes:
changes["name"] = changes["name"].strip()
if not changes["name"]:
raise HTTPException(status_code=422, detail="知识库名称不能为空")
if changes["name"] == CONVERSATION_DEPOSIT_KB_NAME:
raise HTTPException(status_code=409, detail="对话沉淀是 cmzs 系统保留知识库")
if not changes:
return _mapping_view(row, None, actor_user_id=user_id, is_admin=is_admin)
try:
remote = await _client_or_503().update_knowledge_base(str(row["weknora_id"]), changes)
except WeKnoraError as exc:
_raise_upstream(exc)
updated = await _store(request).update_mapping(mapping_id, name=body.name, description=body.description)
updated = await _store(request).update_mapping(mapping_id, name=changes.get("name"), description=changes.get("description"))
return _mapping_view(updated or row, remote, actor_user_id=user_id, is_admin=is_admin)
@@ -3122,12 +3349,14 @@ async def upload_document(request: Request, mapping_id: str, file: UploadFile =
if not file.filename:
raise HTTPException(status_code=422, detail="File name is required")
try:
return await _client_or_503().upload_document(
result = await _client_or_503().upload_document(
str(row["weknora_id"]),
filename=file.filename,
content=content,
content_type=file.content_type or "application/octet-stream",
)
_schedule_local_wiki_sync(request, row)
return result
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -3139,7 +3368,9 @@ async def import_document_url(request: Request, mapping_id: str, body: Knowledge
if not url.lower().startswith(("http://", "https://")):
raise HTTPException(status_code=422, detail="URL must start with http:// or https://")
try:
return await _client_or_503().import_document_url(str(row["weknora_id"]), url=url)
result = await _client_or_503().import_document_url(str(row["weknora_id"]), url=url)
_schedule_local_wiki_sync(request, row)
return result
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -3148,11 +3379,13 @@ async def import_document_url(request: Request, mapping_id: str, body: Knowledge
async def create_manual_document(request: Request, mapping_id: str, body: ManualKnowledgeCreate) -> dict[str, Any]:
row, _, _ = await _authorized_mapping(request, mapping_id, write=True)
try:
return await _client_or_503().create_manual_document(
result = await _client_or_503().create_manual_document(
str(row["weknora_id"]),
title=body.title.strip(),
content=body.content,
)
_schedule_local_wiki_sync(request, row)
return result
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -3164,6 +3397,7 @@ async def delete_document(request: Request, mapping_id: str, document_id: str) -
try:
await _verify_document(client, row, document_id)
await client.delete_document(document_id)
_schedule_local_wiki_sync(request, row)
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -3174,7 +3408,9 @@ async def reprocess_document(request: Request, mapping_id: str, document_id: str
client = _client_or_503()
try:
await _verify_document(client, row, document_id)
return await client.reprocess_document(document_id)
result = await client.reprocess_document(document_id)
_schedule_local_wiki_sync(request, row)
return result
except WeKnoraError as exc:
_raise_upstream(exc)
@@ -3201,12 +3437,7 @@ async def search_llmwiki(request: Request, body: SearchRequest) -> dict[str, Any
except WikiIndexError as exc:
raise HTTPException(status_code=exc.status_code, detail={"code": exc.code, "message": str(exc)}) from None
mapping_by_id = {str(row["id"]): row for row in rows}
result["results"] = [
_sanitize_conversation_deposit_value(item)
if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {}))
else item
for item in result.get("results") or []
]
result["results"] = [_sanitize_conversation_deposit_value(item) if _is_conversation_deposit_mapping(mapping_by_id.get(str(item.get("knowledge_base_id") or ""), {})) else item for item in result.get("results") or []]
return result
@@ -2,15 +2,20 @@
from __future__ import annotations
import asyncio
import logging
import shutil
from pathlib import Path, PurePosixPath
from typing import Any
from uuid import uuid4
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi import APIRouter, File, Form, HTTPException, Query, Request, UploadFile
from pydantic import BaseModel, Field
from app.gateway.deps import get_optional_user_from_request
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
from deerflow.config.runtime_paths import resolve_path, runtime_home
from deerflow.config.system_settings import load_system_settings, save_system_settings
from deerflow.integrations.weknora.local_index.retrieval import is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError
@@ -37,6 +42,10 @@ class WikiIndexSettingsRequest(BaseModel):
external_search_enabled: bool | None = None
class LocalEncoderConfigRequest(BaseModel):
model_path: str = Field(min_length=1, max_length=4096)
async def _actor(request: Request) -> tuple[str, bool]:
user = await get_optional_user_from_request(request)
if user is None:
@@ -55,6 +64,149 @@ def _services(request: Request):
return config, search, index_store
def _local_encoder(request: Request):
config = request.app.state.config.llmwiki.local_wiki_index.embedding
embedding = getattr(request.app.state, "llmwiki_embedding", None)
if config.provider != "local":
raise HTTPException(status_code=409, detail={"code": "EMBEDDING_NOT_LOCAL", "message": "当前 embedding provider 不是 local"})
if embedding is None:
raise HTTPException(status_code=503, detail={"code": "LOCAL_ENCODER_UNAVAILABLE", "message": "本地编码器未配置或未启用"})
return embedding
def _validated_encoder_path(raw_path: str) -> Path:
path = resolve_path(raw_path.strip())
if not path.is_dir():
raise HTTPException(status_code=422, detail="编码器目录不存在")
if not ((path / "onnx" / "model.onnx").is_file() or (path / "model.onnx").is_file()):
raise HTTPException(status_code=422, detail="编码器目录缺少 onnx/model.onnx 或 model.onnx")
return path
def _safe_relative_upload_path(value: str) -> Path:
normalized = str(value or "").replace("\\", "/").strip("/")
candidate = PurePosixPath(normalized)
if not normalized or candidate.is_absolute() or ".." in candidate.parts:
raise HTTPException(status_code=422, detail=f"非法模型文件路径:{value}")
if len(normalized) > 512:
raise HTTPException(status_code=422, detail="模型文件路径过长")
return Path(*candidate.parts)
def _copy_encoder_uploads(files: list[UploadFile], paths: list[str], staging: Path) -> int:
total = 0
for upload, raw_path in zip(files, paths, strict=True):
relative = _safe_relative_upload_path(raw_path)
destination = staging / relative
destination.parent.mkdir(parents=True, exist_ok=True)
upload.file.seek(0)
with destination.open("wb") as output:
while chunk := upload.file.read(4 * 1024 * 1024):
output.write(chunk)
total += len(chunk)
return total
def _activate_uploaded_encoder(staging: Path) -> Path:
model_root = (runtime_home() / "models").resolve()
model_root.mkdir(parents=True, exist_ok=True)
active = (model_root / "bge-m3").resolve()
if active.parent != model_root:
raise RuntimeError("编码器目标目录越界")
backup = (model_root / f".bge-m3-backup-{uuid4().hex}").resolve()
if active.exists():
active.replace(backup)
try:
staging.replace(active)
except Exception:
if backup.exists() and not active.exists():
backup.replace(active)
raise
if backup.exists():
shutil.rmtree(backup)
return active
async def _save_encoder_path(request: Request, path: Path) -> dict[str, object]:
embedding = _local_encoder(request)
current = load_system_settings()
encoder_settings = current.local_wiki_encoder.model_copy(update={"model_path": str(path)})
save_system_settings(current.model_copy(update={"local_wiki_encoder": encoder_settings}))
await embedding.reset_local_encoder(model_path=str(path))
return embedding.local_encoder_status()
@router.get("/local-encoder/status")
async def local_encoder_status(request: Request) -> dict[str, object]:
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
return _local_encoder(request).local_encoder_status()
@router.post("/local-encoder/start", status_code=202)
async def start_local_encoder(request: Request) -> dict[str, object]:
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
try:
return _local_encoder(request).start_local_encoder()
except ValueError as exc:
raise HTTPException(status_code=409, detail={"code": "LOCAL_ENCODER_START_REJECTED", "message": str(exc)}) from None
@router.put("/local-encoder/config")
async def configure_local_encoder(request: Request, body: LocalEncoderConfigRequest) -> dict[str, object]:
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
return await _save_encoder_path(request, _validated_encoder_path(body.model_path))
@router.post("/local-encoder/upload")
async def upload_local_encoder(
request: Request,
files: list[UploadFile] = File(...),
paths: list[str] = Form(...),
) -> dict[str, object]:
"""Stream an offline model directory into runtime storage and activate it.
Starlette spools multipart files to disk, and the copy loop is chunked, so
multi-gigabyte ONNX models do not need to be loaded into Gateway memory.
"""
_, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="Administrator access required")
if not files or len(files) != len(paths):
raise HTTPException(status_code=422, detail="模型文件与相对路径不匹配")
if len(files) > 500:
raise HTTPException(status_code=422, detail="模型目录文件数不能超过 500")
staging_root = (runtime_home() / "model-uploads").resolve()
staging = (staging_root / uuid4().hex).resolve()
if staging.parent != staging_root:
raise HTTPException(status_code=500, detail="模型暂存目录越界")
staging.mkdir(parents=True, exist_ok=False)
try:
uploaded_bytes = await asyncio.to_thread(_copy_encoder_uploads, files, paths, staging)
_validated_encoder_path(str(staging))
await _local_encoder(request).reset_local_encoder()
active = await asyncio.to_thread(_activate_uploaded_encoder, staging)
status = await _save_encoder_path(request, active)
return {**status, "uploaded_files": len(files), "uploaded_bytes": uploaded_bytes}
except HTTPException:
raise
except OSError as exc:
logger.exception("Failed to store uploaded local encoder")
raise HTTPException(status_code=507, detail=f"编码器文件保存失败:{exc}") from None
finally:
for upload in files:
await upload.close()
if staging.exists():
await asyncio.to_thread(shutil.rmtree, staging, True)
async def _authorized_mappings(request: Request, mapping_ids: list[str]) -> tuple[list[dict[str, Any]], str, bool]:
user_id, is_admin = await _actor(request)
mapping_store = request.app.state.llmwiki_store
@@ -313,7 +465,7 @@ async def vectorize_knowledge_base(request: Request, mapping_id: str) -> dict[st
if not mapping.get("wiki_index_enabled", True):
mapping = await mapping_store.update_mapping(mapping_id, wiki_index_enabled=True)
job_id = str(uuid4())
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=True))
spawn_local_wiki_index_job(request.app, _run_one_sync(request.app, mapping, force=False))
return {"job_id": job_id, "status": "accepted", "knowledge_base_id": mapping_id}
@@ -0,0 +1,400 @@
"""User-facing skill distillation APIs plus administrator maintenance APIs."""
from __future__ import annotations
import asyncio
from typing import Any, Literal
from croniter import croniter
from fastapi import APIRouter, HTTPException, Query, Request
from pydantic import BaseModel, Field, field_validator
from app.gateway.deps import get_optional_user_from_request
from app.gateway.skill_knowledge_job_executor import _skill_directory
from deerflow.config.system_settings import (
SkillKnowledgeAutoSyncSettings,
load_system_settings,
save_system_settings,
)
from deerflow.integrations.weknora.runtime import (
build_weknora_client,
get_resolved_llmwiki_runtime,
)
from deerflow.skill_knowledge import scan_skill
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.types import SkillCategory
router = APIRouter(prefix="/api/skill-knowledge", tags=["skill-knowledge"])
async def _actor(request: Request) -> tuple[str, bool]:
user = await get_optional_user_from_request(request)
if user is None:
return "system", True
return str(user.id), getattr(user, "system_role", None) == "admin"
async def _require_admin(request: Request) -> str:
actor, is_admin = await _actor(request)
if not is_admin:
raise HTTPException(status_code=403, detail="技能知识归纳仅管理员可操作")
return actor
def _owned_by_actor(row: dict[str, Any] | None, actor: str, is_admin: bool) -> bool:
return bool(row and (is_admin or str(row.get("created_by") or "") == actor))
def _store(request: Request):
value = getattr(request.app.state, "skill_knowledge_store", None)
if value is None:
raise HTTPException(status_code=503, detail="技能知识归纳存储不可用")
return value
class SyncTarget(BaseModel):
target_type: Literal["weknora", "assistant"]
target_mode: Literal["wiki", "document", "faq", "conversation_archive", "readonly_sidecar", "assistant"]
target_id: str = Field(min_length=1, max_length=128)
target_name: str = Field(default="", max_length=255)
remote_business_key: str | None = Field(default=None, max_length=512)
readonly: bool = False
class CreateSyncJobRequest(BaseModel):
skill_names: list[str] = Field(min_length=1, max_length=200)
targets: list[SyncTarget] = Field(min_length=1, max_length=1)
force: bool = False
review_mode: Literal["required", "auto_high_confidence", "off"] = "off"
@field_validator("skill_names")
@classmethod
def validate_names(cls, values: list[str]) -> list[str]:
result = list(dict.fromkeys(value.strip() for value in values if value.strip()))
if not result:
raise ValueError("At least one skill is required")
return result
class ReviewRequest(BaseModel):
review_ids: list[str] = Field(min_length=1, max_length=500)
action: Literal["approve", "reject"]
edits: dict[str, dict[str, Any]] = Field(default_factory=dict)
reason: str | None = Field(default=None, max_length=1024)
class DetachRequest(BaseModel):
delete_contributions: bool
class AutoSyncSettingsUpdate(BaseModel):
watch_enabled: bool | None = None
schedule_enabled: bool | None = None
schedule_cron: str | None = Field(default=None, min_length=5, max_length=128)
debounce_seconds: int | None = Field(default=None, ge=10, le=86_400)
write_remote_enabled: bool | None = None
review_mode: Literal["required", "auto_high_confidence", "off"] | None = None
@field_validator("schedule_cron")
@classmethod
def validate_cron(cls, value: str | None) -> str | None:
if value is not None and not croniter.is_valid(value):
raise ValueError("无效的 cron 表达式")
return value
@router.get("/bindings")
async def list_bindings(
request: Request,
skill_names: list[str] | None = Query(default=None),
target_id: str | None = Query(default=None),
include_detached: bool = Query(default=False),
mine_only: bool = Query(default=False),
) -> dict[str, Any]:
actor, is_admin = await _actor(request)
rows = await _store(request).list_bindings(
skill_names=skill_names,
target_id=target_id,
include_detached=include_detached,
created_by=actor if mine_only or not is_admin else None,
)
return {"bindings": rows}
@router.get("/source-status")
async def source_status(request: Request, skill_names: list[str] = Query(...)) -> dict[str, Any]:
await _require_admin(request)
bindings = await _store(request).list_bindings(skill_names=skill_names)
by_skill: dict[str, dict[str, Any]] = {}
for name in list(dict.fromkeys(skill_names)):
try:
root = _skill_directory(request.app.state.config, name)
result = await asyncio.to_thread(scan_skill, name, root)
by_skill[name] = {"source_digest": result.source_digest, "counts": result.counts, "blocked_files": result.blocked_files}
except Exception as exc: # noqa: BLE001
by_skill[name] = {"error": str(exc)}
for binding in bindings:
status = by_skill.get(binding["skill_name"])
if status and status.get("source_digest") and status["source_digest"] != binding.get("synced_digest"):
binding["computed_status"] = "stale"
return {"skills": by_skill, "bindings": bindings}
@router.get("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
async def get_auto_sync_settings(request: Request) -> SkillKnowledgeAutoSyncSettings:
await _require_admin(request)
return load_system_settings().skill_knowledge_auto_sync
@router.put("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
async def update_auto_sync_settings(request: Request, body: AutoSyncSettingsUpdate) -> SkillKnowledgeAutoSyncSettings:
await _require_admin(request)
current = load_system_settings()
updated = current.skill_knowledge_auto_sync.model_copy(update=body.model_dump(exclude_unset=True))
saved = save_system_settings(current.model_copy(update={"skill_knowledge_auto_sync": updated}))
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
if service is not None:
service.nudge()
return saved.skill_knowledge_auto_sync
async def _set_binding_auto_sync(request: Request, binding_id: str, *, enabled: bool) -> dict[str, Any]:
await _require_admin(request)
row = await _store(request).set_binding_auto_sync(binding_id, enabled=enabled)
if row is None:
raise HTTPException(status_code=404, detail="绑定不存在")
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
if service is not None:
service.nudge()
return row
@router.post("/bindings/{binding_id}/enable-auto-sync")
async def enable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
return await _set_binding_auto_sync(request, binding_id, enabled=True)
@router.post("/bindings/{binding_id}/disable-auto-sync")
async def disable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
return await _set_binding_auto_sync(request, binding_id, enabled=False)
@router.post("/sync-jobs", status_code=202)
async def create_sync_job(request: Request, body: CreateSyncJobRequest) -> dict[str, Any]:
actor, is_admin = await _actor(request)
# Built-in and legacy skills need not have an ownership row. Validate
# against the same running filesystem used by the skills list/executor.
missing = []
favorite_names: set[str] = set()
custom_root: Any = None
if not is_admin:
favorite_names = set(await request.app.state.skill_store.list_favorite_skill_names(actor))
storage_root = get_or_new_skill_storage(
app_config=request.app.state.config
).get_skills_root_path().resolve()
custom_root = (storage_root / SkillCategory.CUSTOM.value).resolve()
for name in body.skill_names:
try:
skill_path = await asyncio.to_thread(_skill_directory, request.app.state.config, name)
if not is_admin:
visible = await request.app.state.skill_store.get_visible(name, actor)
is_public_filesystem_skill = not skill_path.resolve().is_relative_to(custom_root)
if visible is None and name not in favorite_names and not is_public_filesystem_skill:
missing.append(name)
except (FileNotFoundError, ValueError):
missing.append(name)
if missing:
raise HTTPException(status_code=404, detail=f"技能不存在:{', '.join(missing[:10])}")
targets: list[dict[str, Any]] = []
for target in body.targets:
data = target.model_dump()
if target.target_type != "weknora":
raise HTTPException(status_code=422, detail="不支持技能直接归纳到助手知识库,请先归纳到 WeKnora 普通知识库")
if target.target_mode != "wiki":
raise HTTPException(status_code=422, detail="技能归纳目标仅支持 WeKnora Wiki 知识库")
mapping = await request.app.state.llmwiki_store.get_authorized(
target.target_id,
actor,
write=True,
is_admin=is_admin,
)
if mapping is None:
raise HTTPException(status_code=404, detail=f"普通知识库不存在或当前用户不可写:{target.target_id}")
data["target_name"] = target.target_name or str(mapping.get("name") or target.target_id)
data["remote_business_key"] = target.remote_business_key or f"weknora:kb:{mapping['weknora_id']}"
targets.append(data)
job = await _store(request).create_job(
skill_names=body.skill_names,
targets=targets,
created_by=actor,
force=body.force,
review_mode=body.review_mode if is_admin else "off",
)
dispatcher = getattr(request.app.state, "skill_knowledge_dispatcher", None)
if dispatcher is not None:
dispatcher.nudge()
return job
@router.get("/sync-jobs")
async def list_sync_jobs(
request: Request,
limit: int = Query(default=50, ge=1, le=200),
mine_only: bool = Query(default=False),
) -> dict[str, Any]:
actor, is_admin = await _actor(request)
return {
"jobs": await _store(request).list_jobs(
limit=limit,
created_by=actor if mine_only or not is_admin else None,
)
}
@router.get("/sync-jobs/{job_id}")
async def get_sync_job(request: Request, job_id: str) -> dict[str, Any]:
actor, is_admin = await _actor(request)
row = await _store(request).get_job(job_id)
if not _owned_by_actor(row, actor, is_admin):
raise HTTPException(status_code=404, detail="归纳任务不存在")
return row
@router.post("/sync-jobs/{job_id}/cancel")
async def cancel_sync_job(request: Request, job_id: str) -> dict[str, bool]:
actor, is_admin = await _actor(request)
row = await _store(request).get_job(job_id)
if not _owned_by_actor(row, actor, is_admin):
raise HTTPException(status_code=404, detail="归纳任务不存在")
if not await _store(request).cancel_job(job_id):
raise HTTPException(status_code=404, detail="归纳任务不存在")
return {"canceled": True}
@router.post("/sync-items/{item_id}/retry", status_code=202)
async def retry_sync_item(request: Request, item_id: str) -> dict[str, Any]:
actor, is_admin = await _actor(request)
jobs = await _store(request).list_jobs(limit=200, created_by=None if is_admin else actor)
item = next(
(item for job in jobs for item in job.get("items", []) if item.get("id") == item_id),
None,
)
if item is None:
raise HTTPException(status_code=404, detail="归纳任务项不存在")
if item.get("target_type") == "weknora":
mapping = await request.app.state.llmwiki_store.get_authorized(
str(item["target_id"]), actor, write=True, is_admin=is_admin
)
if mapping is None:
raise HTTPException(status_code=404, detail="目标知识库不存在或当前用户不可写")
row = await _store(request).requeue_item(item_id)
if row is None:
raise HTTPException(status_code=404, detail="归纳任务项不存在")
request.app.state.skill_knowledge_dispatcher.nudge()
return row
@router.post("/bindings/{binding_id}/resync", status_code=202)
async def resync_binding(request: Request, binding_id: str) -> dict[str, Any]:
actor, is_admin = await _actor(request)
binding = await _store(request).get_binding(binding_id)
if not _owned_by_actor(binding, actor, is_admin) or not binding.get("active"):
raise HTTPException(status_code=404, detail="绑定不存在")
if binding.get("target_type") == "weknora":
mapping = await request.app.state.llmwiki_store.get_authorized(
str(binding["target_id"]), actor, write=True, is_admin=is_admin
)
if mapping is None:
raise HTTPException(status_code=404, detail="目标知识库不存在或当前用户不可写")
target = {
"target_type": binding["target_type"],
"target_mode": binding["target_mode"],
"target_id": binding["target_id"],
"target_name": binding["target_name"],
"remote_business_key": binding.get("remote_business_key"),
}
job = await _store(request).create_job(
skill_names=[binding["skill_name"]],
targets=[target],
created_by=actor,
force=True,
review_mode="auto_high_confidence",
)
request.app.state.skill_knowledge_dispatcher.nudge()
return job
@router.post("/bindings/{binding_id}/detach")
async def detach_binding(request: Request, binding_id: str, body: DetachRequest) -> dict[str, Any]:
actor, is_admin = await _actor(request)
store = _store(request)
binding = await store.get_binding(binding_id)
if not _owned_by_actor(binding, actor, is_admin):
raise HTTPException(status_code=404, detail="绑定不存在")
if await store.has_active_item(binding_id):
raise HTTPException(status_code=409, detail="该绑定仍有运行中的归纳项,请先取消任务")
cleanup_failed = False
if body.delete_contributions:
try:
if binding["target_type"] == "assistant":
sources = await request.app.state.assistant_knowledge_store.list_sources()
for source in sources:
if source.get("source_type") == "skill" and source.get("source_key") == binding["skill_name"]:
await request.app.state.assistant_knowledge_store.delete_source(source["id"])
elif binding["target_type"] == "weknora":
mapping = await request.app.state.llmwiki_store.get_authorized(
str(binding["target_id"]),
actor,
write=True,
is_admin=is_admin,
)
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
if mapping is None or not runtime.weknora_enabled:
raise RuntimeError("远端普通知识库当前不可写")
client = build_weknora_client(runtime)
remote_kb_id = str(mapping["weknora_id"])
for artifact in await store.list_artifacts(binding_id):
if artifact["kind"] == "wiki_page" and artifact.get("slug"):
await client.delete_wiki_page(remote_kb_id, artifact["slug"])
elif artifact.get("remote_id") and artifact["kind"] in {
"raw_source",
"summary_document",
"faq_projection",
"conversation_archive",
"graph_projection",
}:
await client.delete_document(str(artifact["remote_id"]))
except Exception: # noqa: BLE001
cleanup_failed = True
row = await store.detach_binding(
binding_id,
delete_contributions=body.delete_contributions,
cleanup_failed=cleanup_failed,
)
if row is None:
raise HTTPException(status_code=404, detail="绑定不存在")
return row
@router.get("/reviews")
async def list_reviews(
request: Request,
job_id: str | None = Query(default=None),
item_id: str | None = Query(default=None),
status: str | None = Query(default="pending"),
) -> dict[str, Any]:
await _require_admin(request)
return {"reviews": await _store(request).list_reviews(job_id=job_id, item_id=item_id, status=status)}
@router.post("/reviews/actions")
async def review_actions(request: Request, body: ReviewRequest) -> dict[str, Any]:
actor = await _require_admin(request)
item_ids = await _store(request).review(body.review_ids, action=body.action, actor=actor, edits=body.edits, reason=body.reason)
resumed: list[str] = []
executor = request.app.state.skill_knowledge_executor
for item_id in item_ids:
if await _store(request).pending_review_count(item_id) == 0:
asyncio.create_task(executor.resume_reviewed_item(item_id))
resumed.append(item_id)
return {"updated": len(body.review_ids), "resumed_item_ids": resumed}
@@ -48,6 +48,14 @@ class PromptPrefixResponse(BaseModel):
)
class KnowledgeBaseProductVersionResponse(BaseModel):
version: Literal["v1", "v2"] = "v1"
class KnowledgeBaseProductVersionUpdateRequest(BaseModel):
version: Literal["v1", "v2"]
class PromptPrefixUpdateRequest(BaseModel):
enabled: bool | None = None
switch_label: str | None = None
@@ -175,6 +183,21 @@ async def update_prompt_prefix(body: PromptPrefixUpdateRequest, request: Request
return _to_response(saved.prompt_prefix)
@router.get("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
async def get_knowledge_base_product_version() -> KnowledgeBaseProductVersionResponse:
return KnowledgeBaseProductVersionResponse(version=load_system_settings().knowledge_base_product_version)
@router.put("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
async def update_knowledge_base_product_version(
body: KnowledgeBaseProductVersionUpdateRequest,
request: Request,
) -> KnowledgeBaseProductVersionResponse:
await _require_admin(request)
saved = save_system_settings(load_system_settings().model_copy(update={"knowledge_base_product_version": body.version}))
return KnowledgeBaseProductVersionResponse(version=saved.knowledge_base_product_version)
@router.get("/plugin-management", response_model=PluginManagementResponse)
async def get_plugin_management() -> PluginManagementResponse:
"""Return the iframe plugin domain -> agent mappings."""
@@ -0,0 +1,193 @@
"""Asynchronous, file-backed assistant export and ordinary Wiki restore."""
from __future__ import annotations
import asyncio
from pathlib import Path
from uuid import uuid4
from fastapi import APIRouter, File, HTTPException, Request, UploadFile
from fastapi.responses import FileResponse
from app.gateway.llmwiki_index_scheduler import spawn_local_wiki_index_job
from app.gateway.routers.assistant_knowledge import _actor, _list_export_manifests, _package_root, _read_export_manifest, _store, _write_export_manifest
from deerflow.assistant_knowledge.archive import PackageArchive
from deerflow.assistant_knowledge.package_io import package_path, write_jsonl_record
from deerflow.integrations.weknora.local_index.normalize import normalize_wiki_page
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.persistence.assistant_knowledge.store import _vector_blob_from_payload
router = APIRouter(tags=["wiki-packages"])
def job_view(row):
if row["mapping_id"].startswith("assistant:"):
base_id = row["mapping_id"].split(":", 1)[1]
row["package_download_url"] = f"/api/assistant-knowledge/bases/{base_id}/export-package/{row['id']}/download"
elif row["mapping_id"].startswith("restore:"):
row["package_download_url"] = None
return row
async def export_assistant(app, base_id, job_id):
scope = f"assistant:{base_id}"
path = package_path(_package_root(), job_id)
temp = path.with_suffix(".tmp")
counts = {}
try:
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="exporting")
with temp.open("w", encoding="utf-8", newline="\n") as handle:
async for kind, payload in app.state.assistant_knowledge_store.iter_export_records(base_id):
await asyncio.to_thread(write_jsonl_record, handle, kind, payload)
counts[kind] = counts.get(kind, 0) + 1
if counts.get("wiki_page") and not counts.get("vector"):
raise ValueError("该助手知识库尚无可导出的 Wiki 向量,请先从已向量化的普通知识库同步")
await asyncio.to_thread(temp.replace, path)
_write_export_manifest(job_id=job_id, mapping_id=scope, status="completed", phase="package_ready", counts=counts)
except Exception as exc:
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="failed", counts=counts, error=str(exc))
@router.post("/api/assistant-knowledge/bases/{base_id}/export-package", status_code=202)
async def create_export(request: Request, base_id: str):
await _actor(request, admin=True)
if not await _store(request).get_base(base_id):
raise HTTPException(404, "助手知识库不存在")
job_id = str(uuid4())
row = _write_export_manifest(job_id=job_id, mapping_id=f"assistant:{base_id}", status="queued", phase="queued")
spawn_local_wiki_index_job(request.app, export_assistant(request.app, base_id, job_id))
return job_view(row)
@router.get("/api/assistant-knowledge/bases/{base_id}/export-package")
async def list_exports(request: Request, base_id: str):
await _actor(request, admin=True)
return {"jobs": [job_view(row) for row in await asyncio.to_thread(_list_export_manifests, f"assistant:{base_id}", limit=100)]}
@router.get("/api/assistant-knowledge/bases/{base_id}/export-package/{job_id}/download")
async def download_export(request: Request, base_id: str, job_id: str):
await _actor(request, admin=True)
row = _read_export_manifest(job_id)
if not row or row["mapping_id"] != f"assistant:{base_id}" or row["status"] != "completed" or not row["file_exists"]:
raise HTTPException(404, "导出包尚未就绪")
return FileResponse(package_path(_package_root(), job_id), media_type="application/x-ndjson", filename=f"wiki-vector-{job_id}.jsonl")
async def restore_package(app, mapping, job_id, path: Path):
scope = f"restore:{mapping['id']}"
archive = None
counts = {"wiki_page": 0, "vector": 0}
store = app.state.llmwiki_index_store
owner = f"package:{job_id}"
leased = False
try:
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="validating")
archive = await asyncio.to_thread(PackageArchive, path)
embedding = app.state.llmwiki_embedding
if embedding is None:
raise ValueError("请配置与数据包相同的 Wiki 编码模型,以便复用向量进行检索")
if embedding.dimensions is None:
await embedding.embed_query("Wiki embedding dimension probe")
# Validate the complete package before the first remote mutation.
slugs = {p["slug"] for p in archive["wiki_pages"]}
vector_slugs = set()
for raw in archive["vectors"]:
slug = str(raw.get("wiki_slug") or raw.get("page_slug") or raw.get("slug") or "")
if slug not in slugs or _vector_blob_from_payload(raw) is None:
raise ValueError(f"向量数据或 Wiki 引用无效:{slug}")
if raw.get("embedding_fingerprint") != embedding.fingerprint or raw.get("embedding_dimensions") != embedding.dimensions:
raise ValueError("数据包编码模型指纹与当前配置不一致;请使用同一模型配置,禁止静默重新编码")
vector_slugs.add(slug)
if not vector_slugs:
raise ValueError("数据包中没有 Wiki 向量")
for page in archive["wiki_pages"]:
if page["slug"] not in vector_slugs and page.get("page_type") not in {"index", "directory"} and (page.get("content") or "").strip():
raise ValueError(f"Wiki 缺少向量:{page['slug']}")
leased = await store.try_acquire_sync_lease(mapping["id"], owner=owner, sync_id=job_id, lease_seconds=900, fingerprint=embedding.fingerprint)
if not leased:
raise ValueError("目标知识库正在同步,请稍后重试")
client = build_weknora_client(get_resolved_llmwiki_runtime(app.state.config))
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="restoring")
folders = {(): ""}
for raw in archive["wiki_pages"]:
if not await store.renew_sync_lease(mapping["id"], owner=owner, lease_seconds=900):
raise RuntimeError("导入租约已失效,请重新发起")
# Restore the provider's Wiki schema, never raw documents; raw
# uploads would trigger generation and embedding a second time.
categories = raw.get("category_path") or []
if isinstance(categories, str):
categories = [part for part in categories.split("/") if part]
parent = ()
for name in categories:
path_key = (*parent, name)
if path_key not in folders:
folder = await client.create_wiki_folder(mapping["weknora_id"], name=name, parent_id=folders[parent])
folders[path_key] = folder["id"]
parent = path_key
payload = {key: value for key, value in raw.items() if key not in {"id", "tenant_id", "knowledge_base_id", "folder_id", "origins"}}
payload["folder_id"] = folders[parent]
if raw.get("tags"):
payload["page_metadata"] = {**(raw.get("page_metadata") or {}), "tags": raw["tags"]}
remote = await client.create_wiki_page(mapping["weknora_id"], payload)
page = normalize_wiki_page(mapping["id"], {**raw, **remote})
vectors = []
for value in archive.vectors_for(raw["slug"]):
blob, dimensions = _vector_blob_from_payload(value)
vectors.append({**value, "vector_blob": blob, "embedding_dimensions": dimensions})
await store.replace_page_vectors(page=page, vectors=vectors, fingerprint=embedding.fingerprint, sync_id=job_id)
counts["wiki_page"] += 1
counts["vector"] += len(vectors)
_write_export_manifest(job_id=job_id, mapping_id=scope, status="running", phase="restoring", counts=counts)
await client.rebuild_wiki_links(mapping["weknora_id"])
await store.finish_sync(mapping["id"], owner=owner, sync_id=job_id, success=True, remote_page_count=counts["wiki_page"])
leased = False
if app.state.llmwiki_vector_cache:
app.state.llmwiki_vector_cache.invalidate(mapping["id"])
from app.gateway.knowledge_transfer import sync_to_global
await sync_to_global(app, mapping)
_write_export_manifest(job_id=job_id, mapping_id=scope, status="completed", phase="completed", counts=counts)
except Exception as exc:
if leased:
await store.finish_sync(mapping["id"], owner=owner, sync_id=job_id, success=False, remote_page_count=counts["wiki_page"], error=str(exc))
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="failed", counts=counts, error=str(exc))
finally:
if archive is not None:
await asyncio.to_thread(archive.close)
@router.post("/api/llmwiki/knowledge-bases/{mapping_id}/import-package", status_code=202)
async def upload_restore(request: Request, mapping_id: str, file: UploadFile = File(...)):
actor = await _actor(request, admin=True)
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, actor or "system", write=True, is_admin=True)
if not mapping or mapping.get("is_conversation_deposit") or mapping.get("owner_user_id") == "system":
raise HTTPException(404, "目标普通知识库不存在或不可导入")
client = build_weknora_client(get_resolved_llmwiki_runtime(request.app.state.config))
remote = await client.get_knowledge_base(mapping["weknora_id"])
if not (remote.get("indexing_strategy") or {}).get("wiki_enabled"):
raise HTTPException(422, "请先开启目标知识库的 Wiki")
existing = await client.list_wiki_pages(mapping["weknora_id"], page_size=1)
if existing.get("total"):
raise HTTPException(409, "请导入到新建的空 Wiki 知识库,避免覆盖已有页面")
job_id = str(uuid4())
scope = f"restore:{mapping_id}"
row = _write_export_manifest(job_id=job_id, mapping_id=scope, status="queued", phase="uploading")
path = package_path(_package_root(), job_id)
try:
with path.open("wb") as handle:
while data := await file.read(1024 * 1024):
await asyncio.to_thread(handle.write, data)
except Exception as exc:
_write_export_manifest(job_id=job_id, mapping_id=scope, status="failed", phase="upload_failed", error=str(exc))
raise
finally:
await file.close()
spawn_local_wiki_index_job(request.app, restore_package(request.app, mapping, job_id, path))
return job_view(row)
@router.get("/api/llmwiki/knowledge-bases/{mapping_id}/import-package")
async def list_restores(request: Request, mapping_id: str):
await _actor(request, admin=True)
return {"jobs": [job_view(row) for row in await asyncio.to_thread(_list_export_manifests, f"restore:{mapping_id}", limit=100)]}
@@ -405,6 +405,7 @@ _CONTEXT_CONFIGURABLE_KEYS: frozenset[str] = frozenset(
"reasoning_effort",
"rag_mode_enabled",
"llmwiki_knowledge_base_ids",
"assistant_knowledge_base_ids",
"writing_mode",
"writing_artifact_path",
"canvas_artifact_title",
@@ -1116,7 +1117,7 @@ async def start_run(
if prefix_settings.enabled and prefix_settings.prompt_prefix.strip():
graph_input = _prepend_prompt_prefix(graph_input, prefix_settings.prompt_prefix.strip())
llmwiki_rag = (
LlmWikiRagContext([], 0, "", [], "")
LlmWikiRagContext([], [], 0, "", [], "")
if fixed_answer is not None
else await build_llmwiki_rag_context(
request=request,
@@ -1128,12 +1129,14 @@ async def start_run(
graph_input = attach_llmwiki_rag_to_graph_input(graph_input, llmwiki_rag)
effective_rag_mode = True
run_metadata["llmwiki_knowledge_base_ids"] = llmwiki_rag.knowledge_base_ids
run_metadata["assistant_knowledge_base_ids"] = llmwiki_rag.assistant_knowledge_base_ids
run_metadata["llmwiki_rag_source_count"] = llmwiki_rag.source_count
try:
await run_ctx.thread_store.update_metadata(
thread_id,
{
"llmwiki_knowledge_base_ids": llmwiki_rag.knowledge_base_ids,
"assistant_knowledge_base_ids": llmwiki_rag.assistant_knowledge_base_ids,
"rag_mode_enabled": True,
"rag_mode": True,
},
@@ -1153,6 +1156,8 @@ async def start_run(
body_context["rag_mode_enabled"] = True
if "llmwiki_knowledge_base_ids" not in body_context and llmwiki_rag.knowledge_base_ids:
body_context["llmwiki_knowledge_base_ids"] = llmwiki_rag.knowledge_base_ids
if "assistant_knowledge_base_ids" not in body_context and llmwiki_rag.assistant_knowledge_base_ids:
body_context["assistant_knowledge_base_ids"] = llmwiki_rag.assistant_knowledge_base_ids
if not llmwiki_rag.knowledge_base_ids:
existing_excluded = list(body_context.get("excluded_tools") or getattr(body, "excluded_tools", None) or [])
if "llmwiki_search" not in existing_excluded:
@@ -0,0 +1,155 @@
"""Debounced file watcher and cron scheduler for opted-in knowledge bindings."""
from __future__ import annotations
import asyncio
import logging
from datetime import UTC, datetime
from typing import Any
from croniter import croniter
from app.gateway.skill_knowledge_job_executor import _skill_directory
from deerflow.config.system_settings import load_system_settings
from deerflow.skill_knowledge import scan_skill
logger = logging.getLogger(__name__)
class SkillKnowledgeAutoSyncService:
def __init__(self, app: Any, *, poll_seconds: float = 10.0) -> None:
self.app = app
self.store = app.state.skill_knowledge_store
self.poll_seconds = poll_seconds
self._task: asyncio.Task | None = None
self._wake = asyncio.Event()
self._observed: dict[str, tuple[str, datetime]] = {}
self._schedule_expression = ""
self._next_schedule: datetime | None = None
def start(self) -> None:
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._loop())
def nudge(self) -> None:
self._wake.set()
async def close(self) -> None:
if self._task is None:
return
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
async def _queue_skill(
self,
skill_name: str,
bindings: list[dict[str, Any]],
*,
trigger: str,
review_mode: str,
) -> None:
targets = [
{
"target_type": binding["target_type"],
"target_mode": binding["target_mode"],
"target_id": binding["target_id"],
"target_name": binding["target_name"],
"remote_business_key": binding.get("remote_business_key"),
}
for binding in bindings
]
job = await self.store.create_job(
skill_names=[skill_name],
targets=targets,
created_by=None,
trigger=trigger,
review_mode=review_mode,
)
if job["total_count"]:
self.app.state.skill_knowledge_dispatcher.nudge()
async def _watch(
self,
bindings_by_skill: dict[str, list[dict[str, Any]]],
*,
debounce_seconds: int,
write_remote_enabled: bool,
review_mode: str,
) -> None:
now = datetime.now(UTC)
for skill_name, bindings in bindings_by_skill.items():
try:
root = _skill_directory(self.app.state.config, skill_name)
scan = await asyncio.to_thread(scan_skill, skill_name, root)
except Exception:
logger.exception("Failed to fingerprint skill %s", skill_name)
continue
synced = {str(binding.get("synced_digest") or "") for binding in bindings}
if synced == {scan.source_digest}:
self._observed.pop(skill_name, None)
continue
await self.store.mark_skill_stale(skill_name)
previous = self._observed.get(skill_name)
if previous is None or previous[0] != scan.source_digest:
self._observed[skill_name] = (scan.source_digest, now)
continue
if (now - previous[1]).total_seconds() < debounce_seconds:
continue
if write_remote_enabled:
current = await self.store.list_bindings(skill_names=[skill_name])
stale = [binding for binding in current if binding.get("auto_sync_enabled") and binding.get("status") == "stale"]
if stale:
await self._queue_skill(
skill_name,
stale,
trigger="watcher",
review_mode=review_mode,
)
self._observed.pop(skill_name, None)
def _schedule_due(self, expression: str, now: datetime) -> bool:
if expression != self._schedule_expression or self._next_schedule is None:
self._schedule_expression = expression
self._next_schedule = croniter(expression, now).get_next(datetime)
return False
if now < self._next_schedule:
return False
self._next_schedule = croniter(expression, now).get_next(datetime)
return True
async def _tick(self) -> None:
settings = (await asyncio.to_thread(load_system_settings)).skill_knowledge_auto_sync
bindings = [row for row in await self.store.list_bindings() if row.get("auto_sync_enabled")]
by_skill: dict[str, list[dict[str, Any]]] = {}
for binding in bindings:
by_skill.setdefault(str(binding["skill_name"]), []).append(binding)
if settings.watch_enabled:
await self._watch(
by_skill,
debounce_seconds=settings.debounce_seconds,
write_remote_enabled=settings.write_remote_enabled,
review_mode=settings.review_mode,
)
if settings.schedule_enabled and settings.write_remote_enabled and by_skill and self._schedule_due(settings.schedule_cron, datetime.now(UTC)):
for skill_name, skill_bindings in by_skill.items():
await self._queue_skill(
skill_name,
skill_bindings,
trigger="scheduled",
review_mode=settings.review_mode,
)
async def _loop(self) -> None:
while True:
try:
await self._tick()
except Exception:
logger.exception("Skill knowledge automatic sync tick failed")
try:
await asyncio.wait_for(self._wake.wait(), timeout=self.poll_seconds)
except TimeoutError:
pass
self._wake.clear()
@@ -0,0 +1,96 @@
"""Small persistent-queue dispatcher for skill knowledge items."""
from __future__ import annotations
import asyncio
import logging
import socket
from typing import Any
from uuid import uuid4
logger = logging.getLogger(__name__)
class SkillKnowledgeJobDispatcher:
def __init__(
self,
store: Any,
executor: Any,
*,
interval_seconds: float = 2.0,
concurrency: int = 3,
lease_seconds: int = 900,
) -> None:
self.store = store
self.executor = executor
self.interval_seconds = interval_seconds
self.lease_seconds = lease_seconds
self.owner = f"{socket.gethostname()}:{uuid4()}"
self.semaphore = asyncio.Semaphore(concurrency)
self._wake = asyncio.Event()
self._task: asyncio.Task | None = None
self._running: set[str] = set()
def start(self) -> None:
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._loop())
def nudge(self) -> None:
self._wake.set()
async def close(self) -> None:
if self._task is not None:
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
async def _run(self, item: dict) -> None:
item_id = str(item["id"])
async with self.semaphore:
heartbeat = asyncio.create_task(self._heartbeat(item_id))
try:
await self.executor.run_item(item)
finally:
heartbeat.cancel()
try:
await heartbeat
except asyncio.CancelledError:
pass
self._running.discard(item_id)
self._wake.set()
async def _heartbeat(self, item_id: str) -> None:
while True:
await asyncio.sleep(min(30, max(5, self.lease_seconds // 3)))
if not await self.store.renew_lease(
item_id,
owner=self.owner,
lease_seconds=self.lease_seconds,
):
return
async def _loop(self) -> None:
while True:
try:
for item in await self.store.list_queued_items(limit=20):
item_id = str(item["id"])
if item_id in self._running:
continue
claimed = await self.store.claim_item(
item_id,
owner=self.owner,
lease_seconds=self.lease_seconds,
)
if claimed is None:
continue
self._running.add(item_id)
asyncio.create_task(self._run(claimed))
except Exception:
logger.exception("Skill knowledge dispatcher tick failed")
try:
await asyncio.wait_for(self._wake.wait(), timeout=self.interval_seconds)
except TimeoutError:
pass
self._wake.clear()
@@ -0,0 +1,466 @@
"""Background executor for one skill x target matrix item."""
from __future__ import annotations
import asyncio
import json
import logging
from pathlib import Path
from typing import Any
import httpx
from deerflow.assistant_knowledge import export_package_from_skill_ir
from deerflow.integrations.weknora.client import WeKnoraError
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
from deerflow.skill_knowledge import build_skill_ir, enrich_converted_documents, scan_skill
from deerflow.skill_knowledge.graph_writer import parameterized_statements, plan_from_ir
from deerflow.skills.parser import parse_skill_file
from deerflow.skills.storage import get_or_new_skill_storage
from deerflow.skills.types import SkillCategory
logger = logging.getLogger(__name__)
def _skill_directory(app_config: Any, skill_name: str) -> Path:
storage = get_or_new_skill_storage(app_config=app_config)
root = storage.get_skills_root_path().resolve()
for category in SkillCategory:
direct = root / category.value / skill_name
if direct.resolve().is_relative_to((root / category.value).resolve()) and (direct / "SKILL.md").is_file():
return direct
for category in SkillCategory:
category_root = root / category.value
if not category_root.is_dir():
continue
for skill_md in category_root.rglob("SKILL.md"):
if skill_md.is_symlink():
continue
parsed = parse_skill_file(skill_md, category=category, relative_path=skill_md.parent.relative_to(category_root))
if parsed and parsed.name == skill_name:
return skill_md.parent
raise FileNotFoundError(f"Skill '{skill_name}' not found under running skill roots")
def _graph_projection(ir: dict[str, Any]) -> str:
rows = []
entities = {str(item.get("local_id")): item for item in ir.get("entities", [])}
for relation in ir.get("relations", []):
if float(relation.get("confidence") or 0) < 0.75 or not relation.get("evidence"):
continue
source = entities.get(str(relation.get("source_local_id")), {})
target = entities.get(str(relation.get("target_local_id")), {})
rows.append(
json.dumps(
{
"source": source.get("name"),
"source_type": source.get("type"),
"predicate": relation.get("predicate"),
"target": target.get("name"),
"target_type": target.get("type"),
"description": relation.get("description"),
"confidence": relation.get("confidence"),
"evidence": relation.get("evidence"),
},
ensure_ascii=False,
)
)
return "\n".join(rows)
class SkillKnowledgeJobExecutor:
def __init__(self, app: Any) -> None:
self.app = app
self.store = app.state.skill_knowledge_store
async def _cancel_if_requested(self, item: dict[str, Any]) -> bool:
job = await self.store.get_job(str(item["job_id"]))
if not (job or {}).get("cancel_requested"):
return False
await self.store.update_item(
str(item["id"]),
status="canceled",
phase="completed",
progress=100,
)
return True
async def _assert_source_unchanged(self, item: dict[str, Any], expected_digest: str) -> None:
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
current = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
current = await enrich_converted_documents(current)
if current.source_digest == expected_digest:
return
await self.store.mark_skill_stale(str(item["skill_name"]))
raise RuntimeError("技能知识文件在归纳期间发生变化,旧版本保持激活,请重试")
async def _write_direct_graph(
self,
item: dict[str, Any],
snapshot_id: str,
ir: dict[str, Any],
) -> dict[str, Any] | None:
config = self.app.state.config.llmwiki.weknora.graph
if not config.direct_write_enabled:
return None
if not config.neo4j_http_url or not config.username or not config.password:
raise RuntimeError("Neo4j direct writer 已开启但后端连接配置不完整")
plan = plan_from_ir(
ir,
target_type=str(item["target_type"]),
target_id=str(item["target_id"]),
binding_id=str(item["binding_id"]),
snapshot_id=snapshot_id,
job_id=str(item["job_id"]),
)
statements = parameterized_statements(plan)
if not statements:
return {"nodes": 0, "edges": 0, "statement_count": 0}
endpoint = f"{config.neo4j_http_url.rstrip('/')}/db/{config.database}/tx/commit"
async with httpx.AsyncClient(
auth=(config.username, config.password),
timeout=config.timeout_seconds,
) as client:
response = await client.post(endpoint, json={"statements": statements})
response.raise_for_status()
payload = response.json()
errors = payload.get("errors") or []
if errors:
raise RuntimeError(f"Neo4j transaction failed: {str(errors[0].get('code') or 'unknown')}")
return {
"nodes": len(plan.nodes),
"edges": len(plan.edges),
"statement_count": len(statements),
"scope": plan.scope,
}
async def run_item(self, item: dict[str, Any]) -> None:
item_id = str(item["id"])
try:
await self.store.update_item(item_id, status="running", phase="scanning", progress=5)
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
scan = await enrich_converted_documents(scan)
if await self._cancel_if_requested(item):
return
await self.store.update_item(item_id, phase="extracting", progress=30)
skill_record = await self.app.state.skill_store.get_any(str(item["skill_name"]))
ir = await asyncio.to_thread(
build_skill_ir,
scan,
display_name=(skill_record or {}).get("name_zh"),
description=str((skill_record or {}).get("description") or ""),
)
snapshot = await self.store.get_or_create_snapshot(
skill_name=str(item["skill_name"]),
source_digest=scan.source_digest,
extractor_version=str(ir["extractor_version"]),
prompt_hash=str(ir["prompt_hash"]),
model_name=str(ir["model_name"]),
manifest=scan.manifest,
ir=ir,
counts=scan.counts,
)
snapshot_id = str(snapshot["id"])
await self.store.update_item(item_id, phase="awaiting_review", progress=45, snapshot_id=snapshot_id)
job = await self.store.get_job(str(item["job_id"]))
review_mode = str((job or {}).get("review_mode") or "auto_high_confidence")
review_objects: list[tuple[str, dict]] = []
if review_mode == "required":
review_objects = [("entity", obj) for obj in ir.get("entities", [])] + [("relation", obj) for obj in ir.get("relations", [])]
elif review_mode == "auto_high_confidence":
review_objects = [(kind, obj) for kind, rows in (("entity", ir.get("entities", [])), ("relation", ir.get("relations", []))) for obj in rows if float(obj.get("confidence") or 0) < 0.9]
if review_objects:
existing = await self.store.list_reviews(item_id=item_id, status=None)
if not existing:
await self.store.create_reviews(
job_id=str(item["job_id"]),
item_id=item_id,
binding_id=str(item["binding_id"]),
snapshot_id=snapshot_id,
objects=review_objects,
)
await self.store.update_item(item_id, status="awaiting_review", phase="awaiting_review", progress=50)
return
await self._project_and_activate(item, snapshot_id, ir, scan)
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001
logger.exception("Skill knowledge item failed: item=%s", item_id)
await self.store.update_item(
item_id,
status="failed",
phase="completed",
progress=100,
error_code=type(exc).__name__,
error_message=str(exc)[:2000],
)
async def resume_reviewed_item(self, item_id: str) -> None:
if await self.store.pending_review_count(item_id):
return
jobs = await self.store.list_jobs(limit=100)
item = None
for job in jobs:
detail = await self.store.get_job(str(job["id"]))
item = next((row for row in (detail or {}).get("items", []) if row["id"] == item_id), None)
if item:
break
if not item or not item.get("snapshot_id"):
return
snapshot = await self.store.get_snapshot(str(item["snapshot_id"]))
ir = await self.store.effective_ir(str(item["snapshot_id"]), item_id)
if not snapshot or not ir:
return
try:
await self.store.update_item(item_id, status="running", phase="merging", progress=55)
await self._project_and_activate(item, str(item["snapshot_id"]), ir, None)
except Exception as exc: # noqa: BLE001
logger.exception("Reviewed skill knowledge item failed: item=%s", item_id)
await self.store.update_item(
item_id,
status="failed",
phase="completed",
progress=100,
error_code=type(exc).__name__,
error_message=str(exc)[:2000],
)
async def _project_and_activate(self, item: dict[str, Any], snapshot_id: str, ir: dict[str, Any], scan: Any | None) -> None:
if await self._cancel_if_requested(item):
return
expected_digest = str(ir.get("skill", {}).get("source_digest") or "")
if scan is None:
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
scan = await enrich_converted_documents(scan)
await self._assert_source_unchanged(item, expected_digest)
if item["target_type"] == "weknora" and item["target_mode"] == "wiki":
await self._generate_vectorize_and_deposit(item, snapshot_id, ir, scan)
return
await self.store.update_item(str(item["id"]), phase="writing_wiki", progress=65)
target_type = str(item["target_type"])
artifacts: list[dict[str, Any]] = []
binding_status = "synced"
page = ir["wiki"]["skill_page"]
if target_type == "assistant":
package = export_package_from_skill_ir(ir)
await self.app.state.assistant_knowledge_store.import_package(
package=package,
source_type="skill",
source_key=str(item["skill_name"]),
source_name=str(ir["skill"].get("display_name") or item["skill_name"]),
trigger="skill_direct",
created_by=(await self.store.get_job(str(item["job_id"]))).get("created_by"),
skill_name=str(item["skill_name"]),
)
artifacts.append({"kind": "assistant_wiki_revision", "slug": page["slug"], "content": page["content"], "business_key": f"assistant:wiki:{page['slug']}"})
elif str(item["target_mode"]) == "readonly_sidecar":
artifacts.append({"kind": "sidecar_wiki", "slug": page["slug"], "content": page["content"], "business_key": f"sidecar:{item['target_id']}:{page['slug']}"})
artifacts.append({"kind": "sidecar_graph", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "content": _graph_projection(ir)})
binding_status = "pending_remote_write"
else:
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
if mapping is None:
raise ValueError("Target WeKnora mapping does not exist")
runtime = get_resolved_llmwiki_runtime(self.app.state.config)
if not runtime.weknora_enabled:
raise RuntimeError("普通知识库服务未配置")
client = build_weknora_client(runtime)
remote_kb_id = str(mapping["weknora_id"])
if scan is not None:
for file in scan.files:
source_path = Path(scan.source_root) / file.path
if file.text:
remote = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/source/{file.path}", content=file.text)
else:
remote = await client.upload_document(
remote_kb_id,
filename=f"{item['skill_name']}-{source_path.name}",
content=await asyncio.to_thread(source_path.read_bytes),
)
remote_id = str(remote.get("id") or remote.get("knowledge_id") or "") or None
artifacts.append(
{
"kind": "raw_source",
"slug": file.path,
"remote_id": remote_id,
"business_key": f"weknora:document:{remote_id}" if remote_id else None,
"remote_refs": [{"provider": "weknora", "remote_kind": "document", "remote_id": remote_id}] if remote_id else [],
}
)
target_mode = str(item["target_mode"])
if target_mode == "wiki":
remote_page: dict[str, Any]
try:
await client.get_wiki_page(remote_kb_id, page["slug"])
except WeKnoraError as exc:
if exc.status_code != 404:
raise
remote_page = await client.create_wiki_page(
remote_kb_id,
{
"slug": page["slug"],
"title": page["title"],
"content": page["content"],
"summary": str(ir["skill"].get("description") or ""),
"page_type": "skill",
},
)
else:
remote_page = await client.update_wiki_page(
remote_kb_id,
page["slug"],
{
"title": page["title"],
"content": page["content"],
"summary": str(ir["skill"].get("description") or ""),
},
)
remote_page_id = str(remote_page.get("id") or remote_page.get("page_id") or page["slug"])
artifacts.append(
{
"kind": "wiki_page",
"slug": page["slug"],
"content": page["content"],
"remote_id": remote_page_id,
"business_key": f"weknora:wiki_page:{remote_page_id}",
"remote_refs": [
{
"provider": "weknora",
"remote_kind": "wiki_page",
"remote_id": remote_page_id,
"slug": page["slug"],
}
],
}
)
else:
document_content = page["content"]
artifact_kind = "summary_document"
if target_mode == "faq":
artifact_kind = "faq_projection"
document_content = f"问题:{page['title']} 能解决什么问题,如何使用?\n\n回答:\n{page['content']}"
elif target_mode == "conversation_archive":
artifact_kind = "conversation_archive"
document_content = f"# 技能归纳会话归档:{page['title']}\n\n以下内容由技能知识归纳任务生成。\n\n{page['content']}"
remote_document = await client.create_manual_document(
remote_kb_id,
title=f"skills/{item['skill_name']}/{target_mode}.md",
content=document_content,
)
remote_document_id = str(remote_document.get("id") or remote_document.get("knowledge_id") or "") or None
artifacts.append(
{
"kind": artifact_kind,
"slug": f"skills/{item['skill_name']}/{target_mode}.md",
"content": document_content,
"remote_id": remote_document_id,
"business_key": (f"weknora:document:{remote_document_id}" if remote_document_id else None),
"remote_refs": (
[
{
"provider": "weknora",
"remote_kind": "document",
"remote_id": remote_document_id,
}
]
if remote_document_id
else []
),
}
)
projection = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/graph-projection.jsonl", content=_graph_projection(ir))
projection_id = str(projection.get("id") or projection.get("knowledge_id") or "") or None
artifacts.append({"kind": "graph_projection", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "remote_id": projection_id})
try:
graph_audit = await self._write_direct_graph(item, snapshot_id, ir)
if graph_audit is not None:
artifacts.append(
{
"kind": "neo4j_direct_write_audit",
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
"content": json.dumps(graph_audit, ensure_ascii=False),
"metadata": graph_audit,
}
)
except Exception as exc: # noqa: BLE001
logger.exception("Optional Neo4j direct graph write failed: item=%s", item["id"])
binding_status = "partial_failed"
artifacts.append(
{
"kind": "neo4j_direct_write_audit",
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
"content": json.dumps(
{"status": "failed", "error": type(exc).__name__},
ensure_ascii=False,
),
"metadata": {
"status": "failed",
"error_type": type(exc).__name__,
},
}
)
await self._assert_source_unchanged(item, expected_digest)
await self.store.update_item(str(item["id"]), phase="activating", progress=90)
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status=binding_status)
async def _generate_vectorize_and_deposit(self, item, snapshot_id, ir, scan):
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
if mapping is None:
raise ValueError("目标普通知识库不存在")
sync = getattr(self.app.state, "llmwiki_sync_service", None)
if sync is None:
raise ValueError("请先配置 Wiki 编码模型并启用本地向量索引;归纳需要完成向量化后同步知识梳理")
client = build_weknora_client(get_resolved_llmwiki_runtime(self.app.state.config))
remote_id = mapping["weknora_id"]
remote = await client.get_knowledge_base(remote_id)
if not (remote.get("indexing_strategy") or {}).get("wiki_enabled"):
raise ValueError("目标普通知识库尚未开启 Wiki")
artifacts = []
document_ids = []
await self.store.update_item(str(item["id"]), phase="writing_sources", progress=55)
for file in scan.files:
if await self._cancel_if_requested(item):
return
if not file.text:
continue
document = await client.create_manual_document(remote_id, title=f"{ir['skill'].get('display_name') or item['skill_name']} · {file.path}", content=file.text)
doc_id = str(document.get("id") or document.get("knowledge_id") or "")
if not doc_id:
raise ValueError("资料上传未返回文档标识")
document_ids.append(doc_id)
artifacts.append({"kind": "raw_source", "slug": file.path, "remote_id": doc_id})
if not document_ids:
raise ValueError("该技能没有可归纳的知识正文,代码、脚本和代码块不参与归纳")
await self.store.update_item(str(item["id"]), phase="generating_wiki", progress=65)
# WeKnora performs model extraction, categorization and linking. Wait
# for both parsing and Wiki's debounced finalize before vectorization.
stable = 0
for _ in range(720):
if await self._cancel_if_requested(item):
return
docs = [await client.get_document(doc_id) for doc_id in document_ids]
if any(doc.get("parse_status") in {"failed", "error"} for doc in docs):
raise RuntimeError("WeKnora 资料解析失败,请查看普通知识库中的文档状态")
stats = await client.get_wiki_stats(remote_id)
done = all(doc.get("parse_status") == "completed" for doc in docs) and not stats.get("is_active") and not stats.get("pending_tasks")
stable = stable + 1 if done else 0
if stable >= 3:
if not int(stats.get("total_pages") or 0):
raise RuntimeError("WeKnora 解析已结束,但未生成 Wiki;请检查 Wiki 生成模型的调用错误、余额及资料是否含实质知识,再重试")
break
await asyncio.sleep(5)
else:
raise TimeoutError("Wiki 分析尚未完成;请检查普通知识库的生成任务后重新归纳")
await self.store.update_item(str(item["id"]), phase="vectorizing_wiki", progress=80)
result = await sync.sync_mapping(mapping, force=False, notify=False)
if result.get("status") != "completed":
raise RuntimeError("Wiki 向量化未全部完成,请检查向量化进度")
await self.store.update_item(str(item["id"]), phase="syncing_assistant", progress=92)
from app.gateway.knowledge_transfer import sync_to_global
deposited = await sync_to_global(self.app, mapping)
artifacts.append({"kind": "assistant_deposit", "slug": "knowledge-summary", "metadata": deposited})
await self._assert_source_unchanged(item, str(ir["skill"]["source_digest"]))
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status="synced")
+30 -7
View File
@@ -458,7 +458,11 @@ auth_login:
database:
backend: sqlite
sqlite_dir: .deer-flow/data
pool_size: 20 # 基础连接池(postgres/mysql 生效;sqlite 忽略)
# 高斯/OpenGauss 业务库示例(启用前先执行 uv sync --extra gauss):
# backend: gauss
# gauss_url: $GAUSS_DATABASE_URL # opengauss+asyncpg://user:pass@host:26000/deerflow
# checkpointer 请继续使用下方 sqlite;不要把 PostgreSQL 专用 checkpointer 迁移直接用于高斯。
pool_size: 20 # 基础连接池(postgres/gauss/mysql 生效;sqlite 忽略)
max_overflow: 20 # 高峰期可临时超出 pool_size 的额外连接数,真实上限 = pool_size + max_overflow
pool_timeout: 30 # 池满时等待空闲连接的秒数,超时报错
pool_recycle: 1800 # 连接回收周期(秒);防火墙/pgbouncer/MySQL wait_timeout 静默断连兜底。多 worker 部署:每 worker 上限 = pool_size + max_overflow,总数 × worker 数 < DB max_connections
@@ -585,17 +589,36 @@ llmwiki:
web_base_url: "http://119.254.86.251:8088"
admin_email: "lqq@qq.com"
admin_password: "1qaz@WSX"
# 本地 Wiki 向量化:只需把 enabled 改为 true,并填写下面 3 个 embedding 值。
# 默认只在页面点击“向量化”时执行;auto_sync=true 才会后台自动增量同步,
# 且服务启动时只启动定时器,不会立刻全量向量化。
# 高风险能力,默认关闭。连接参数仅允许后端配置;生产环境请使用环境变量。
graph:
direct_write_enabled: false
neo4j_http_url: ""
username: ""
password: ""
database: "neo4j"
timeout_seconds: 30
# 本地 Wiki 向量化:用于普通知识库“下载向量化数据包”和助手知识库向量检索。
# 已有相同内容哈希 + 相同 embedding fingerprint 的 Wiki 向量会直接复用,不会重复编码。
# local 模式使用 CPU BGE-M3;离线包必须包含 fastembed,并提前放置完整 ONNX 权重。
# 启动后先补扫一次,此后按间隔自动增量向量化并沉淀到“知识梳理总库”。
local_wiki_index:
enabled: false
enabled: true
# 免登录 POST /api/knowledge/vector-search 返回的 Wiki 一键跳转链接前缀。
frontend_base_url: "http://127.0.0.1:5174"
auto_sync: false
auto_sync: true
sync_interval_seconds: 30
database_batch_size: 50
embedding:
model: bge-m3
provider: local
model: bge-embedding-m3
dimensions: 1024
batch_size: 2
local_threads: 2
# 独立子进程承载大模型内存;编码器异常退出不会带停 DeerFlow 主服务。
local_process_isolation: true
# 离线部署必须指向提前放置好的完整 BGE-M3 模型目录;也可设置
# DEERFLOW_BGE_M3_MODEL_PATH。系统不会联网下载模型。
local_model_path: ""
base_url: ""
api_key: ""
@@ -109,8 +109,8 @@ def _filter_available_skills_for_excluded_tools(
available_skills = []
return [name for name in available_skills if name.strip().lower() not in blocked]
def _log_es_query_routing_registration(
def _log_es_query_routing_registration(
*,
system_prompt: str,
app_config: AppConfig,
@@ -0,0 +1,5 @@
"""Assistant knowledge package helpers."""
from deerflow.assistant_knowledge.importer import export_package_from_skill_ir
__all__ = ["export_package_from_skill_ir"]
@@ -0,0 +1,79 @@
"""Disk-backed, replayable package records. Memory is bounded by one JSONL record.
The staging index is local SQLite, never a remote provider's database. It also
allows restoration to fetch only the vectors belonging to the current page.
"""
from __future__ import annotations
import json
import sqlite3
import tempfile
from collections.abc import Iterator
from contextlib import closing
from pathlib import Path
from typing import Any
class Records:
def __init__(self, path: Path, kind: str, slug: str | None = None):
self.path, self.kind, self.slug = path, kind, slug
def _where(self):
return ("kind=?", (self.kind,)) if self.slug is None else ("kind=? AND slug=?", (self.kind, self.slug))
def __iter__(self) -> Iterator[dict[str, Any]]:
where, args = self._where()
with closing(sqlite3.connect(self.path)) as db:
for (payload,) in db.execute(f"SELECT payload FROM records WHERE {where} ORDER BY seq", args):
yield json.loads(payload)
def __len__(self) -> int:
where, args = self._where()
with closing(sqlite3.connect(self.path)) as db:
return db.execute(f"SELECT count(*) FROM records WHERE {where}", args).fetchone()[0]
class PackageArchive(dict):
def __init__(self, path: Path):
super().__init__()
temporary = tempfile.NamedTemporaryFile(prefix="wiki-package-", suffix=".sqlite", dir=path.parent, delete=False)
self.path = Path(temporary.name)
temporary.close()
try:
with closing(sqlite3.connect(self.path)) as db, path.open(encoding="utf-8-sig") as handle:
db.execute("CREATE TABLE records(seq INTEGER PRIMARY KEY, kind TEXT, slug TEXT, payload TEXT)")
for line_no, line in enumerate(handle, 1):
if not line.strip():
continue
record = json.loads(line)
kind, payload = record.get("record_type"), record.get("payload")
if not isinstance(payload, dict):
raise ValueError(f"导出包第 {line_no} 行格式错误")
if kind == "source":
self["source"] = {**self.get("source", {}), **payload}
elif kind == "graph":
for key, subkind in (("nodes", "graph_node"), ("edges", "graph_edge")):
db.executemany("INSERT INTO records(kind, payload) VALUES (?, ?)", [(subkind, json.dumps(v, ensure_ascii=False)) for v in payload.get(key) or []])
elif kind in {"wiki_page", "document", "chunk", "vector", "entity", "relation"}:
slug = str(payload.get("wiki_slug") or payload.get("page_slug") or payload.get("slug") or "").strip("/")
db.execute("INSERT INTO records(kind, slug, payload) VALUES (?, ?, ?)", (kind, slug, json.dumps(payload, ensure_ascii=False)))
else:
raise ValueError(f"不支持的导出记录类型:{kind}")
db.execute("CREATE INDEX by_kind_slug ON records(kind, slug)")
db.commit()
source = self.get("source", {})
if source.get("package_version") != 1 or (source.get("provider") or source.get("type")) not in {"weknora", "assistant"}:
raise ValueError("仅接受 WeKnora / 助手知识库生成的兼容 Wiki 向量包(v1)")
for key, kind in (("wiki_pages", "wiki_page"), ("documents", "document"), ("chunks", "chunk"), ("vectors", "vector"), ("entities", "entity"), ("relations", "relation")):
self[key] = Records(self.path, kind)
self["graph"] = {"nodes": Records(self.path, "graph_node"), "edges": Records(self.path, "graph_edge")}
except BaseException:
self.close()
raise
def vectors_for(self, slug: str) -> Records:
return Records(self.path, "vector", slug)
def close(self):
self.path.unlink(missing_ok=True)
@@ -0,0 +1,22 @@
"""Normalize heterogeneous sources into the assistant import-package shape."""
from __future__ import annotations
from typing import Any
def export_package_from_skill_ir(ir: dict[str, Any]) -> dict[str, Any]:
skill = ir.get("skill") or {}
return {
"source": {
"type": "skill",
"skill_name": skill.get("name"),
"source_digest": skill.get("source_digest"),
},
"wiki_pages": [ir.get("wiki", {}).get("skill_page")],
"documents": ir.get("files") or [],
"chunks": [],
"entities": ir.get("entities") or [],
"relations": ir.get("relations") or [],
"graph": {"nodes": ir.get("entities") or [], "edges": ir.get("relations") or []},
}
@@ -0,0 +1,109 @@
"""File-backed assistant knowledge import/export packages.
The assistant knowledge base is fed by WeKnora-exported data. Keep that
boundary explicit: a WeKnora export is first materialized as a durable JSONL
package, then imported asynchronously from that package. JSONL keeps large
exports append-friendly and avoids treating the HTTP request body as the job
state.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from pathlib import Path
from typing import Any
PACKAGE_VERSION = 1
PACKAGE_RECORD_TYPES = {
"source",
"wiki_page",
"document",
"chunk",
"vector",
"entity",
"relation",
"graph",
}
def package_path(root: Path, job_id: str) -> Path:
safe = "".join(ch for ch in str(job_id) if ch.isalnum() or ch in {"-", "_"})
return root / f"{safe}.jsonl"
def write_jsonl_record(handle: Any, record_type: str, payload: Mapping[str, Any]) -> None:
handle.write(json.dumps({"record_type": record_type, "payload": dict(payload)}, ensure_ascii=False))
handle.write("\n")
def write_package(path: Path, records: Iterable[tuple[str, Mapping[str, Any]]]) -> dict[str, int]:
path.parent.mkdir(parents=True, exist_ok=True)
counts: dict[str, int] = {key: 0 for key in PACKAGE_RECORD_TYPES}
tmp_path = path.with_suffix(path.suffix + ".tmp")
with tmp_path.open("w", encoding="utf-8", newline="\n") as handle:
write_jsonl_record(
handle,
"source",
{
"package_version": PACKAGE_VERSION,
},
)
counts["source"] += 1
for record_type, payload in records:
if record_type not in PACKAGE_RECORD_TYPES:
continue
write_jsonl_record(handle, record_type, payload)
counts[record_type] = counts.get(record_type, 0) + 1
tmp_path.replace(path)
return counts
def read_package(path: Path) -> dict[str, Any]:
package: dict[str, Any] = {
"source": {},
"wiki_pages": [],
"documents": [],
"chunks": [],
"vectors": [],
"entities": [],
"relations": [],
"graph": {"nodes": [], "edges": []},
}
with path.open("r", encoding="utf-8") as handle:
for line in handle:
line = line.strip()
if not line:
continue
record = json.loads(line)
if not isinstance(record, dict):
continue
record_type = str(record.get("record_type") or "")
payload = record.get("payload")
if not isinstance(payload, dict):
continue
if record_type == "source":
package["source"] = {**package.get("source", {}), **payload}
elif record_type == "wiki_page":
package["wiki_pages"].append(payload)
elif record_type == "document":
package["documents"].append(payload)
elif record_type == "chunk":
package["chunks"].append(payload)
elif record_type == "vector":
package["vectors"].append(payload)
elif record_type == "entity":
package["entities"].append(payload)
elif record_type == "relation":
package["relations"].append(payload)
elif record_type == "graph":
nodes = payload.get("nodes")
edges = payload.get("edges")
if isinstance(nodes, list):
package["graph"]["nodes"].extend(item for item in nodes if isinstance(item, dict))
if isinstance(edges, list):
package["graph"]["edges"].extend(item for item in edges if isinstance(item, dict))
for key, value in payload.items():
if key not in {"nodes", "edges"}:
package["graph"][key] = value
return package
@@ -11,10 +11,15 @@ blocking, making a unified file safe for both workloads. Writers
that contend for the lock wait via the default 5-second sqlite3
busy timeout rather than failing immediately.
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
Postgres mode: both use the same database URL but maintain independent
connection pools with different lifecycles.
Gauss mode: application ORM data uses the official openGauss SQLAlchemy
dialect over asyncpg. LangGraph's PostgreSQL checkpointer is intentionally not
reused for GaussDB; configure the legacy ``checkpointer:`` section separately
(normally sqlite).
MySQL mode: application ORM data uses MySQL. LangGraph's checkpointer does
not have a built-in MySQL backend in this project, so deployments that choose
MySQL for business tables should configure the legacy ``checkpointer:``
section separately (usually sqlite) to preserve conversation checkpoints.
@@ -42,24 +47,40 @@ from typing import Literal
from pydantic import BaseModel, Field
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "mysql"] = Field(
default="memory",
description=("Storage backend for application data. 'memory' for development (no persistence across restarts), 'sqlite' for single-node deployment, 'postgres' for production multi-node deployment, 'mysql' for MySQL-backed business tables."),
class DatabaseConfig(BaseModel):
backend: Literal["memory", "sqlite", "postgres", "gauss", "mysql"] = Field(
default="memory",
description=(
"Storage backend for application data. 'memory' for development "
"(no persistence across restarts), 'sqlite' for single-node "
"deployment, 'postgres' for production multi-node deployment, "
"'gauss' for openGauss/GaussDB business tables, and 'mysql' for "
"MySQL-backed business tables."
),
)
sqlite_dir: str = Field(
default=".deer-flow/data",
description=("Directory for the SQLite database file. Both checkpointer and application data share {sqlite_dir}/deerflow.db."),
)
postgres_url: str = Field(
postgres_url: str = Field(
default="",
description=(
"PostgreSQL connection URL, shared by checkpointer and app. "
"Use $DATABASE_URL in config.yaml to reference .env. "
"Example: postgresql://user:pass@host:5432/deerflow "
"(the +asyncpg driver suffix is added automatically where needed)."
),
)
),
)
gauss_url: str = Field(
default="",
description=(
"openGauss/GaussDB connection URL for the application ORM. "
"Use $GAUSS_DATABASE_URL in config.yaml to reference .env. "
"Example: opengauss+asyncpg://user:pass@host:26000/deerflow. "
"Plain opengauss:// or PostgreSQL-style URLs are normalized to the "
"official asynchronous openGauss SQLAlchemy dialect."
),
)
mysql_url: str = Field(
default="",
description=(
@@ -75,26 +96,26 @@ class DatabaseConfig(BaseModel):
)
pool_size: int = Field(
default=5,
description="Connection pool size for the app ORM engine (postgres/mysql only).",
description="Connection pool size for the app ORM engine (postgres/gauss/mysql only).",
)
max_overflow: int = Field(
default=10,
description=(
"Extra connections allowed beyond pool_size under load (postgres/mysql only). "
"Extra connections allowed beyond pool_size under load (postgres/gauss/mysql only). "
"The real connection ceiling is pool_size + max_overflow. SQLAlchemy default is 10."
),
)
pool_timeout: int = Field(
default=30,
description=(
"Seconds a caller waits for a free pooled connection before erroring (postgres/mysql only). "
"Seconds a caller waits for a free pooled connection before erroring (postgres/gauss/mysql only). "
"SQLAlchemy default is 30."
),
)
pool_recycle: int = Field(
default=1800,
description=(
"Recycle pooled connections after this many seconds (postgres/mysql only). "
"Recycle pooled connections after this many seconds (postgres/gauss/mysql only). "
"Prevents stale-connection errors when a firewall, pgbouncer, or MySQL "
"wait_timeout silently drops idle connections. pool_pre_ping catches most "
"cases, but pool_recycle is the belt-and-suspenders safety net. "
@@ -133,12 +154,38 @@ class DatabaseConfig(BaseModel):
"""SQLAlchemy async URL for the application ORM engine."""
if self.backend == "sqlite":
return f"sqlite+aiosqlite:///{self.sqlite_path}"
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "mysql":
if self.backend == "postgres":
url = self.postgres_url
if url.startswith("postgresql://"):
url = url.replace("postgresql://", "postgresql+asyncpg://", 1)
return url
if self.backend == "gauss":
url = self.gauss_url.strip()
if not url:
raise ValueError("database.gauss_url is required for backend='gauss'")
# GaussDB speaks the PostgreSQL wire protocol, but its server
# version string and DDL capabilities are not identical to
# PostgreSQL. Always select the official openGauss dialect instead
# of silently sending the URL through SQLAlchemy's generic PG
# dialect (the latter commonly connects and then fails while
# SQLAlchemy initializes/creates the schema).
for prefix in (
"opengauss://",
"gaussdb://",
"postgres://",
"postgresql://",
"postgresql+asyncpg://",
):
if url.startswith(prefix):
return "opengauss+asyncpg://" + url[len(prefix) :]
if url.startswith("opengauss+asyncpg://"):
return url
raise ValueError(
"database.gauss_url must use opengauss://, "
"opengauss+asyncpg://, or a PostgreSQL-style URL"
)
if self.backend == "mysql":
url = self.mysql_url
if not url:
raise ValueError("database.mysql_url is required for backend='mysql'")
@@ -44,6 +44,7 @@ class WeKnoraConnectionConfig(BaseModel):
repr=False,
description=("Optional WeKnora administrator password used by DeerFlow's server-side WeKnora API client and detail-page iframe proxy. Prefer $ENV references."),
)
graph: WeKnoraDirectGraphConfig = Field(default_factory=lambda: WeKnoraDirectGraphConfig())
model_config = ConfigDict(extra="ignore")
@field_validator("api_base_url", "web_base_url")
@@ -52,12 +53,33 @@ class WeKnoraConnectionConfig(BaseModel):
return normalize_service_url(value)
class WeKnoraDirectGraphConfig(BaseModel):
"""Backend-only, opt-in Neo4j transactional HTTP writer settings."""
direct_write_enabled: bool = False
neo4j_http_url: str = ""
username: str = ""
password: str = Field(default="", repr=False)
database: str = Field(default="neo4j", pattern=r"^[A-Za-z0-9_.-]+$")
timeout_seconds: float = Field(default=30, gt=0, le=300)
model_config = ConfigDict(extra="ignore")
@field_validator("neo4j_http_url")
@classmethod
def _validate_url(cls, value: str) -> str:
return normalize_service_url(value)
class LocalWikiEmbeddingConfig(BaseModel):
enabled: bool = True
provider: str = "openai"
model: str = "bge-m3"
model: str = "bge-embedding-m3"
base_url: str = ""
api_key: str = Field(default="", repr=False)
local_model_path: str = ""
local_cache_dir: str = ""
local_process_isolation: bool = True
local_threads: int = Field(default=2, ge=1, le=32)
dimensions: int | None = Field(
default=None,
ge=1,
@@ -73,8 +95,8 @@ class LocalWikiEmbeddingConfig(BaseModel):
@classmethod
def _validate_provider(cls, value: str) -> str:
normalized = value.strip().lower()
if normalized not in {"openai", "openai_compatible"}:
raise ValueError("local Wiki embedding provider must be openai or openai_compatible")
if normalized not in {"openai", "openai_compatible", "local"}:
raise ValueError("Wiki embedding provider must be openai, openai_compatible or local")
return normalized
@field_validator("base_url")
@@ -112,8 +134,8 @@ class LocalWikiIndexConfig(BaseModel):
default="http://127.0.0.1:5174",
description="DeerFlow frontend base URL used to build directly usable Wiki links in public vector-search results.",
)
auto_sync: bool = False
sync_interval_seconds: int = Field(default=120, ge=10, le=86400)
auto_sync: bool = True
sync_interval_seconds: int = Field(default=30, ge=10, le=86400)
sync_page_size: int = Field(default=100, ge=1, le=500)
sync_concurrency: int = Field(default=3, ge=1, le=32)
sync_lease_seconds: int = Field(default=600, ge=30, le=86400)
@@ -180,7 +202,7 @@ class LocalWikiIndexConfig(BaseModel):
raise ValueError("Only the numpy_exact local Wiki vector engine is currently supported")
if self.enabled and not self.embedding.enabled:
raise ValueError("local Wiki index embedding must be enabled when the index is enabled")
if self.enabled and (not self.embedding.base_url or not self.embedding.model.strip()):
if self.enabled and ((not self.embedding.base_url and self.embedding.provider != "local") or not self.embedding.model.strip()):
raise ValueError("local Wiki index embedding base_url and model are required when enabled")
return self
@@ -160,6 +160,18 @@ class PromptPrefixSettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class SkillKnowledgeAutoSyncSettings(BaseModel):
"""Watcher/scheduler policy for bindings that explicitly opt in."""
watch_enabled: bool = True
schedule_enabled: bool = True
schedule_cron: str = Field(default="0 */6 * * *", min_length=5, max_length=128)
debounce_seconds: int = Field(default=120, ge=10, le=86_400)
write_remote_enabled: bool = True
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
model_config = ConfigDict(extra="ignore")
class AppearanceRouteParam(BaseModel):
"""Runtime query-param descriptor for top-bar shortcut buttons.
@@ -493,6 +505,17 @@ class ModelConcurrencySettings(BaseModel):
model_config = ConfigDict(extra="ignore")
class LocalWikiEncoderSettings(BaseModel):
"""Runtime override for the administrator-managed offline Wiki encoder."""
model_path: str = Field(
default="",
max_length=4096,
description="Absolute or project-relative directory containing the offline BGE-M3 ONNX model.",
)
model_config = ConfigDict(extra="ignore")
class SystemSettings(BaseModel):
"""Top-level system settings document."""
@@ -508,7 +531,13 @@ class SystemSettings(BaseModel):
agent_modules: AgentModuleSettings = Field(default_factory=AgentModuleSettings)
concurrency_monitor: ConcurrencyMonitorSettings = Field(default_factory=ConcurrencyMonitorSettings)
model_concurrency: ModelConcurrencySettings = Field(default_factory=ModelConcurrencySettings)
local_wiki_encoder: LocalWikiEncoderSettings = Field(default_factory=LocalWikiEncoderSettings)
llmwiki: LlmWikiRuntimeOverride = Field(default_factory=LlmWikiRuntimeOverride)
skill_knowledge_auto_sync: SkillKnowledgeAutoSyncSettings = Field(default_factory=SkillKnowledgeAutoSyncSettings)
knowledge_base_product_version: Literal["v1", "v2"] = Field(
default="v1",
description="v1 opens the ordinary WeKnora experience; v2 opens the DeerFlow assistant knowledge base.",
)
model_config = ConfigDict(extra="ignore")
@@ -494,13 +494,28 @@ class WeKnoraClient:
await self._request(
"GET",
f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/graph",
params={"mode": "overview", "limit": max(1, min(limit, 500))},
params={"mode": "overview", "limit": max(1, min(limit, 10000))},
)
)
if not isinstance(value, dict):
raise WeKnoraError("WeKnora returned an invalid wiki graph")
return dict(value)
async def get_wiki_stats(self, knowledge_base_id: str) -> dict[str, Any]:
value = _data(await self._request("GET", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/stats"))
if not isinstance(value, dict):
raise WeKnoraError("Invalid Wiki generation status")
return value
async def create_wiki_folder(self, knowledge_base_id: str, *, name: str, parent_id: str = "") -> dict[str, Any]:
value = _data(await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/folders", json={"name": name, "parent_id": parent_id}))
if not isinstance(value, dict) or not value.get("id"):
raise WeKnoraError("Invalid Wiki folder response")
return value
async def rebuild_wiki_links(self, knowledge_base_id: str) -> None:
await self._request("POST", f"/api/v1/knowledgebase/{quote(knowledge_base_id, safe='')}/wiki/rebuild-links")
async def list_wiki_pages(
self,
knowledge_base_id: str,
@@ -1,7 +1,37 @@
"""DeerFlow-local mirror and Wiki-only retrieval for WeKnora Wiki pages."""
from deerflow.integrations.weknora.local_index.retrieval import WikiRetrievalService, is_fully_vectorized
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
from __future__ import annotations
__all__ = ["WikiIndexError", "WikiRetrievalService", "WikiVectorSearchService", "WikiSyncService", "is_fully_vectorized"]
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from deerflow.integrations.weknora.local_index.retrieval import (
WikiRetrievalService,
is_fully_vectorized,
)
from deerflow.integrations.weknora.local_index.search import WikiIndexError, WikiVectorSearchService
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
__all__ = [
"WikiIndexError",
"WikiRetrievalService",
"WikiVectorSearchService",
"WikiSyncService",
"is_fully_vectorized",
]
def __getattr__(name: str) -> Any:
if name in {"WikiRetrievalService", "is_fully_vectorized"}:
from deerflow.integrations.weknora.local_index import retrieval
return getattr(retrieval, name)
if name in {"WikiIndexError", "WikiVectorSearchService"}:
from deerflow.integrations.weknora.local_index import search
return getattr(search, name)
if name == "WikiSyncService":
from deerflow.integrations.weknora.local_index.sync import WikiSyncService
return WikiSyncService
raise AttributeError(name)
@@ -105,4 +105,12 @@ def chunk_wiki_page(
context.append(f"章节:{heading}")
embedding_text = "\n".join([*context, "", chunk]).strip()
result.append(WikiSection(len(result), heading, chunk, embedding_text, digest))
# A short article with a short summary must not disappear merely because
# both sections fall below min_chars. Keep one combined, useful vector.
if not result and (content or summary):
fallback = "\n\n".join(dict.fromkeys(text for text in (summary, content) if text))
normalized = re.sub(r"\s+", " ", fallback).strip()
digest = hashlib.sha256(normalized.encode("utf-8")).hexdigest()
prefix = prefix_lines if title_prefix_enabled else []
result.append(WikiSection(0, None, fallback, "\n".join([*prefix, "", fallback]).strip(), digest))
return result
@@ -4,7 +4,13 @@ from __future__ import annotations
import asyncio
import hashlib
import json
import logging
import os
import subprocess
import sys
from datetime import UTC, datetime
from pathlib import Path
from time import monotonic
import httpx
@@ -43,6 +49,14 @@ class StrictWikiEmbeddingClient:
self._client = http_client
self._owns_client = http_client is None
self._resolved_dimensions = config.dimensions
self._local_model = None
self._local_process: subprocess.Popen[str] | None = None
self._local_lock = asyncio.Lock()
self._local_state = "idle"
self._local_error: str | None = None
self._local_started_at: str | None = None
self._local_ready_at: str | None = None
self._local_warmup_task: asyncio.Task[None] | None = None
@property
def dimensions(self) -> int | None:
@@ -54,16 +68,57 @@ class StrictWikiEmbeddingClient:
return hashlib.sha256(raw.encode("utf-8")).hexdigest()
async def close(self) -> None:
await self.reset_local_encoder()
if self._owns_client and self._client is not None:
await self._client.aclose()
self._client = None
async def reset_local_encoder(self, *, model_path: str | None = None) -> None:
"""Stop only the embedded worker and optionally switch its model path.
This is intentionally independent from the Gateway lifecycle: an
administrator can replace the offline model without restarting or
stopping the rest of DeerFlow.
"""
warmup = self._local_warmup_task
if warmup is not None and not warmup.done():
warmup.cancel()
await asyncio.gather(warmup, return_exceptions=True)
self._local_warmup_task = None
process = self._local_process
self._local_process = None
if process is not None:
await asyncio.to_thread(self._stop_local_process, process)
self._local_model = None
self._local_state = "idle"
self._local_error = None
self._local_started_at = None
self._local_ready_at = None
if model_path is not None:
self.config = self.config.model_copy(update={"local_model_path": model_path})
async def _get_client(self) -> httpx.AsyncClient:
if self._client is None:
self._client = httpx.AsyncClient(timeout=self.config.timeout_seconds)
return self._client
async def embed_texts(self, texts: list[str]) -> list[list[float]]:
if self.config.provider == "local":
if not texts:
return []
async with self._local_lock:
self._local_state = "starting"
self._local_started_at = self._local_started_at or datetime.now(UTC).isoformat()
try:
values = await asyncio.to_thread(self._embed_local, texts)
except Exception as exc:
self._local_state = "failed"
self._local_error = " ".join(str(exc).split())[:1000] or type(exc).__name__
raise
self._local_state = "ready"
self._local_error = None
self._local_ready_at = datetime.now(UTC).isoformat()
return values
if not self.config.enabled or not self.config.base_url or not self.config.model:
raise WikiEmbeddingError("Wiki embedding is not configured", category="configuration", retryable=False)
if not texts:
@@ -76,6 +131,214 @@ class StrictWikiEmbeddingClient:
raise WikiEmbeddingError("Embedding response count does not match request", category="response", retryable=False)
return vectors
def _embed_local(self, texts: list[str]) -> list[list[float]]:
if not self.config.enabled:
raise WikiEmbeddingError("本地编码已关闭", retryable=False)
model_path = self._validated_local_model_path()
if self.config.local_process_isolation:
return self._embed_local_process(texts, model_path)
return self._embed_local_in_process(texts, model_path)
def _validated_local_model_path(self) -> Path:
model_path = self._local_model_path()
if model_path is None:
raise WikiEmbeddingError(
"离线本地编码器目录未配置;请设置 local_model_path 或 DEERFLOW_BGE_M3_MODEL_PATH",
category="configuration",
retryable=False,
)
if not model_path.is_dir() or not (
(model_path / "onnx" / "model.onnx").is_file()
or (model_path / "model.onnx").is_file()
):
raise WikiEmbeddingError(
f"离线本地编码器目录无效或缺少 ONNX 模型:{model_path}",
category="configuration",
retryable=False,
)
return model_path
def _embed_local_in_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
if self._local_model is None:
try:
from fastembed import TextEmbedding
except ImportError as exc:
raise WikiEmbeddingError("标准后端依赖缺少 fastembed,请检查离线 wheelhouse 和安装结果", retryable=False) from exc
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
if name == "BAAI/bge-m3" and not any(m["model"] == name for m in TextEmbedding.list_supported_models()):
from fastembed.common.model_description import ModelSource, PoolingType
TextEmbedding.add_custom_model(
model=name,
pooling=PoolingType.CLS,
normalization=True,
dim=1024,
sources=ModelSource(hf="BAAI/bge-m3"),
model_file="onnx/model.onnx",
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
license="mit",
size_in_gb=2.29,
)
cache_dir = Path(self.config.local_cache_dir).expanduser() if self.config.local_cache_dir else Path.home() / ".cache" / "deerflow" / "embeddings"
options = {
"model_name": name,
"threads": self.config.local_threads,
"cache_dir": str(cache_dir),
"providers": ["CPUExecutionProvider"],
"specific_model_path": str(model_path),
"local_files_only": True,
}
self._local_model = TextEmbedding(**options)
values = []
for vector in self._local_model.embed(texts, batch_size=min(self.config.batch_size, 4)):
normalized = normalize_vector(vector, dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(normalized.size)
values.append(normalized.tolist())
if len(values) != len(texts):
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
return values
def _embed_local_process(self, texts: list[str], model_path: Path) -> list[list[float]]:
process = self._local_process
if process is None or process.poll() is not None:
process = self._start_local_process(model_path)
self._local_process = process
if process.stdin is None or process.stdout is None:
raise WikiEmbeddingError("本地编码器子进程通信通道不可用", retryable=False)
request_id = hashlib.sha256(f"{monotonic()}:{len(texts)}".encode()).hexdigest()[:16]
try:
process.stdin.write(
json.dumps({"id": request_id, "texts": texts}, ensure_ascii=False) + "\n"
)
process.stdin.flush()
response = self._read_local_process_message(process, request_id=request_id)
except (BrokenPipeError, OSError, ValueError, json.JSONDecodeError) as exc:
self._stop_local_process(process)
self._local_process = None
raise WikiEmbeddingError(
f"本地编码器子进程异常退出:{str(exc)[:500]}",
category="local_process",
retryable=True,
) from exc
if response.get("status") != "ok":
error = str(response.get("error") or "本地编码失败")[:1000]
if process.poll() is not None:
self._local_process = None
raise WikiEmbeddingError(error, category="local_process", retryable=False)
raw_vectors = response.get("vectors")
if not isinstance(raw_vectors, list) or len(raw_vectors) != len(texts):
raise WikiEmbeddingError("本地编码返回数量不符", retryable=False)
values: list[list[float]] = []
for raw in raw_vectors:
normalized = normalize_vector(raw, dimensions=self._resolved_dimensions)
if self._resolved_dimensions is None:
self._resolved_dimensions = int(normalized.size)
values.append(normalized.tolist())
return values
def _start_local_process(self, model_path: Path) -> subprocess.Popen[str]:
name = "BAAI/bge-m3" if self.config.model.lower() in {"bge-m3", "bge-embedding-m3"} else self.config.model
cache_dir = (
Path(self.config.local_cache_dir).expanduser()
if self.config.local_cache_dir
else Path.home() / ".cache" / "deerflow" / "embeddings"
)
command = [
sys.executable,
"-m",
"deerflow.integrations.weknora.local_index.embedding_worker",
"--model-name",
name,
"--model-path",
str(model_path),
"--cache-dir",
str(cache_dir),
"--threads",
str(self.config.local_threads),
"--batch-size",
str(min(self.config.batch_size, 4)),
]
environment = os.environ.copy()
environment.update(
{
"HF_HUB_OFFLINE": "1",
"TRANSFORMERS_OFFLINE": "1",
"HF_DATASETS_OFFLINE": "1",
"PYTHONIOENCODING": "utf-8",
"PYTHONUTF8": "1",
}
)
creation_flags = 0
if os.name == "nt":
creation_flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0)) | int(
getattr(subprocess, "BELOW_NORMAL_PRIORITY_CLASS", 0)
)
try:
process = subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
text=True,
encoding="utf-8",
bufsize=1,
env=environment,
creationflags=creation_flags,
)
response = self._read_local_process_message(process, ready=True)
except (OSError, ValueError, json.JSONDecodeError) as exc:
if "process" in locals():
self._stop_local_process(process)
raise WikiEmbeddingError(
f"无法启动本地编码器子进程:{str(exc)[:500]}",
category="local_process",
retryable=False,
) from exc
if response.get("status") != "ready":
error = str(response.get("error") or "本地编码器初始化失败")[:1000]
self._stop_local_process(process)
raise WikiEmbeddingError(error, category="local_process", retryable=False)
return process
@staticmethod
def _read_local_process_message(
process: subprocess.Popen[str],
*,
request_id: str | None = None,
ready: bool = False,
) -> dict[str, object]:
if process.stdout is None:
raise ValueError("本地编码器没有输出通道")
for _ in range(20):
line = process.stdout.readline()
if not line:
raise ValueError(f"本地编码器已退出(exit={process.poll()})")
payload = json.loads(line)
if not isinstance(payload, dict):
continue
if ready and payload.get("status") in {"ready", "error"}:
return payload
if request_id is not None and payload.get("id") == request_id:
return payload
raise ValueError("本地编码器返回了无法识别的响应")
@staticmethod
def _stop_local_process(process: subprocess.Popen[str]) -> None:
if process.poll() is not None:
return
try:
if process.stdin is not None:
process.stdin.write('{"command":"shutdown"}\n')
process.stdin.flush()
process.wait(timeout=2)
except Exception:
process.terminate()
try:
process.wait(timeout=2)
except Exception:
process.kill()
async def _embed_batch(self, texts: list[str]) -> list[list[float]]:
url = self.config.base_url.rstrip("/") + "/embeddings"
headers = {"Content-Type": "application/json"}
@@ -138,3 +401,80 @@ class StrictWikiEmbeddingClient:
async def embed_query(self, text: str) -> list[float]:
result = await self.embed_texts([text])
return result[0]
def local_encoder_status(self) -> dict[str, object]:
running = self._local_warmup_task is not None and not self._local_warmup_task.done()
process_ready = self._local_process is not None and self._local_process.poll() is None
process_failed = self._local_process is not None and not process_ready
state = (
"starting"
if running
else "ready"
if (self._local_model is not None or process_ready)
else "failed"
if process_failed
else self._local_state
)
last_error = self._local_error
if process_failed and not last_error:
last_error = f"本地编码器子进程已退出(exit={self._local_process.poll()})"
model_path = self._local_model_path()
model_files_ready = bool(
model_path
and model_path.is_dir()
and ((model_path / "onnx" / "model.onnx").is_file() or (model_path / "model.onnx").is_file())
)
return {
"provider": self.config.provider,
"model": self.config.model,
"dimensions": self.dimensions,
"state": state,
"running": running,
"last_error": last_error,
"started_at": self._local_started_at,
"ready_at": self._local_ready_at,
"offline_only": self.config.provider == "local",
"model_path_configured": model_path is not None,
"model_path_exists": bool(model_path and model_path.is_dir()),
"model_files_ready": model_files_ready,
"process_isolation": self.config.local_process_isolation,
"model_path": str(model_path) if model_path is not None else "",
}
def _local_model_path(self) -> Path | None:
raw = self.config.local_model_path.strip() or os.environ.get(
"DEERFLOW_BGE_M3_MODEL_PATH", ""
).strip()
if not raw:
return None
path = Path(raw).expanduser()
return path.resolve() if path.is_absolute() else (Path.cwd() / path).resolve()
def start_local_encoder(self) -> dict[str, object]:
"""Warm the embedded CPU encoder without coupling it to app startup."""
if self.config.provider != "local":
raise ValueError("当前 embedding provider 不是 local")
if not self.config.enabled:
raise ValueError("本地编码器未启用")
process_ready = self._local_process is not None and self._local_process.poll() is None
if self._local_model is not None or process_ready:
self._local_state = "ready"
return self.local_encoder_status()
if self._local_warmup_task is None or self._local_warmup_task.done():
self._local_state = "starting"
self._local_error = None
self._local_started_at = datetime.now(UTC).isoformat()
self._local_warmup_task = asyncio.create_task(
self._warm_local_encoder(),
name="llmwiki-local-encoder-warmup",
)
return self.local_encoder_status()
async def _warm_local_encoder(self) -> None:
try:
await self.embed_query("本地编码器启动检查")
except asyncio.CancelledError:
self._local_state = "idle"
raise
except Exception:
logger.exception("Local Wiki encoder warmup failed model=%s", self.config.model)
@@ -0,0 +1,98 @@
"""Offline FastEmbed worker isolated from the DeerFlow gateway process.
The protocol is UTF-8 JSON Lines over stdin/stdout. Model weights must already
exist on disk; every offline flag is enforced before FastEmbed is imported.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
def _write(payload: dict[str, object]) -> None:
sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n")
sys.stdout.flush()
def _build_model(args: argparse.Namespace):
os.environ["HF_HUB_OFFLINE"] = "1"
os.environ["TRANSFORMERS_OFFLINE"] = "1"
os.environ["HF_DATASETS_OFFLINE"] = "1"
from fastembed import TextEmbedding
if args.model_name == "BAAI/bge-m3" and not any(
item["model"] == args.model_name for item in TextEmbedding.list_supported_models()
):
from fastembed.common.model_description import ModelSource, PoolingType
TextEmbedding.add_custom_model(
model=args.model_name,
pooling=PoolingType.CLS,
normalization=True,
dim=1024,
sources=ModelSource(hf="BAAI/bge-m3"),
model_file="onnx/model.onnx",
additional_files=["onnx/model.onnx_data", "onnx/Constant_7_attr__value"],
license="mit",
size_in_gb=2.29,
)
return TextEmbedding(
model_name=args.model_name,
threads=args.threads,
cache_dir=args.cache_dir,
providers=["CPUExecutionProvider"],
specific_model_path=args.model_path,
local_files_only=True,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--model-name", required=True)
parser.add_argument("--model-path", required=True)
parser.add_argument("--cache-dir", required=True)
parser.add_argument("--threads", type=int, default=2)
parser.add_argument("--batch-size", type=int, default=2)
args = parser.parse_args()
model_path = Path(args.model_path)
if not model_path.is_dir():
_write({"status": "error", "error": f"离线模型目录不存在:{model_path}"})
return 2
try:
model = _build_model(args)
except Exception as exc: # noqa: BLE001
_write({"status": "error", "error": f"离线模型加载失败:{type(exc).__name__}: {exc}"})
return 2
_write({"status": "ready"})
for line in sys.stdin:
request: dict[str, object] = {}
try:
request = json.loads(line)
if request.get("command") == "shutdown":
return 0
request_id = str(request.get("id") or "")
texts = request.get("texts")
if not request_id or not isinstance(texts, list) or not all(isinstance(value, str) for value in texts):
raise ValueError("编码请求格式无效")
vectors = [
value.tolist()
for value in model.embed(texts, batch_size=max(1, min(args.batch_size, 4)))
]
_write({"id": request_id, "status": "ok", "vectors": vectors})
except Exception as exc: # noqa: BLE001
_write(
{
"id": str(request.get("id") or "") if isinstance(request, dict) else "",
"status": "error",
"error": f"本地编码失败:{type(exc).__name__}: {exc}",
}
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
@@ -36,8 +36,9 @@ class WikiSyncService:
self.cache = cache
self.client = client
self.owner = f"{platform.node()}:{uuid4()}"
self.on_completed = None
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False) -> dict[str, Any]:
async def sync_mapping(self, mapping: dict[str, Any], *, force: bool = False, notify: bool = True) -> dict[str, Any]:
mapping_id = str(mapping["id"])
sync_id = str(uuid4())
if self.embedding.dimensions is None:
@@ -111,6 +112,7 @@ class WikiSyncService:
and existing.get("content_hash") == page["content_hash"]
and existing.get("indexed_fingerprint") == active_fingerprint
and existing.get("index_status") == "ready"
and (existing.get("vector_count", 1) > 0 or page["status"] == "archived" or not (page["content_md"] or page["summary"]))
and not existing.get("is_remote_deleted")
):
await self.store.touch_page(str(existing["id"]), sync_id)
@@ -160,7 +162,13 @@ class WikiSyncService:
remote_page_count=remote_count,
error=f"{failed} page(s) failed" if failed else None,
)
return {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
result = {"mapping_id": mapping_id, "status": "partial" if failed else "completed", "remote_pages": remote_count, "changed_pages": changed, "deleted_pages": deleted, "failed_pages": failed}
# Notify even when the remote library is now empty. The global
# assistant base must then remove stale contributions from this
# source instead of retaining deleted Wiki pages forever.
if not failed and notify and self.on_completed:
result["assistant_sync"] = await self.on_completed(mapping)
return result
except asyncio.CancelledError:
await self.store.finish_sync(
mapping_id,
@@ -0,0 +1,4 @@
from deerflow.persistence.assistant_knowledge.model import * # noqa: F401,F403
from deerflow.persistence.assistant_knowledge.store import AssistantKnowledgeRepository, make_assistant_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("AssistantKnowledge") or name == "make_assistant_knowledge_store"]
@@ -0,0 +1,175 @@
"""ORM model for DeerFlow-native assistant knowledge bases."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Float, Index, Integer, LargeBinary, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class AssistantKnowledgeBaseRow(Base):
__tablename__ = "assistant_knowledge_base"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
name: Mapped[str] = mapped_column(String(255), default="知识梳理")
description: Mapped[str | None] = mapped_column(String(2048), nullable=True)
# ``global`` is the full-library aggregation base. ``custom`` bases are
# user-created assistant knowledge bases fed from WeKnora export packages.
# Nullable keeps startup column reconciliation safe for existing databases.
base_kind: Mapped[str | None] = mapped_column(String(24), nullable=True, index=True)
status: Mapped[str] = mapped_column(String(64), default="active")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeSourceRow(Base):
__tablename__ = "assistant_knowledge_sources"
__table_args__ = (UniqueConstraint("base_id", "source_type", "source_key", name="uq_assistant_knowledge_source"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_type: Mapped[str] = mapped_column(String(32), index=True)
source_key: Mapped[str] = mapped_column(String(255))
knowledge_base_mapping_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
upload_batch_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_name: Mapped[str] = mapped_column(String(255), default="")
last_import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeImportJobRow(Base):
__tablename__ = "assistant_knowledge_import_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
trigger: Mapped[str] = mapped_column(String(48), default="manual_from_weknora")
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="exporting")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeWikiPageRow(Base):
__tablename__ = "assistant_knowledge_wiki_pages"
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
__table_args__ = (UniqueConstraint("base_id", "slug", name="uq_assistant_wiki_slug"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
slug: Mapped[str] = mapped_column(String(512), index=True)
title: Mapped[str] = mapped_column(String(512), index=True)
page_type: Mapped[str] = mapped_column(String(32), index=True)
current_revision_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
search_text: Mapped[str] = mapped_column(PortableLongText(), default="")
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class AssistantKnowledgeWikiRevisionRow(Base):
__tablename__ = "assistant_knowledge_wiki_revisions"
__table_args__ = (UniqueConstraint("page_id", "revision_no", name="uq_assistant_wiki_revision"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
revision_no: Mapped[int] = mapped_column(Integer)
content_markdown: Mapped[str] = mapped_column(PortableLongText(), default="")
summary: Mapped[str] = mapped_column(String(2048), default="")
source_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
import_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
change_type: Mapped[str] = mapped_column(String(32), default="import")
status: Mapped[str] = mapped_column(String(24), default="published")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class AssistantKnowledgeVectorRow(Base):
__tablename__ = "assistant_knowledge_vectors"
__table_args__ = (
Index("ix_assistant_knowledge_vector_base_fingerprint", "base_id", "embedding_fingerprint"),
Index("ix_assistant_knowledge_vector_page", "page_id"),
Index("ix_assistant_knowledge_vector_source", "source_id"),
)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
page_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_vector_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
section_index: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
heading: Mapped[str | None] = mapped_column(String(1024), nullable=True)
section_content: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
embedding_model: Mapped[str | None] = mapped_column(String(255), nullable=True)
embedding_fingerprint: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
embedding_dimensions: Mapped[int] = mapped_column(Integer, nullable=False)
vector_blob: Mapped[bytes] = mapped_column(LargeBinary, nullable=False)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class AssistantKnowledgeEntityRow(Base):
__tablename__ = "assistant_knowledge_entities"
__table_args__ = (UniqueConstraint("base_id", "entity_type", "normalized_name", name="uq_assistant_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512), index=True)
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeEntityContributionRow(Base):
__tablename__ = "assistant_knowledge_entity_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
source_chunk_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_skill_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class AssistantKnowledgeRelationRow(Base):
__tablename__ = "assistant_knowledge_relations"
__table_args__ = (UniqueConstraint("base_id", "source_entity_id", "predicate", "target_entity_id", name="uq_assistant_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
base_id: Mapped[str] = mapped_column(String(36), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
wiki_page_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
class AssistantKnowledgeRelationContributionRow(Base):
__tablename__ = "assistant_knowledge_relation_contributions"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
source_id: Mapped[str] = mapped_column(String(36), index=True)
import_job_id: Mapped[str] = mapped_column(String(36), index=True)
source_wiki_slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
graph_edge_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -176,7 +176,7 @@ async def _auto_create_postgres_db(url: str) -> None:
try:
async with maint_engine.connect() as conn:
await conn.execute(text(f'CREATE DATABASE "{db_name}"'))
logger.info("Auto-created PostgreSQL database: %s", db_name)
logger.info("Auto-created PostgreSQL-compatible database: %s", db_name)
finally:
await maint_engine.dispose()
@@ -230,10 +230,10 @@ def _ensure_orm_columns_sync(sync_conn) -> None:
added column are created as well. Table-level ``UniqueConstraint`` objects
that are declared on the model but missing from the table are also created
(as a ``CREATE UNIQUE INDEX`` of the same name, which is what the migrations
emit too, and works on SQLite/MySQL/PostgreSQL alike) — without this a newly
emit too, and works on SQLite/MySQL/PostgreSQL/openGauss alike) — without this a newly
declared uniqueness rule (e.g. ``active_dedupe_key``) would silently be absent
on an existing database, weakening dedupe.
Works on sqlite/mysql/postgres — the column DDL is compiled through the
Works on sqlite/mysql/postgres/gauss — the column DDL is compiled through the
active dialect, so portable types (``PortableJSON``/``PortableLongText``/
``BeijingDateTime``) resolve to the right backend type. Everything is
per-column best-effort: a failure is logged and startup continues.
@@ -355,13 +355,13 @@ async def init_engine(
"""Create the async engine and session factory, then auto-create tables.
Args:
backend: "memory", "sqlite", "postgres", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/mysql).
backend: "memory", "sqlite", "postgres", "gauss", or "mysql".
url: SQLAlchemy async URL (for sqlite/postgres/gauss/mysql).
echo: Echo SQL to log.
pool_size: Base connection pool size (postgres/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/mysql).
pool_recycle: Recycle connections after N seconds (postgres/mysql). Prevents stale
pool_size: Base connection pool size (postgres/gauss/mysql).
max_overflow: Extra connections allowed beyond pool_size under load (postgres/gauss/mysql).
pool_timeout: Seconds to wait for a free pooled connection before erroring (postgres/gauss/mysql).
pool_recycle: Recycle connections after N seconds (postgres/gauss/mysql). Prevents stale
connections dropped by firewalls / MySQL wait_timeout / pgbouncer.
sqlite_dir: Directory to create for SQLite (ensured to exist).
"""
@@ -379,11 +379,21 @@ async def init_engine(
except RuntimeError:
_engine_loop = None
if backend == "postgres":
if backend in {"postgres", "gauss"}:
try:
import asyncpg # noqa: F401
except ImportError:
raise ImportError("database.backend is set to 'postgres' but asyncpg is not installed.\nInstall it with:\n uv sync --extra postgres\nOr switch to backend: sqlite in config.yaml for single-node deployment.") from None
extra = "gauss" if backend == "gauss" else "postgres"
raise ImportError(
f"database.backend is set to {backend!r} but asyncpg is not installed.\n"
f"Install it with:\n uv sync --extra {extra}\n"
"Or switch to backend: sqlite in config.yaml for single-node deployment."
) from None
if backend == "gauss" and find_spec("opengauss_sqlalchemy") is None:
raise ImportError(
"database.backend is set to 'gauss' but the openGauss SQLAlchemy dialect "
"is not installed.\nInstall it with:\n uv sync --extra gauss"
)
if backend == "mysql":
_add_runtime_vendor_path()
driver = _mysql_driver_name(url)
@@ -423,7 +433,7 @@ async def init_engine(
cursor.execute("PRAGMA foreign_keys=ON;")
finally:
cursor.close()
elif backend == "postgres":
elif backend in {"postgres", "gauss"}:
_engine = create_async_engine(
url,
echo=echo,
@@ -468,7 +478,7 @@ async def init_engine(
async with _engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
except Exception as exc:
if backend == "postgres" and "does not exist" in str(exc):
if backend in {"postgres", "gauss"} and "does not exist" in str(exc).lower():
# Database not yet created — attempt to auto-create it, then retry.
await _auto_create_postgres_db(url)
# Rebuild engine against the now-existing database
@@ -511,7 +521,7 @@ async def init_engine(
# cannot do on its own. Failures don't abort startup — see the helper.
# await _run_alembic_upgrade(url)
logger.info("Persistence engine initialized: backend=%s", backend)
logger.info("Persistence engine initialized: backend=%s dialect=%s", backend, _engine.dialect.name)
async def init_engine_from_config(config) -> None:
@@ -3,6 +3,7 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator
from typing import Any
@@ -35,6 +36,23 @@ class LlmWikiIndexStore(ABC):
@abstractmethod
async def load_vector_snapshot(self, mapping_id: str, fingerprint: str) -> dict[str, Any]: ...
async def iter_vector_snapshot_rows(
self,
mapping_id: str,
fingerprint: str,
*,
batch_size: int = 1000,
) -> AsyncIterator[dict[str, Any]]:
"""Iterate vector rows for streaming export.
Implementations with a database backend should override this method so
very large vector packages do not need to be materialized in memory.
"""
snapshot = await self.load_vector_snapshot(mapping_id, fingerprint)
for row in snapshot.get("rows") or []:
if isinstance(row, dict):
yield row
@abstractmethod
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]: ...
@@ -24,7 +24,7 @@ class MemoryLlmWikiIndexStore(LlmWikiIndexStore):
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug_hash"] == slug_hash), None)
return dict(row) if row else None
return {**row, "vector_count": len(self.vectors.get(row["id"], []))} if row else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
row = next((p for p in self.pages.values() if p["knowledge_base_mapping_id"] == mapping_id and p["slug"] == slug), None)
@@ -2,6 +2,7 @@
from __future__ import annotations
from collections.abc import AsyncIterator
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
@@ -40,10 +41,11 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
return row
async def get_page_by_slug_hash(self, mapping_id: str, slug_hash: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
count = select(func.count(LlmWikiWikiVectorRow.id)).where(LlmWikiWikiVectorRow.page_id == LlmWikiWikiPageRow.id).correlate(LlmWikiWikiPageRow).scalar_subquery()
stmt = select(LlmWikiWikiPageRow, count).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug_hash == slug_hash)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return _dict(row) if row else None
result = (await session.execute(stmt)).one_or_none()
return {**_dict(result[0]), "vector_count": result[1]} if result else None
async def get_page(self, mapping_id: str, slug: str) -> dict[str, Any] | None:
stmt = select(LlmWikiWikiPageRow).where(LlmWikiWikiPageRow.knowledge_base_mapping_id == mapping_id, LlmWikiWikiPageRow.slug == slug)
@@ -181,6 +183,41 @@ class SqlLlmWikiIndexStore(LlmWikiIndexStore):
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
return {"revision": int(state.index_revision if state else 0), "fingerprint": fingerprint, "rows": rows}
async def iter_vector_snapshot_rows(
self,
mapping_id: str,
fingerprint: str,
*,
batch_size: int = 1000,
) -> AsyncIterator[dict[str, Any]]:
batch = max(1, min(int(batch_size), 5000))
offset = 0
while True:
stmt = (
select(LlmWikiWikiVectorRow, LlmWikiWikiPageRow)
.join(LlmWikiWikiPageRow, LlmWikiWikiPageRow.id == LlmWikiWikiVectorRow.page_id)
.where(
LlmWikiWikiVectorRow.knowledge_base_mapping_id == mapping_id,
LlmWikiWikiVectorRow.embedding_fingerprint == fingerprint,
LlmWikiWikiPageRow.index_status == "ready",
LlmWikiWikiPageRow.is_remote_deleted.is_(False),
LlmWikiWikiPageRow.status.in_(["published", "draft"]),
)
.order_by(LlmWikiWikiVectorRow.page_id, LlmWikiWikiVectorRow.section_index)
.limit(batch)
.offset(offset)
)
async with self._sf() as session:
pairs = list((await session.execute(stmt)).all())
rows = [{**_dict(vector), "page": _dict(page)} for vector, page in pairs]
if not rows:
break
for row in rows:
yield row
if len(rows) < batch:
break
offset += len(rows)
async def get_index_revision(self, mapping_id: str) -> tuple[int, str | None, str]:
async with self._sf() as session:
state = await session.get(LlmWikiWikiSyncStateRow, mapping_id)
@@ -0,0 +1,73 @@
"""Add skill distillation and assistant knowledge tables.
Revision ID: 20260901_01
Revises: 20260825_01
Create Date: 2026-09-01
"""
from __future__ import annotations
from alembic import op
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
revision = "20260901_01"
down_revision = "20260825_01"
branch_labels = None
depends_on = None
TABLES = [
SkillKnowledgeBindingRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncJobRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeArtifactRow,
SkillKnowledgeReviewRow,
SkillKnowledgeEntityRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeRelationContributionRow,
AssistantKnowledgeBaseRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeRelationContributionRow,
]
def upgrade() -> None:
bind = op.get_bind()
for model in TABLES:
model.__table__.create(bind, checkfirst=True)
def downgrade() -> None:
bind = op.get_bind()
for model in reversed(TABLES):
model.__table__.drop(bind, checkfirst=True)
@@ -0,0 +1,25 @@
"""Add assistant knowledge vector table.
Revision ID: 20260902_01
Revises: 20260901_01
Create Date: 2026-09-02
"""
from __future__ import annotations
from alembic import op
from deerflow.persistence.assistant_knowledge.model import AssistantKnowledgeVectorRow
revision = "20260902_01"
down_revision = "20260901_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
AssistantKnowledgeVectorRow.__table__.create(op.get_bind(), checkfirst=True)
def downgrade() -> None:
AssistantKnowledgeVectorRow.__table__.drop(op.get_bind(), checkfirst=True)
@@ -0,0 +1,21 @@
"""Preserve source Wiki metadata during package round trips."""
import sqlalchemy as sa
from alembic import op
from deerflow.persistence.types import PortableJSON
revision = "20260903_01"
down_revision = ("20260902_01", "20260831_01")
branch_labels = None
depends_on = None
def upgrade():
columns = {col["name"] for col in sa.inspect(op.get_bind()).get_columns("assistant_knowledge_wiki_pages")}
if "metadata_json" not in columns:
op.add_column("assistant_knowledge_wiki_pages", sa.Column("metadata_json", PortableJSON(), nullable=True))
def downgrade():
op.drop_column("assistant_knowledge_wiki_pages", "metadata_json")
@@ -35,6 +35,18 @@ from deerflow.persistence.agents.model import (
AgentRow,
AgentSkillRow,
)
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeBaseRow,
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
AssistantKnowledgeImportJobRow,
AssistantKnowledgeRelationContributionRow,
AssistantKnowledgeRelationRow,
AssistantKnowledgeSourceRow,
AssistantKnowledgeVectorRow,
AssistantKnowledgeWikiPageRow,
AssistantKnowledgeWikiRevisionRow,
)
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.compact_menu_overrides.model import CompactMenuOverrideRow
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
@@ -100,6 +112,18 @@ from deerflow.persistence.scheduled_tasks.model import (
SchedulerThreadRow,
)
from deerflow.persistence.sensitive_words.model import SensitiveWordRow
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skills.model import SkillDisplayAdapterRow, SkillFavoriteRow, SkillRow
from deerflow.persistence.tags.model import TagAssignmentRow, TagRow
from deerflow.persistence.task_buttons.model import TaskButtonRow
@@ -133,6 +157,15 @@ __all__ = [
"AgentSkillRow",
"AdminLeaderboardDailyStatRow",
"BusinessMappingRow",
"AssistantKnowledgeBaseRow",
"AssistantKnowledgeSourceRow",
"AssistantKnowledgeImportJobRow",
"AssistantKnowledgeWikiPageRow",
"AssistantKnowledgeWikiRevisionRow",
"AssistantKnowledgeEntityRow",
"AssistantKnowledgeEntityContributionRow",
"AssistantKnowledgeRelationRow",
"AssistantKnowledgeRelationContributionRow",
"CompactMenuOverrideRow",
"ConcurrencySampleRow",
"DashboardSessionRow",
@@ -211,6 +244,16 @@ __all__ = [
"SkillRow",
"SkillDisplayAdapterRow",
"SkillFavoriteRow",
"SkillKnowledgeArtifactRow",
"SkillKnowledgeBindingRow",
"SkillKnowledgeEntityContributionRow",
"SkillKnowledgeEntityRow",
"SkillKnowledgeRelationContributionRow",
"SkillKnowledgeRelationRow",
"SkillKnowledgeReviewRow",
"SkillKnowledgeSnapshotRow",
"SkillKnowledgeSyncItemRow",
"SkillKnowledgeSyncJobRow",
"TagAssignmentRow",
"TagRow",
"TaskButtonRow",
@@ -0,0 +1,15 @@
from deerflow.persistence.skill_knowledge.model import ( # noqa: F401
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.persistence.skill_knowledge.store import SkillKnowledgeRepository, make_skill_knowledge_store # noqa: F401
__all__ = [name for name in globals() if name.startswith("SkillKnowledge") or name == "make_skill_knowledge_store"]
@@ -0,0 +1,203 @@
"""ORM rows for skill-to-knowledge distillation and review."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Float, Integer, String, UniqueConstraint, false, true
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
def _now() -> datetime:
return datetime.now(UTC)
class SkillKnowledgeBindingRow(Base):
__tablename__ = "skill_knowledge_bindings"
__table_args__ = (UniqueConstraint("skill_name", "target_type", "target_id", name="uq_skill_knowledge_binding"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
skill_name: Mapped[str] = mapped_column(String(255), index=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_mode: Mapped[str] = mapped_column(String(32), default="wiki")
target_id: Mapped[str] = mapped_column(String(128), index=True)
target_name: Mapped[str] = mapped_column(String(255), default="")
remote_business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(48), default="pending", index=True)
active: Mapped[bool] = mapped_column(Boolean, default=True, server_default=true(), index=True)
auto_sync_enabled: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false(), index=True)
current_snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
source_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
synced_digest: Mapped[str | None] = mapped_column(String(80), nullable=True)
last_job_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
last_item_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
last_synced_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeSnapshotRow(Base):
__tablename__ = "skill_knowledge_snapshots"
__table_args__ = (UniqueConstraint("skill_name", "source_digest", "extractor_version", "prompt_hash", "model_name", name="uq_skill_knowledge_snapshot"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
skill_name: Mapped[str] = mapped_column(String(255), index=True)
source_digest: Mapped[str] = mapped_column(String(80), index=True)
extractor_version: Mapped[str] = mapped_column(String(64))
prompt_hash: Mapped[str] = mapped_column(String(80))
model_name: Mapped[str] = mapped_column(String(128))
manifest_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
ir_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
status: Mapped[str] = mapped_column(String(24), default="ready")
counts_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class SkillKnowledgeSyncJobRow(Base):
__tablename__ = "skill_knowledge_sync_jobs"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
trigger: Mapped[str] = mapped_column(String(32), default="manual")
target_count: Mapped[int] = mapped_column(Integer, default=0)
targets_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
total_count: Mapped[int] = mapped_column(Integer, default=0)
success_count: Mapped[int] = mapped_column(Integer, default=0)
partial_failed_count: Mapped[int] = mapped_column(Integer, default=0)
failed_count: Mapped[int] = mapped_column(Integer, default=0)
cancel_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default=false())
review_mode: Mapped[str] = mapped_column(String(32), default="auto_high_confidence")
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class SkillKnowledgeSyncItemRow(Base):
__tablename__ = "skill_knowledge_sync_items"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
job_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
target_type: Mapped[str] = mapped_column(String(32))
target_mode: Mapped[str] = mapped_column(String(32))
target_id: Mapped[str] = mapped_column(String(128))
skill_name: Mapped[str] = mapped_column(String(255), index=True)
snapshot_id: Mapped[str | None] = mapped_column(String(36), nullable=True)
status: Mapped[str] = mapped_column(String(32), default="queued", index=True)
phase: Mapped[str] = mapped_column(String(32), default="scanning")
progress: Mapped[int] = mapped_column(Integer, default=0)
attempts: Mapped[int] = mapped_column(Integer, default=0)
active_dedupe_key: Mapped[str | None] = mapped_column(String(36), nullable=True, unique=True)
lease_owner: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True, index=True)
last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
last_error_message: Mapped[str | None] = mapped_column(String(2048), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
completed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
class SkillKnowledgeArtifactRow(Base):
__tablename__ = "skill_knowledge_artifacts"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
kind: Mapped[str] = mapped_column(String(48), index=True)
slug: Mapped[str | None] = mapped_column(String(512), nullable=True)
content_text: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
remote_id: Mapped[str | None] = mapped_column(String(255), nullable=True)
business_key: Mapped[str | None] = mapped_column(String(512), nullable=True)
remote_refs_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
status: Mapped[str] = mapped_column(String(32), default="staged", index=True)
metadata_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeReviewRow(Base):
__tablename__ = "skill_knowledge_reviews"
id: Mapped[str] = mapped_column(String(36), primary_key=True)
job_id: Mapped[str] = mapped_column(String(36), index=True)
item_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
object_type: Mapped[str] = mapped_column(String(32), index=True)
object_key: Mapped[str] = mapped_column(String(128))
original_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
edited_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
status: Mapped[str] = mapped_column(String(24), default="pending", index=True)
reason: Mapped[str | None] = mapped_column(String(1024), nullable=True)
reviewed_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
reviewed_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
class SkillKnowledgeEntityRow(Base):
__tablename__ = "skill_knowledge_entities"
__table_args__ = (UniqueConstraint("target_type", "target_id", "entity_type", "normalized_name", name="uq_skill_knowledge_entity"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_id: Mapped[str] = mapped_column(String(128), index=True)
entity_type: Mapped[str] = mapped_column(String(64), index=True)
normalized_name: Mapped[str] = mapped_column(String(512))
display_name: Mapped[str] = mapped_column(String(512))
aliases_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
status: Mapped[str] = mapped_column(String(24), default="active")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeEntityContributionRow(Base):
__tablename__ = "skill_knowledge_entity_contributions"
__table_args__ = (UniqueConstraint("entity_id", "binding_id", "snapshot_id", "local_entity_id", name="uq_skill_entity_contribution"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
entity_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
local_entity_id: Mapped[str] = mapped_column(String(128))
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
attributes_json: Mapped[dict | None] = mapped_column(PortableJSON(), default=dict, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
class SkillKnowledgeRelationRow(Base):
__tablename__ = "skill_knowledge_relations"
__table_args__ = (UniqueConstraint("target_type", "target_id", "source_entity_id", "predicate", "target_entity_id", name="uq_skill_knowledge_relation"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
target_type: Mapped[str] = mapped_column(String(32), index=True)
target_id: Mapped[str] = mapped_column(String(128), index=True)
source_entity_id: Mapped[str] = mapped_column(String(36), index=True)
predicate: Mapped[str] = mapped_column(String(128), index=True)
target_entity_id: Mapped[str] = mapped_column(String(36), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
status: Mapped[str] = mapped_column(String(24), default="active")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now)
updated_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=_now, onupdate=_now)
class SkillKnowledgeRelationContributionRow(Base):
__tablename__ = "skill_knowledge_relation_contributions"
__table_args__ = (UniqueConstraint("relation_id", "binding_id", "snapshot_id", "local_relation_id", name="uq_skill_relation_contribution"),)
id: Mapped[str] = mapped_column(String(36), primary_key=True)
relation_id: Mapped[str] = mapped_column(String(36), index=True)
binding_id: Mapped[str] = mapped_column(String(36), index=True)
snapshot_id: Mapped[str] = mapped_column(String(36), index=True)
local_relation_id: Mapped[str] = mapped_column(String(128))
evidence_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
confidence: Mapped[float] = mapped_column(Float, default=1.0)
@@ -0,0 +1,795 @@
"""SQL repository for skill knowledge jobs, snapshots and contributions."""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
from typing import Any
from uuid import uuid4
from sqlalchemy import delete, func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.skill_knowledge.model import (
SkillKnowledgeArtifactRow,
SkillKnowledgeBindingRow,
SkillKnowledgeEntityContributionRow,
SkillKnowledgeEntityRow,
SkillKnowledgeRelationContributionRow,
SkillKnowledgeRelationRow,
SkillKnowledgeReviewRow,
SkillKnowledgeSnapshotRow,
SkillKnowledgeSyncItemRow,
SkillKnowledgeSyncJobRow,
)
from deerflow.skill_knowledge.merge import normalize_name, normalize_predicate
def _now() -> datetime:
return datetime.now(UTC)
def _dict(row: Any) -> dict[str, Any]:
list_json_fields = {
"aliases_json",
"evidence_json",
"manifest_json",
"refs_json",
"targets_json",
}
data = row.to_dict()
for key, value in tuple(data.items()):
if isinstance(value, datetime):
data[key] = value.isoformat()
elif key.endswith("_json"):
data[key[:-5]] = data.pop(key) or ([] if key in list_json_fields else {})
return data
class SkillKnowledgeRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def _retire_target_orphans(self, session: AsyncSession, *, target_type: str, target_id: str) -> None:
entities = (
await session.execute(
select(SkillKnowledgeEntityRow).where(
SkillKnowledgeEntityRow.target_type == target_type,
SkillKnowledgeEntityRow.target_id == target_id,
)
)
).scalars()
for entity in entities:
contribution = (await session.execute(select(SkillKnowledgeEntityContributionRow.id).where(SkillKnowledgeEntityContributionRow.entity_id == entity.id).limit(1))).scalar_one_or_none()
entity.status = "active" if contribution else "deleted"
relations = (
await session.execute(
select(SkillKnowledgeRelationRow).where(
SkillKnowledgeRelationRow.target_type == target_type,
SkillKnowledgeRelationRow.target_id == target_id,
)
)
).scalars()
for relation in relations:
contribution = (await session.execute(select(SkillKnowledgeRelationContributionRow.id).where(SkillKnowledgeRelationContributionRow.relation_id == relation.id).limit(1))).scalar_one_or_none()
relation.status = "active" if contribution else "deleted"
async def create_job(
self,
*,
skill_names: list[str],
targets: list[dict[str, Any]],
created_by: str | None,
force: bool = False,
trigger: str = "manual",
review_mode: str = "auto_high_confidence",
) -> dict[str, Any]:
names = list(dict.fromkeys(name.strip() for name in skill_names if name.strip()))
normalized_targets: list[dict[str, Any]] = []
seen_targets: set[tuple[str, str]] = set()
for target in targets:
target_type = str(target.get("target_type") or "").strip()
target_id = str(target.get("target_id") or "").strip()
if not target_type or not target_id or (target_type, target_id) in seen_targets:
continue
seen_targets.add((target_type, target_id))
normalized_targets.append(
{
"target_type": target_type,
"target_mode": str(target.get("target_mode") or ("wiki" if target_type == "weknora" else "assistant")),
"target_id": target_id,
"target_name": str(target.get("target_name") or target_id),
"remote_business_key": target.get("remote_business_key"),
"readonly": bool(target.get("readonly", False)),
}
)
job_id = str(uuid4())
now = _now()
async with self._sf() as session:
job = SkillKnowledgeSyncJobRow(
id=job_id,
trigger=trigger,
target_count=len(normalized_targets),
targets_json=normalized_targets,
status="queued",
total_count=0,
review_mode=review_mode,
created_by=created_by,
created_at=now,
)
session.add(job)
for skill_name in names:
for target in normalized_targets:
stmt = select(SkillKnowledgeBindingRow).where(
SkillKnowledgeBindingRow.skill_name == skill_name,
SkillKnowledgeBindingRow.target_type == target["target_type"],
SkillKnowledgeBindingRow.target_id == target["target_id"],
)
binding = (await session.execute(stmt)).scalar_one_or_none()
if binding is None:
binding = SkillKnowledgeBindingRow(
id=str(uuid4()),
skill_name=skill_name,
target_type=target["target_type"],
target_mode=target["target_mode"],
target_id=target["target_id"],
target_name=target["target_name"],
remote_business_key=target.get("remote_business_key"),
status="queued",
created_by=created_by,
created_at=now,
updated_at=now,
)
session.add(binding)
else:
active_item = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == binding.id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
if active_item is not None:
continue
binding.target_mode = target["target_mode"]
binding.target_name = target["target_name"]
binding.remote_business_key = target.get("remote_business_key")
binding.status = "queued"
binding.active = True
# The current manual submitter owns the relationship UI
# from this point onward. Target write authorization is
# enforced by the router before this method is called.
if trigger == "manual":
binding.created_by = created_by
binding.updated_at = now
item = SkillKnowledgeSyncItemRow(
id=str(uuid4()),
job_id=job_id,
binding_id=binding.id,
target_type=target["target_type"],
target_mode=target["target_mode"],
target_id=target["target_id"],
skill_name=skill_name,
status="queued",
phase="scanning",
progress=0,
active_dedupe_key=binding.id,
created_at=now,
updated_at=now,
)
session.add(item)
binding.last_job_id = job_id
binding.last_item_id = item.id
job.total_count += 1
if job.total_count == 0:
job.status = "completed"
job.completed_at = now
await session.flush()
result = _dict(job)
result["force"] = force
await session.commit()
return result
async def list_bindings(
self,
*,
skill_names: list[str] | None = None,
target_id: str | None = None,
include_detached: bool = False,
created_by: str | None = None,
) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeBindingRow)
if skill_names:
stmt = stmt.where(SkillKnowledgeBindingRow.skill_name.in_(skill_names))
if target_id:
stmt = stmt.where(SkillKnowledgeBindingRow.target_id == target_id)
if not include_detached:
stmt = stmt.where(SkillKnowledgeBindingRow.active.is_(True))
if created_by is not None:
stmt = stmt.where(SkillKnowledgeBindingRow.created_by == created_by)
stmt = stmt.order_by(SkillKnowledgeBindingRow.updated_at.desc())
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def get_binding(self, binding_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeBindingRow, binding_id)
return _dict(row) if row else None
async def list_artifacts(self, binding_id: str, *, active_only: bool = True) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeArtifactRow).where(SkillKnowledgeArtifactRow.binding_id == binding_id)
if active_only:
stmt = stmt.where(SkillKnowledgeArtifactRow.status == "active")
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def has_active_item(self, binding_id: str) -> bool:
async with self._sf() as session:
value = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == binding_id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
return value is not None
async def set_binding_auto_sync(self, binding_id: str, *, enabled: bool) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeBindingRow, binding_id)
if row is None or not row.active:
return None
row.auto_sync_enabled = enabled
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
return data
async def get_job(self, job_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return None
items = (await session.execute(select(SkillKnowledgeSyncItemRow).where(SkillKnowledgeSyncItemRow.job_id == job_id).order_by(SkillKnowledgeSyncItemRow.created_at))).scalars()
result = _dict(job)
result["items"] = [_dict(item) for item in items]
return result
async def list_jobs(self, *, limit: int = 50, created_by: str | None = None) -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeSyncJobRow).order_by(SkillKnowledgeSyncJobRow.created_at.desc()).limit(limit)
if created_by is not None:
stmt = stmt.where(SkillKnowledgeSyncJobRow.created_by == created_by)
async with self._sf() as session:
jobs = list((await session.execute(stmt)).scalars())
if not jobs:
return []
job_ids = [row.id for row in jobs]
item_rows = list(
(
await session.execute(
select(SkillKnowledgeSyncItemRow)
.where(SkillKnowledgeSyncItemRow.job_id.in_(job_ids))
.order_by(SkillKnowledgeSyncItemRow.created_at)
)
).scalars()
)
items_by_job: dict[str, list[dict[str, Any]]] = {job_id: [] for job_id in job_ids}
for item in item_rows:
items_by_job[item.job_id].append(_dict(item))
result = []
for job in jobs:
data = _dict(job)
data["items"] = items_by_job[job.id]
result.append(data)
return result
async def list_queued_items(self, *, limit: int = 20) -> list[dict[str, Any]]:
now = _now()
stmt = (
select(SkillKnowledgeSyncItemRow)
.where(
or_(
SkillKnowledgeSyncItemRow.status == "queued",
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
)
)
.order_by(SkillKnowledgeSyncItemRow.created_at)
.limit(limit)
)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def claim_item(
self,
item_id: str,
*,
owner: str,
lease_seconds: int = 900,
) -> dict[str, Any] | None:
"""Atomically claim a queued item, or reclaim one whose worker lease expired."""
now = _now()
stmt = (
update(SkillKnowledgeSyncItemRow)
.where(
SkillKnowledgeSyncItemRow.id == item_id,
or_(
SkillKnowledgeSyncItemRow.status == "queued",
((SkillKnowledgeSyncItemRow.status == "running") & (SkillKnowledgeSyncItemRow.lease_until < now)),
),
)
.values(
status="running",
started_at=func.coalesce(SkillKnowledgeSyncItemRow.started_at, now),
attempts=SkillKnowledgeSyncItemRow.attempts + 1,
lease_owner=owner,
lease_until=now + timedelta(seconds=lease_seconds),
updated_at=now,
)
)
async with self._sf() as session:
result = await session.execute(stmt)
if result.rowcount != 1:
await session.rollback()
return None
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def renew_lease(self, item_id: str, *, owner: str, lease_seconds: int = 900) -> bool:
now = _now()
async with self._sf() as session:
result = await session.execute(
update(SkillKnowledgeSyncItemRow)
.where(
SkillKnowledgeSyncItemRow.id == item_id,
SkillKnowledgeSyncItemRow.status == "running",
SkillKnowledgeSyncItemRow.lease_owner == owner,
)
.values(
lease_until=now + timedelta(seconds=lease_seconds),
updated_at=now,
)
)
await session.commit()
return result.rowcount == 1
async def update_item(
self,
item_id: str,
*,
status: str | None = None,
phase: str | None = None,
progress: int | None = None,
snapshot_id: str | None = None,
error_code: str | None = None,
error_message: str | None = None,
) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
if row is None:
return None
if status is not None:
row.status = status
if status == "running" and row.started_at is None:
row.started_at = _now()
row.attempts += 1
if status in {"completed", "partial_failed", "failed", "canceled"}:
row.completed_at = _now()
row.active_dedupe_key = None
elif status == "queued":
row.completed_at = None
if status != "running":
row.lease_owner = None
row.lease_until = None
if phase is not None:
row.phase = phase
if progress is not None:
row.progress = max(0, min(100, progress))
if snapshot_id is not None:
row.snapshot_id = snapshot_id
row.last_error_code = error_code
row.last_error_message = error_message
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def refresh_job(self, job_id: str) -> None:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return
rows = (await session.execute(select(SkillKnowledgeSyncItemRow.status, func.count()).where(SkillKnowledgeSyncItemRow.job_id == job_id).group_by(SkillKnowledgeSyncItemRow.status))).all()
counts = {str(status): int(count) for status, count in rows}
job.success_count = counts.get("completed", 0)
job.partial_failed_count = counts.get("partial_failed", 0)
job.failed_count = counts.get("failed", 0)
active = sum(counts.get(key, 0) for key in ("queued", "running", "awaiting_review"))
if counts.get("running", 0) or counts.get("awaiting_review", 0):
job.status = "running"
job.started_at = job.started_at or _now()
elif active == 0 and sum(counts.values()) >= job.total_count:
if job.failed_count and job.success_count == 0 and job.partial_failed_count == 0:
job.status = "failed"
elif job.failed_count or job.partial_failed_count:
job.status = "partial_failed"
else:
job.status = "completed"
job.completed_at = _now()
await session.commit()
async def cancel_job(self, job_id: str) -> bool:
async with self._sf() as session:
job = await session.get(SkillKnowledgeSyncJobRow, job_id)
if job is None:
return False
job.cancel_requested = True
await session.execute(
SkillKnowledgeSyncItemRow.__table__.update()
.where(
SkillKnowledgeSyncItemRow.job_id == job_id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "awaiting_review"]),
)
.values(
status="canceled",
phase="completed",
completed_at=_now(),
progress=100,
active_dedupe_key=None,
lease_owner=None,
lease_until=None,
)
)
job.status = "canceled"
job.completed_at = _now()
await session.commit()
return True
async def requeue_item(self, item_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSyncItemRow, item_id)
if row is None:
return None
active_item = (
await session.execute(
select(SkillKnowledgeSyncItemRow.id)
.where(
SkillKnowledgeSyncItemRow.binding_id == row.binding_id,
SkillKnowledgeSyncItemRow.id != row.id,
SkillKnowledgeSyncItemRow.status.in_(["queued", "running", "awaiting_review"]),
)
.limit(1)
)
).scalar_one_or_none()
if active_item is not None:
return None
row.status = "queued"
row.phase = "scanning"
row.progress = 0
row.completed_at = None
row.active_dedupe_key = row.binding_id
row.updated_at = _now()
await session.flush()
data = _dict(row)
await session.commit()
await self.refresh_job(data["job_id"])
return data
async def get_or_create_snapshot(self, *, skill_name: str, source_digest: str, extractor_version: str, prompt_hash: str, model_name: str, manifest: list, ir: dict, counts: dict) -> dict[str, Any]:
stmt = select(SkillKnowledgeSnapshotRow).where(
SkillKnowledgeSnapshotRow.skill_name == skill_name,
SkillKnowledgeSnapshotRow.source_digest == source_digest,
SkillKnowledgeSnapshotRow.extractor_version == extractor_version,
SkillKnowledgeSnapshotRow.prompt_hash == prompt_hash,
SkillKnowledgeSnapshotRow.model_name == model_name,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
if row is None:
row = SkillKnowledgeSnapshotRow(
id=str(uuid4()),
skill_name=skill_name,
source_digest=source_digest,
extractor_version=extractor_version,
prompt_hash=prompt_hash,
model_name=model_name,
manifest_json=manifest,
ir_json=ir,
counts_json=counts,
status="ready",
created_at=_now(),
)
session.add(row)
await session.flush()
data = _dict(row)
await session.commit()
return data
async def get_snapshot(self, snapshot_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(SkillKnowledgeSnapshotRow, snapshot_id)
return _dict(row) if row else None
async def create_reviews(self, *, job_id: str, item_id: str, binding_id: str, snapshot_id: str, objects: list[tuple[str, dict]]) -> int:
rows = [
SkillKnowledgeReviewRow(
id=str(uuid4()),
job_id=job_id,
item_id=item_id,
binding_id=binding_id,
snapshot_id=snapshot_id,
object_type=kind,
object_key=str(obj.get("local_id") or obj.get("slug") or uuid4()),
original_json=obj,
status="pending",
created_at=_now(),
)
for kind, obj in objects
]
async with self._sf() as session:
session.add_all(rows)
await session.commit()
return len(rows)
async def list_reviews(self, *, job_id: str | None = None, item_id: str | None = None, status: str | None = "pending") -> list[dict[str, Any]]:
stmt = select(SkillKnowledgeReviewRow)
if job_id:
stmt = stmt.where(SkillKnowledgeReviewRow.job_id == job_id)
if item_id:
stmt = stmt.where(SkillKnowledgeReviewRow.item_id == item_id)
if status:
stmt = stmt.where(SkillKnowledgeReviewRow.status == status)
stmt = stmt.order_by(SkillKnowledgeReviewRow.created_at)
async with self._sf() as session:
return [_dict(row) for row in (await session.execute(stmt)).scalars()]
async def review(self, review_ids: list[str], *, action: str, actor: str | None, edits: dict[str, dict] | None = None, reason: str | None = None) -> list[str]:
final_status = "approved" if action == "approve" else "rejected"
affected_items: set[str] = set()
async with self._sf() as session:
rows = (await session.execute(select(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.id.in_(review_ids)))).scalars()
for row in rows:
row.status = final_status
row.edited_json = (edits or {}).get(row.id)
row.reason = reason
row.reviewed_by = actor
row.reviewed_at = _now()
affected_items.add(row.item_id)
await session.commit()
return sorted(affected_items)
async def pending_review_count(self, item_id: str) -> int:
stmt = select(func.count()).select_from(SkillKnowledgeReviewRow).where(SkillKnowledgeReviewRow.item_id == item_id, SkillKnowledgeReviewRow.status == "pending")
async with self._sf() as session:
return int((await session.execute(stmt)).scalar_one())
async def effective_ir(self, snapshot_id: str, item_id: str) -> dict[str, Any] | None:
snapshot = await self.get_snapshot(snapshot_id)
if snapshot is None:
return None
ir = dict(snapshot.get("ir") or {})
reviews = await self.list_reviews(item_id=item_id, status=None)
if not reviews:
return ir
by_key = {(row["object_type"], row["object_key"]): row for row in reviews}
for key in ("entities", "relations"):
kind = key[:-1]
accepted = []
for obj in ir.get(key, []):
review = by_key.get((kind, str(obj.get("local_id"))))
if review and review.get("status") == "rejected":
continue
accepted.append((review or {}).get("edited") or obj)
ir[key] = accepted
return ir
async def stage_and_activate(self, *, item_id: str, snapshot_id: str, ir: dict[str, Any], artifacts: list[dict[str, Any]], status: str = "synced") -> dict[str, Any]:
now = _now()
async with self._sf() as session:
item = await session.get(SkillKnowledgeSyncItemRow, item_id)
if item is None:
raise KeyError(item_id)
binding = await session.get(SkillKnowledgeBindingRow, item.binding_id)
if binding is None:
raise KeyError(item.binding_id)
if not binding.active:
raise RuntimeError("Binding was detached before staged activation")
await session.execute(
SkillKnowledgeArtifactRow.__table__.update()
.where(
SkillKnowledgeArtifactRow.binding_id == binding.id,
SkillKnowledgeArtifactRow.status == "active",
)
.values(status="retired", updated_at=now)
)
for artifact in artifacts:
session.add(
SkillKnowledgeArtifactRow(
id=str(uuid4()),
binding_id=binding.id,
snapshot_id=snapshot_id,
kind=str(artifact.get("kind") or "wiki_page"),
slug=artifact.get("slug"),
content_text=artifact.get("content"),
remote_id=artifact.get("remote_id"),
business_key=artifact.get("business_key"),
remote_refs_json=artifact.get("remote_refs") or [],
status="active",
metadata_json=artifact.get("metadata") or {},
created_at=now,
updated_at=now,
)
)
# Retire contributions from the old active version only after all staged artifacts are ready.
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding.id))
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding.id))
entity_ids: dict[str, str] = {}
for entity in ir.get("entities", []):
normalized = normalize_name(str(entity.get("name") or ""))
if not normalized:
continue
entity_type = str(entity.get("type") or "concept")
stmt = select(SkillKnowledgeEntityRow).where(
SkillKnowledgeEntityRow.target_type == binding.target_type,
SkillKnowledgeEntityRow.target_id == binding.target_id,
SkillKnowledgeEntityRow.entity_type == entity_type,
SkillKnowledgeEntityRow.normalized_name == normalized,
)
canonical = (await session.execute(stmt)).scalar_one_or_none()
if canonical is None:
canonical = SkillKnowledgeEntityRow(
id=str(uuid4()),
target_type=binding.target_type,
target_id=binding.target_id,
entity_type=entity_type,
normalized_name=normalized,
display_name=str(entity.get("name") or normalized),
aliases_json=entity.get("aliases") or [],
attributes_json=entity.get("attributes") or {},
status="active",
created_at=now,
updated_at=now,
)
session.add(canonical)
await session.flush()
else:
canonical.status = "active"
entity_ids[str(entity.get("local_id"))] = canonical.id
session.add(
SkillKnowledgeEntityContributionRow(
id=str(uuid4()),
entity_id=canonical.id,
binding_id=binding.id,
snapshot_id=snapshot_id,
local_entity_id=str(entity.get("local_id")),
evidence_json=entity.get("evidence") or [],
attributes_json=entity.get("attributes") or {},
confidence=float(entity.get("confidence") or 0),
)
)
for relation in ir.get("relations", []):
source_id = entity_ids.get(str(relation.get("source_local_id")))
target_id = entity_ids.get(str(relation.get("target_local_id")))
confidence = float(relation.get("confidence") or 0)
if not source_id or not target_id or confidence < 0.75 or not relation.get("evidence"):
continue
predicate = normalize_predicate(str(relation.get("predicate") or "related_to"))
stmt = select(SkillKnowledgeRelationRow).where(
SkillKnowledgeRelationRow.target_type == binding.target_type,
SkillKnowledgeRelationRow.target_id == binding.target_id,
SkillKnowledgeRelationRow.source_entity_id == source_id,
SkillKnowledgeRelationRow.predicate == predicate,
SkillKnowledgeRelationRow.target_entity_id == target_id,
)
canonical = (await session.execute(stmt)).scalar_one_or_none()
if canonical is None:
canonical = SkillKnowledgeRelationRow(
id=str(uuid4()),
target_type=binding.target_type,
target_id=binding.target_id,
source_entity_id=source_id,
predicate=predicate,
target_entity_id=target_id,
description=str(relation.get("description") or ""),
status="active",
created_at=now,
updated_at=now,
)
session.add(canonical)
await session.flush()
else:
canonical.status = "active"
session.add(
SkillKnowledgeRelationContributionRow(
id=str(uuid4()),
relation_id=canonical.id,
binding_id=binding.id,
snapshot_id=snapshot_id,
local_relation_id=str(relation.get("local_id")),
evidence_json=relation.get("evidence") or [],
confidence=confidence,
)
)
await self._retire_target_orphans(
session,
target_type=binding.target_type,
target_id=binding.target_id,
)
binding.current_snapshot_id = snapshot_id
binding.source_digest = str(ir.get("skill", {}).get("source_digest") or "")
binding.synced_digest = binding.source_digest
binding.status = status
binding.last_synced_at = now
binding.last_error_code = None
binding.last_error_message = None
binding.updated_at = now
item.snapshot_id = snapshot_id
item.status = "completed" if status in {"synced", "pending_remote_write"} else status
item.phase = "completed"
item.progress = 100
item.completed_at = now
item.active_dedupe_key = None
item.lease_owner = None
item.lease_until = None
item.updated_at = now
await session.flush()
result = _dict(binding)
await session.commit()
await self.refresh_job(item.job_id)
return result
async def detach_binding(
self,
binding_id: str,
*,
delete_contributions: bool,
cleanup_failed: bool = False,
) -> dict[str, Any] | None:
async with self._sf() as session:
binding = await session.get(SkillKnowledgeBindingRow, binding_id)
if binding is None:
return None
binding.active = False
binding.status = "detached_delete_pending_cleanup" if delete_contributions and cleanup_failed else "detached_deleted" if delete_contributions else "detached_keep"
binding.updated_at = _now()
if delete_contributions:
await session.execute(delete(SkillKnowledgeEntityContributionRow).where(SkillKnowledgeEntityContributionRow.binding_id == binding_id))
await session.execute(delete(SkillKnowledgeRelationContributionRow).where(SkillKnowledgeRelationContributionRow.binding_id == binding_id))
await session.execute(SkillKnowledgeArtifactRow.__table__.update().where(SkillKnowledgeArtifactRow.binding_id == binding_id).values(status="deleted", updated_at=_now()))
await self._retire_target_orphans(
session,
target_type=binding.target_type,
target_id=binding.target_id,
)
await session.flush()
data = _dict(binding)
await session.commit()
return data
async def mark_skill_stale(self, skill_name: str) -> int:
async with self._sf() as session:
result = await session.execute(
SkillKnowledgeBindingRow.__table__.update()
.where(
SkillKnowledgeBindingRow.skill_name == skill_name,
SkillKnowledgeBindingRow.active.is_(True),
SkillKnowledgeBindingRow.status.in_(["synced", "partial_failed", "pending_remote_write"]),
)
.values(status="stale", updated_at=_now())
)
await session.commit()
return int(result.rowcount or 0)
def make_skill_knowledge_store(session_factory):
return SkillKnowledgeRepository(session_factory) if session_factory is not None else None
@@ -3,7 +3,7 @@
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from datetime import UTC, datetime, timedelta, timezone
from typing import Any
from sqlalchemy import JSON, DateTime, Text
@@ -14,6 +14,12 @@ from sqlalchemy.types import TypeDecorator
# fixed offset is exact and avoids depending on the `tzdata` package (which the
# stdlib `zoneinfo` needs on Windows / minimal containers).
BEIJING_TZ = timezone(timedelta(hours=8))
_POSTGRES_FAMILY_DIALECTS = frozenset({"postgresql", "opengauss"})
def _is_postgres_family(dialect) -> bool:
"""Return whether *dialect* preserves timezone-aware timestamps natively."""
return dialect.name in _POSTGRES_FAMILY_DIALECTS
class PortableJSON(TypeDecorator):
@@ -87,17 +93,17 @@ class BeijingDateTime(TypeDecorator):
def process_bind_param(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
if _is_postgres_family(dialect):
return value
if value.tzinfo is None:
# Naive values follow the codebase convention of being UTC.
value = value.replace(tzinfo=timezone.utc)
value = value.replace(tzinfo=UTC)
return value.astimezone(BEIJING_TZ).replace(tzinfo=None)
def process_result_value(self, value: Any, dialect) -> Any:
if not isinstance(value, datetime):
return value
if dialect.name == "postgresql":
if _is_postgres_family(dialect):
return value
if value.tzinfo is None:
# Stored as naive Beijing wall-clock; re-attach the zone.
@@ -0,0 +1,10 @@
"""Skill knowledge distillation primitives.
The package is deliberately independent from ``app.*``. Gateway composition
injects persistence and remote adapters at runtime.
"""
from deerflow.skill_knowledge.extractor import build_skill_ir
from deerflow.skill_knowledge.scanner import ScanResult, enrich_converted_documents, scan_skill
__all__ = ["ScanResult", "build_skill_ir", "enrich_converted_documents", "scan_skill"]
@@ -0,0 +1,152 @@
"""IR construction and validation.
V1 intentionally starts with deterministic, evidence-backed extraction. The
schema is compatible with a future model-backed extractor, while never letting
unvalidated model output flow directly into persistence or graph writers.
"""
from __future__ import annotations
import hashlib
import re
from typing import Any
from deerflow.skill_knowledge.scanner import ScanResult
EXTRACTOR_VERSION = "skill-knowledge-v1"
PROMPT_HASH = f"sha256:{hashlib.sha256(b'skill-knowledge-v1-evidence-only').hexdigest()}"
PREDICATE_PATTERNS = {
"depends_on": re.compile(r"(?:依赖|需要|requires?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"uses": re.compile(r"(?:调用|使用|uses?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"produces": re.compile(r"(?:生成|输出|produces?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
"consumes": re.compile(r"(?:读取|输入|consumes?)\s*[`\"“]?([^,。;;\n`\"”]{2,80})", re.I),
}
def _evidence(path: str, text: str, start: int, end: int) -> dict[str, Any]:
line_start = text.count("\n", 0, start) + 1
line_end = text.count("\n", 0, end) + 1
quote = " ".join(text[start:end].split())[:240]
return {"file_path": path, "line_start": line_start, "line_end": line_end, "quote": quote}
def build_skill_ir(scan: ScanResult, *, display_name: str | None = None, description: str = "") -> dict[str, Any]:
entities: list[dict[str, Any]] = [
{
"local_id": "ent_skill",
"type": "skill",
"name": display_name or scan.skill_name,
"aliases": [scan.skill_name] if display_name and display_name != scan.skill_name else [],
"description": description,
"attributes": {},
"evidence": [],
"confidence": 1.0,
}
]
relations: list[dict[str, Any]] = []
by_key: dict[tuple[str, str], str] = {("skill", (display_name or scan.skill_name).casefold()): "ent_skill"}
for file in scan.files:
if not file.text:
continue
for predicate, pattern in PREDICATE_PATTERNS.items():
for match in pattern.finditer(file.text):
name = match.group(1).strip(" `\"'“”()[]")
if not name or len(name) > 80:
continue
key = ("concept", name.casefold())
entity_id = by_key.get(key)
if entity_id is None:
entity_id = f"ent_{len(entities):04d}"
by_key[key] = entity_id
entities.append(
{
"local_id": entity_id,
"type": "concept",
"name": name,
"aliases": [],
"description": "从技能业务说明中抽取的知识对象",
"attributes": {},
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
"confidence": 0.82,
}
)
relations.append(
{
"local_id": f"rel_{len(relations):04d}",
"source_local_id": "ent_skill",
"predicate": predicate,
"target_local_id": entity_id,
"description": " ".join(match.group(0).split())[:240],
"relation_source": "explicit_text",
"evidence": [_evidence(file.path, file.text, match.start(), match.end())],
"confidence": 0.82,
}
)
for ref in file.references:
name = ref["target_path"]
key = ("file", name.casefold())
entity_id = by_key.get(key)
if entity_id is None:
entity_id = f"ent_{len(entities):04d}"
by_key[key] = entity_id
entities.append(
{
"local_id": entity_id,
"type": "file",
"name": name,
"aliases": [],
"description": "技能文档引用的资料文件",
"attributes": {},
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
"confidence": 1.0,
}
)
relations.append(
{
"local_id": f"rel_{len(relations):04d}",
"source_local_id": "ent_skill",
"predicate": "references",
"target_local_id": entity_id,
"description": f"{file.path} 引用 {name}",
"relation_source": "deterministic_structure",
"evidence": [{"file_path": file.path, "line_start": 1, "line_end": 1, "quote": name}],
"confidence": 1.0,
}
)
title = display_name or scan.skill_name
summaries = [f"- **{item.title}**(`{item.path}`)" for item in scan.files[:100]]
content = "\n".join(
[
f"# {title}",
"",
description or "由技能知识文件安全归纳生成。",
"",
"## 知识资料",
"",
*(summaries or ["- 暂无可归纳知识文件"]),
"",
f"> 源摘要:`{scan.source_digest}`;代码与脚本文件未读取。",
]
)
return {
"extractor_version": EXTRACTOR_VERSION,
"prompt_hash": PROMPT_HASH,
"model_name": "deterministic-v1",
"skill": {
"name": scan.skill_name,
"display_name": title,
"description": description,
"source_root": scan.source_root,
"source_digest": scan.source_digest,
},
"files": scan.manifest,
"entities": entities,
"relations": relations,
"wiki": {
"skill_page": {"slug": f"skills/{scan.skill_name}/overview", "title": title, "content": content},
"pages": [],
},
"safety": {"blocked_files": scan.blocked_files},
}
@@ -0,0 +1,115 @@
"""Guarded graph-write plans.
No caller can submit Cypher. The writer accepts only normalized node/edge
records and executes fixed parameterized templates when explicitly enabled.
"""
from __future__ import annotations
import hashlib
from dataclasses import dataclass
from typing import Any, Protocol
class GraphTransaction(Protocol):
async def run(self, query: str, parameters: dict[str, Any]) -> Any: ...
@dataclass(frozen=True, slots=True)
class GraphWritePlan:
scope: str
nodes: tuple[dict[str, Any], ...]
edges: tuple[dict[str, Any], ...]
NODE_QUERY = """
MERGE (n:DeerFlowKnowledge {scope: $scope, entity_key: $entity_key})
SET n.display_name = $display_name, n.entity_type = $entity_type,
n.binding_id = $binding_id, n.snapshot_id = $snapshot_id, n.job_id = $job_id,
n.confidence = $confidence, n.updated_at = datetime()
""".strip()
EDGE_QUERY = """
MATCH (a:DeerFlowKnowledge {scope: $scope, entity_key: $source_key})
MATCH (b:DeerFlowKnowledge {scope: $scope, entity_key: $target_key})
MERGE (a)-[r:DEERFLOW_RELATION {edge_key: $edge_key}]->(b)
SET r.predicate = $predicate, r.binding_id = $binding_id,
r.snapshot_id = $snapshot_id, r.job_id = $job_id, r.confidence = $confidence,
r.evidence = $evidence, r.updated_at = datetime()
""".strip()
def _key(*parts: str) -> str:
return hashlib.sha256("\x1f".join(parts).encode()).hexdigest()
def plan_from_ir(
ir: dict[str, Any],
*,
target_type: str,
target_id: str,
binding_id: str,
snapshot_id: str,
job_id: str,
) -> GraphWritePlan:
"""Build a stable write plan; callers cannot supply executable Cypher."""
scope = f"{target_type}:{target_id}"
local_keys: dict[str, str] = {}
nodes: list[dict[str, Any]] = []
for entity in ir.get("entities") or []:
entity_type = str(entity.get("type") or "concept")
normalized_name = str(entity.get("normalized_name") or entity.get("name") or "").strip().casefold()
if not normalized_name:
continue
entity_key = _key(entity_type, normalized_name)
local_keys[str(entity.get("local_id") or normalized_name)] = entity_key
nodes.append(
{
"entity_key": entity_key,
"display_name": str(entity.get("name") or normalized_name),
"entity_type": entity_type,
"binding_id": binding_id,
"snapshot_id": snapshot_id,
"job_id": job_id,
"confidence": float(entity.get("confidence") or 1.0),
}
)
edges: list[dict[str, Any]] = []
for relation in ir.get("relations") or []:
source_key = local_keys.get(str(relation.get("source_local_id") or ""))
target_key = local_keys.get(str(relation.get("target_local_id") or ""))
evidence = list(relation.get("evidence") or [])
confidence = float(relation.get("confidence") or 0)
predicate = str(relation.get("predicate") or "related_to")
if not source_key or not target_key or confidence < 0.75 or not evidence:
continue
edges.append(
{
"edge_key": _key(source_key, predicate, target_key, binding_id, snapshot_id),
"source_key": source_key,
"target_key": target_key,
"predicate": predicate,
"binding_id": binding_id,
"snapshot_id": snapshot_id,
"job_id": job_id,
"confidence": confidence,
"evidence": evidence,
}
)
return GraphWritePlan(scope=scope, nodes=tuple(nodes), edges=tuple(edges))
def parameterized_statements(plan: GraphWritePlan) -> list[dict[str, Any]]:
"""Return Neo4j transactional HTTP statements using only fixed templates."""
statements = [{"statement": NODE_QUERY, "parameters": {"scope": plan.scope, **node}} for node in plan.nodes]
statements.extend({"statement": EDGE_QUERY, "parameters": {"scope": plan.scope, **edge}} for edge in plan.edges)
return statements
async def execute_plan(tx: GraphTransaction, plan: GraphWritePlan, *, enabled: bool) -> dict[str, int]:
if not enabled:
return {"nodes": 0, "edges": 0}
for node in plan.nodes:
await tx.run(NODE_QUERY, {"scope": plan.scope, **node})
for edge in plan.edges:
await tx.run(EDGE_QUERY, {"scope": plan.scope, **edge})
return {"nodes": len(plan.nodes), "edges": len(plan.edges)}
@@ -0,0 +1,31 @@
"""Portable entity/relation normalization used by both knowledge products."""
from __future__ import annotations
import re
import unicodedata
def normalize_name(value: str) -> str:
text = unicodedata.normalize("NFKC", value or "").strip().casefold()
text = text.strip("`'\"“”‘’《》〈〉[](){}")
text = re.sub(r"\s+", " ", text)
return text
def normalize_predicate(value: str) -> str:
normalized = normalize_name(value).replace(" ", "_")
aliases = {
"依赖": "depends_on",
"需要": "depends_on",
"requires": "depends_on",
"调用": "uses",
"使用": "uses",
"uses": "uses",
"生成": "produces",
"输出": "produces",
"读取": "consumes",
"输入": "consumes",
"引用": "references",
}
return aliases.get(normalized, normalized or "related_to")
@@ -0,0 +1,351 @@
"""Safe, deterministic scanner for the knowledge-bearing part of a skill."""
from __future__ import annotations
import csv
import hashlib
import json
import re
import zipfile
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
CODE_SUFFIXES = {
".py",
".pyc",
".pyo",
".ts",
".tsx",
".js",
".jsx",
".mjs",
".cjs",
".sh",
".bash",
".zsh",
".fish",
".bat",
".cmd",
".ps1",
".psm1",
".java",
".kt",
".kts",
".go",
".rs",
".c",
".cc",
".cpp",
".h",
".hpp",
".cs",
".php",
".rb",
".pl",
".lua",
".swift",
".scala",
".sql",
".vue",
".svelte",
}
TEXT_SUFFIXES = {".md", ".mdx", ".txt", ".json", ".yaml", ".yml", ".toml", ".ini", ".csv", ".tsv"}
BINARY_KNOWLEDGE_SUFFIXES = {".xlsx", ".xls", ".pdf", ".docx", ".pptx", ".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".zip"}
EXCLUDED_DIRS = {".git", "node_modules", "__pycache__", ".pytest_cache", "dist", "build", ".mypy_cache", ".ruff_cache", ".address-edits"}
EXCLUDED_FILENAMES = {
"package-lock.json",
"pnpm-lock.yaml",
"yarn.lock",
"bun.lockb",
"poetry.lock",
"uv.lock",
"pipfile.lock",
"cargo.lock",
"composer.lock",
}
SENSITIVE_NAME_RE = re.compile(
r"(^|[._-])(\.env|id_rsa|id_dsa|id_ecdsa|id_ed25519|private[_-]?key|credentials?|secrets?)([._-]|$)",
re.IGNORECASE,
)
SECRET_CONTENT_RE = re.compile(r"(?im)^\s*(?:api[_-]?key|access[_-]?key|secret(?:[_-]?key)?|password|token|private[_-]?key)\s*[:=]\s*[^\s]{8,}\s*$")
PEM_PRIVATE_RE = re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----")
FENCE_RE = re.compile(r"(?ms)^\s*(```|~~~).*?^\s*\1\s*$")
MARKDOWN_LINK_RE = re.compile(r"\[[^\]]+\]\(([^)#?]+)(?:#[^)]*)?\)")
MAX_TEXT_BYTES = 2 * 1024 * 1024
MAX_ARCHIVE_FILES = 1_000
MAX_ARCHIVE_ENTRY_BYTES = 20 * 1024 * 1024
MAX_ARCHIVE_TOTAL_BYTES = 200 * 1024 * 1024
@dataclass(slots=True)
class ScannedFile:
path: str
kind: str
sha256: str
size: int
text: str = ""
title: str = ""
references: list[dict[str, Any]] = field(default_factory=list)
safety: dict[str, Any] = field(default_factory=lambda: {"status": "allowed", "warnings": []})
def manifest(self) -> dict[str, Any]:
return {
"path": self.path,
"kind": self.kind,
"sha256": self.sha256,
"size": self.size,
"text_digest": f"sha256:{hashlib.sha256(self.text.encode('utf-8')).hexdigest()}" if self.text else None,
"title": self.title,
"references": self.references,
"safety": self.safety,
}
@dataclass(slots=True)
class ScanResult:
skill_name: str
source_root: str
source_digest: str
files: list[ScannedFile]
blocked_files: list[dict[str, str]]
skipped_code_count: int
skipped_other_count: int
@property
def manifest(self) -> list[dict[str, Any]]:
return [item.manifest() for item in self.files] + [{"path": item["path"], "kind": "blocked", "safety": {"status": "blocked", "warnings": [item["reason"]]}} for item in self.blocked_files]
@property
def counts(self) -> dict[str, int]:
return {
"knowledge_files": len(self.files),
"blocked_files": len(self.blocked_files),
"skipped_code_files": self.skipped_code_count,
"skipped_other_files": self.skipped_other_count,
}
def _safe_relative(path: Path, root: Path) -> str:
resolved = path.resolve()
resolved_root = root.resolve()
try:
relative = resolved.relative_to(resolved_root)
except ValueError as exc:
raise ValueError("Skill file escapes its source root") from exc
return relative.as_posix()
def _strip_markdown_code(text: str) -> str:
return FENCE_RE.sub("\n[示例代码已按安全策略省略]\n", text)
def _read_text(path: Path, suffix: str) -> str:
raw = path.read_bytes()
if len(raw) > MAX_TEXT_BYTES:
raise ValueError("text file exceeds 2 MiB parsing limit")
text = raw.decode("utf-8-sig", errors="replace")
if suffix in {".md", ".mdx"}:
return _strip_markdown_code(text)
if suffix in {".csv", ".tsv"}:
delimiter = "\t" if suffix == ".tsv" else ","
rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
return "\n".join(" | ".join(cell.strip() for cell in row) for row in rows[:500])
if suffix == ".json":
try:
return json.dumps(json.loads(text), ensure_ascii=False, indent=2)
except json.JSONDecodeError:
return text
return text
def _title(text: str, fallback: str) -> str:
match = re.search(r"(?m)^#\s+(.+?)\s*$", text)
return match.group(1).strip() if match else fallback
def _kind(suffix: str) -> str:
return {
".md": "markdown",
".mdx": "markdown",
".txt": "text",
".csv": "table",
".tsv": "table",
".xlsx": "spreadsheet",
".xls": "spreadsheet",
".pdf": "pdf",
".docx": "office",
".pptx": "office",
".png": "image",
".jpg": "image",
".jpeg": "image",
".webp": "image",
".gif": "image",
".bmp": "image",
".zip": "archive",
}.get(suffix, "structured_data")
def _zip_members(path: Path, archive_relative: str) -> tuple[list[ScannedFile], list[dict[str, str]], int, int]:
files: list[ScannedFile] = []
blocked: list[dict[str, str]] = []
skipped_code = 0
skipped_other = 0
with zipfile.ZipFile(path) as archive:
members = archive.infolist()
if len(members) > MAX_ARCHIVE_FILES:
return [], [{"path": archive_relative, "reason": "压缩包文件数超过安全上限"}], 0, 0
total_size = sum(member.file_size for member in members)
if total_size > MAX_ARCHIVE_TOTAL_BYTES:
return [], [{"path": archive_relative, "reason": "压缩包展开体积超过安全上限"}], 0, 0
for member in members:
member_path = Path(member.filename.replace("\\", "/"))
virtual_path = f"{archive_relative}!/{member_path.as_posix()}"
unix_mode = member.external_attr >> 16
if member.is_dir():
continue
if member_path.is_absolute() or ".." in member_path.parts or (unix_mode & 0o170000) == 0o120000:
blocked.append({"path": virtual_path, "reason": "压缩包包含路径穿越或符号链接"})
continue
suffix = member_path.suffix.lower()
lower_name = member_path.name.lower()
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
skipped_code += 1
continue
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
blocked.append({"path": virtual_path, "reason": "疑似密钥或敏感配置文件"})
continue
if suffix not in TEXT_SUFFIXES or member.file_size > MAX_ARCHIVE_ENTRY_BYTES:
skipped_other += 1
continue
raw = archive.read(member)
text = raw.decode("utf-8-sig", errors="replace")
if suffix in {".md", ".mdx"}:
text = _strip_markdown_code(text)
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
blocked.append({"path": virtual_path, "reason": "内容命中高置信敏感信息规则"})
continue
files.append(
ScannedFile(
path=virtual_path,
kind=_kind(suffix),
sha256=hashlib.sha256(raw).hexdigest(),
size=len(raw),
text=text,
title=_title(text, member_path.stem),
)
)
return files, blocked, skipped_code, skipped_other
def scan_skill(skill_name: str, skill_root: Path) -> ScanResult:
"""Scan one already-resolved skill directory without following symlinks."""
root = skill_root.resolve()
if not root.is_dir():
raise FileNotFoundError(f"Skill '{skill_name}' source directory does not exist")
files: list[ScannedFile] = []
blocked: list[dict[str, str]] = []
skipped_code = 0
skipped_other = 0
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix().lower()):
if path.is_symlink() or not path.is_file():
continue
relative = _safe_relative(path, root)
if any(part.lower() in EXCLUDED_DIRS for part in Path(relative).parts[:-1]):
continue
lower_name = path.name.lower()
suffix = path.suffix.lower()
if lower_name in EXCLUDED_FILENAMES or suffix in CODE_SUFFIXES:
skipped_code += 1
continue
if SENSITIVE_NAME_RE.search(lower_name) or suffix in {".pem", ".key", ".p12", ".pfx"}:
blocked.append({"path": relative, "reason": "疑似密钥或敏感配置文件"})
continue
if suffix not in TEXT_SUFFIXES | BINARY_KNOWLEDGE_SUFFIXES:
skipped_other += 1
continue
# Never upload an archive wholesale: it may contain excluded code or a
# blocked secret. Only independently validated knowledge members enter
# the manifest and digest.
if suffix == ".zip":
try:
nested, nested_blocked, nested_code, nested_other = _zip_members(path, relative)
files.extend(nested)
blocked.extend(nested_blocked)
skipped_code += nested_code
skipped_other += nested_other
except (OSError, zipfile.BadZipFile):
blocked.append({"path": relative, "reason": "压缩包损坏或格式不受支持"})
continue
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
text = ""
warnings: list[str] = []
if suffix in TEXT_SUFFIXES:
try:
text = _read_text(path, suffix)
except ValueError as exc:
warnings.append(str(exc))
if text and (SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text)):
blocked.append({"path": relative, "reason": "内容命中高置信敏感信息规则"})
continue
references = []
if suffix in {".md", ".mdx"}:
for target in MARKDOWN_LINK_RE.findall(text):
normalized = target.replace("\\", "/").lstrip("./")
if normalized:
references.append({"target_path": normalized, "evidence": "markdown_link", "confidence": 1.0})
files.append(
ScannedFile(
path=relative,
kind=_kind(suffix),
sha256=digest,
size=len(raw),
text=text,
title=_title(text, path.stem),
references=references,
safety={"status": "allowed", "warnings": warnings},
)
)
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in files).encode("utf-8")
source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
return ScanResult(
skill_name=skill_name,
source_root=str(root),
source_digest=source_digest,
files=files,
blocked_files=blocked,
skipped_code_count=skipped_code,
skipped_other_count=skipped_other,
)
async def enrich_converted_documents(scan: ScanResult) -> ScanResult:
"""Extract Office/PDF/spreadsheet text without modifying the skill tree."""
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, extract_file_to_markdown
allowed: list[ScannedFile] = []
root = Path(scan.source_root)
for item in scan.files:
if Path(item.path).suffix.lower() in CONVERTIBLE_EXTENSIONS:
text = await extract_file_to_markdown(root / item.path)
if text:
text = _strip_markdown_code(text)
if SECRET_CONTENT_RE.search(text) or PEM_PRIVATE_RE.search(text):
scan.blocked_files.append({"path": item.path, "reason": "转换文本命中高置信敏感信息规则"})
continue
item.text = text
item.title = _title(text, item.title)
else:
item.safety["warnings"].append("文档转换不可用,仅保留原始附件")
allowed.append(item)
scan.files = allowed
digest_input = "\n".join(f"{item.path}\0{item.sha256}" for item in scan.files).encode("utf-8")
scan.source_digest = f"sha256:{hashlib.sha256(digest_input).hexdigest()}"
return scan
@@ -256,6 +256,24 @@ async def convert_file_to_markdown(file_path: Path) -> Path | None:
return None
async def extract_file_to_markdown(file_path: Path) -> str | None:
"""Extract a supported document to Markdown without creating a sidecar file.
Knowledge-ingest scanners use this variant because running a read-only
distillation task must never add generated ``.md`` files to a live skill.
Conversion always runs off the event loop; Office/PDF parsers can be CPU
intensive even for deceptively small inputs.
"""
if file_path.suffix.lower() not in CONVERTIBLE_EXTENSIONS:
return None
try:
return await asyncio.to_thread(_do_convert, file_path, _get_pdf_converter())
except Exception as exc:
logger.error("Failed to extract %s as markdown: %s", file_path.name, exc)
return None
# Regex for bold-only lines that look like section headings.
# Targets SEC filing structural headings that pymupdf4llm renders as **bold**
# rather than # Markdown headings (because they use same font size as body text,
@@ -53,6 +53,10 @@ postgres = [
"psycopg-pool>=3.3.0",
]
mysql = ["asyncmy>=0.2.10"]
gauss = [
"asyncpg>=0.29",
"opengauss-sqlalchemy>=2.4,<3",
]
pymupdf = ["pymupdf4llm>=0.0.17"]
[build-system]
@@ -25,12 +25,14 @@ dependencies = [
"hindsight-client>=0.4.22",
"asyncmy>=0.2.11",
"openpyxl>=3.1.5",
"fastembed>=0.7,<0.9",
]
[project.optional-dependencies]
postgres = ["deerflow-harness[postgres]"]
mysql = ["deerflow-harness[mysql]"]
report-collaboration = ["agentscope==2.0.5"]
gauss = ["deerflow-harness[gauss]"]
[dependency-groups]
dev = [
@@ -0,0 +1,180 @@
from __future__ import annotations
import os
from pathlib import Path
from types import SimpleNamespace
import pytest
from app.gateway import deps
from app.gateway.routers import artifacts
class _FakeThreadStore:
def __init__(self, rows: list[dict]) -> None:
self.rows = rows
self.calls: list[dict] = []
async def search(self, **kwargs) -> list[dict]:
self.calls.append(kwargs)
offset = int(kwargs.get("offset", 0))
limit = int(kwargs.get("limit", 100))
return self.rows[offset : offset + limit]
class _FakeAgentStore:
async def list_by_ids(self, agent_ids: list[str], user_id: str) -> list[dict]:
assert user_id == "owner"
agents = {
"analysis-agent": {
"id": "analysis-agent",
"name": "舆情分析智能体",
}
}
return [agents[agent_id] for agent_id in agent_ids if agent_id in agents]
@pytest.mark.asyncio
async def test_artifact_library_aggregates_generated_non_media_files(tmp_path: Path, monkeypatch):
thread_one = tmp_path / "thread-one" / "outputs"
thread_two = tmp_path / "thread-two" / "outputs"
thread_one.mkdir(parents=True)
thread_two.mkdir(parents=True)
report = thread_one / "研判报告.docx"
table = thread_one / "数据汇总.xlsx"
image = thread_one / "配图.png"
audio = thread_two / "录音.mp3"
notes = thread_two / "结论.md"
archive = thread_two / "证据.zip"
for path, content in (
(report, b"doc"),
(table, b"sheet"),
(image, b"image"),
(audio, b"audio"),
(notes, b"# notes"),
(archive, b"zip"),
):
path.write_bytes(content)
os.utime(report, (100, 100))
os.utime(table, (200, 200))
os.utime(notes, (300, 300))
os.utime(archive, (400, 400))
rows = [
{
"thread_id": "thread-one",
"display_name": "战场环境综合研判",
"metadata": {"agent_id": "analysis-agent"},
},
{
"thread_id": "thread-two",
"display_name": "生成发展报告",
"metadata": {},
},
]
store = _FakeThreadStore(rows)
class _Paths:
def sandbox_outputs_dir(self, thread_id: str, *, user_id: str) -> Path:
assert user_id == "owner"
return tmp_path / thread_id / "outputs"
async def _resolve_owner(_thread_id: str) -> str:
return "owner"
monkeypatch.setattr(artifacts, "get_paths", lambda: _Paths())
monkeypatch.setattr(artifacts, "aresolve_path_user_id", _resolve_owner)
items = await artifacts._scan_artifact_library(store)
assert [item["name"] for item in items] == ["证据.zip", "结论.md", "数据汇总.xlsx", "研判报告.docx"]
assert [item["file_kind"] for item in items] == ["archive", "text_data", "spreadsheet", "document"]
assert [item["source_type"] for item in items] == ["general_qa", "general_qa", "agent", "agent"]
assert items[-1]["thread_title"] == "战场环境综合研判"
assert items[-1]["agent_id"] == "analysis-agent"
assert all(item["name"] not in {"配图.png", "录音.mp3"} for item in items)
assert store.calls[0]["exclude_system"] is True
@pytest.mark.asyncio
async def test_artifact_library_endpoint_filters_searches_and_pages(tmp_path: Path, monkeypatch):
output_dir = tmp_path / "thread-one" / "outputs"
agent_output_dir = tmp_path / "thread-agent" / "outputs"
output_dir.mkdir(parents=True)
agent_output_dir.mkdir(parents=True)
(output_dir / "任务报告.md").write_text("report", encoding="utf-8")
(output_dir / "任务数据.csv").write_text("a,b", encoding="utf-8")
(agent_output_dir / "智能体结论.md").write_text("agent report", encoding="utf-8")
store = _FakeThreadStore(
[
{
"thread_id": "thread-one",
"display_name": "任务研判对话",
"metadata": {},
},
{
"thread_id": "thread-agent",
"display_name": "智能体研判对话",
"metadata": {"agent_id": "analysis-agent"},
},
]
)
class _Paths:
def sandbox_outputs_dir(self, thread_id: str, *, user_id: str) -> Path:
return tmp_path / thread_id / "outputs"
async def _resolve_owner(_thread_id: str) -> str:
return "owner"
monkeypatch.setattr(artifacts, "get_paths", lambda: _Paths())
monkeypatch.setattr(artifacts, "aresolve_path_user_id", _resolve_owner)
monkeypatch.setattr(artifacts, "get_effective_user_id", lambda: "owner")
monkeypatch.setattr(deps, "get_thread_store", lambda _request: store)
monkeypatch.setattr(deps, "get_agent_store", lambda _request: _FakeAgentStore())
artifacts._artifact_library_cache.clear()
raw_endpoint = artifacts.list_artifact_library.__wrapped__
response = await raw_endpoint(
request=SimpleNamespace(),
query="研判对话",
file_kind="text_data",
source_type="general_qa",
offset=0,
limit=1,
refresh=True,
)
assert response.total == 1
assert response.offset == 0
assert response.limit == 1
assert [item.name for item in response.items] == ["任务报告.md"]
spreadsheet_response = await raw_endpoint(
request=SimpleNamespace(),
query="任务数据",
file_kind="spreadsheet",
source_type="all",
offset=0,
limit=50,
refresh=False,
)
assert spreadsheet_response.total == 1
assert spreadsheet_response.items[0].name == "任务数据.csv"
agent_response = await raw_endpoint(
request=SimpleNamespace(),
query=None,
file_kind="all",
source_type="agent",
offset=0,
limit=50,
refresh=False,
)
assert agent_response.total == 1
assert agent_response.items[0].source_type == "agent"
assert agent_response.items[0].agent_name == "舆情分析智能体"
assert agent_response.items[0].name == "智能体结论.md"
@@ -0,0 +1,458 @@
import pytest
import pytest_asyncio
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway.llmwiki_rag import _source_from_assistant_result, _source_from_result
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
from deerflow.persistence.assistant_knowledge.model import (
AssistantKnowledgeEntityContributionRow,
AssistantKnowledgeEntityRow,
)
from deerflow.persistence.base import Base
class FakeEmbeddingClient:
dimensions = 3
fingerprint = "test-bge-m3"
async def embed_query(self, _query: str) -> list[float]:
return [1.0, 0.0, 0.0]
@pytest_asyncio.fixture
async def repository():
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as connection:
await connection.run_sync(Base.metadata.create_all)
yield AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
await engine.dispose()
@pytest.mark.asyncio
async def test_single_base_entity_alignment_search_and_revision(repository: AssistantKnowledgeRepository) -> None:
first = await repository.initialize(created_by="admin")
second = await repository.initialize(created_by="someone-else")
assert first["id"] == second["id"]
package = {
"source": {"type": "skill"},
"wiki_pages": [{"slug": "skills/demo/overview", "title": "示例知识", "content": "张三负责测试。"}],
"entities": [{"local_id": "e1", "type": "person", "name": "张三", "aliases": ["Zhang San"]}],
"relations": [],
"documents": [],
"chunks": [],
"graph": {"nodes": [], "edges": []},
}
await repository.import_package(
package=package,
source_type="skill",
source_key="demo",
source_name="示例技能",
trigger="skill_direct",
created_by="admin",
skill_name="demo",
)
await repository.import_package(
package=package,
source_type="weknora",
source_key="kb-1",
source_name="普通库",
trigger="manual_from_weknora",
created_by="admin",
)
results = await repository.list_pages(query="张三")
alias_results = await repository.list_pages(query="Zhang San")
revisions = await repository.list_revisions("skills/demo/overview")
assert {row["page_type"] for row in results} == {"source", "entity"}
assert [row["page_type"] for row in alias_results] == ["entity"]
assert len(revisions) == 2
@pytest.mark.asyncio
async def test_delete_source_keeps_entity_until_last_contribution_is_removed(
repository: AssistantKnowledgeRepository,
) -> None:
package = {
"source": {},
"wiki_pages": [],
"entities": [{"local_id": "e1", "type": "concept", "name": "共享实体"}],
"relations": [],
}
for source_key in ("one", "two"):
await repository.import_package(
package=package,
source_type="skill",
source_key=source_key,
source_name=source_key,
trigger="skill_direct",
created_by="admin",
skill_name=source_key,
)
sources = await repository.list_sources()
await repository.delete_source(next(row["id"] for row in sources if row["source_key"] == "one"))
async with repository._sf() as session:
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
contributions = (await session.execute(select(AssistantKnowledgeEntityContributionRow))).scalars().all()
assert entity.status == "active"
assert len(contributions) == 1
await repository.delete_source(next(row["id"] for row in sources if row["source_key"] == "two"))
async with repository._sf() as session:
entity = (await session.execute(select(AssistantKnowledgeEntityRow))).scalar_one()
assert entity.status == "deleted"
@pytest.mark.asyncio
async def test_custom_base_import_is_also_mirrored_to_global_base(
repository: AssistantKnowledgeRepository,
) -> None:
global_base = await repository.initialize(created_by="admin")
custom_base = await repository.create_base(
created_by="admin",
name="专题助手库",
description="只承接某个专题的 WeKnora 导出数据",
)
package = {
"source": {"type": "weknora"},
"wiki_pages": [{"slug": "wiki/topic", "title": "专题 Wiki", "content": "王五负责统筹。"}],
"entities": [{"local_id": "e1", "type": "person", "name": "王五"}],
"relations": [],
}
job = await repository.import_package(
package=package,
source_type="weknora",
source_key="kb-topic",
source_name="专题普通库",
trigger="manual_from_weknora",
created_by="admin",
mapping_id="kb-topic",
base_id=custom_base["id"],
)
assert job["base_id"] == custom_base["id"]
assert job["mirrored_global_base_id"] == global_base["id"]
assert len(await repository.list_pages(base_id=custom_base["id"], query="王五")) == 2
assert len(await repository.list_pages(base_id=global_base["id"], query="王五")) == 2
bases = await repository.list_bases()
assert [base["id"] for base in bases[:2]] == [global_base["id"], custom_base["id"]]
@pytest.mark.asyncio
async def test_imported_vectors_are_reused_for_assistant_search(
repository: AssistantKnowledgeRepository,
) -> None:
base = await repository.initialize(created_by="admin")
job = await repository.import_package(
package={
"source": {"type": "weknora"},
"wiki_pages": [{"slug": "wiki/vector", "title": "向量页", "content": "页面正文"}],
"vectors": [
{
"wiki_slug": "wiki/vector",
"section_index": 0,
"section_content": "张三负责向量化检索。",
"embedding_model": "bge-embedding-m3",
"embedding_fingerprint": "test-bge-m3",
"embedding_dimensions": 3,
"vector": [1.0, 0.0, 0.0],
},
{
"wiki_slug": "wiki/vector",
"section_index": 1,
"section_content": "无关片段。",
"embedding_model": "bge-embedding-m3",
"embedding_fingerprint": "test-bge-m3",
"embedding_dimensions": 3,
"vector": [0.0, 1.0, 0.0],
},
],
},
source_type="weknora",
source_key="kb-vector",
source_name="向量普通库",
trigger="manual_package_upload",
created_by="admin",
)
assert job["counts"]["vectors"] == 2
results = await repository.search_pages(
base_ids=[base["id"]],
query="这个查询不靠关键词",
embedding_client=FakeEmbeddingClient(),
)
assert results[0]["retrieval_mode"] == "vector"
assert results[0]["matched_section"]["embedding_model"] == "bge-embedding-m3"
assert results[0]["content_markdown"] == "张三负责向量化检索。"
sources = await repository.list_sources()
assert await repository.delete_source(sources[0]["id"])
assert await repository.search_pages(
base_ids=[base["id"]],
query="这个查询不靠关键词",
embedding_client=FakeEmbeddingClient(),
) == []
@pytest.mark.asyncio
async def test_reimport_retires_pages_removed_from_source_snapshot(
repository: AssistantKnowledgeRepository,
) -> None:
base = await repository.initialize(created_by="admin")
common = {
"source_type": "weknora",
"source_key": "kb-refresh",
"source_name": "自动同步普通库",
"trigger": "vectorization_completed",
"created_by": "admin",
"mapping_id": "kb-refresh",
}
await repository.import_package(
package={
"source": {"type": "weknora", "knowledge_base_id": "kb-refresh"},
"wiki_pages": [
{"slug": "wiki/kept", "title": "保留页", "content": "保留内容"},
{"slug": "wiki/removed", "title": "删除页", "content": "旧内容"},
],
},
**common,
)
await repository.import_package(
package={
"source": {"type": "weknora", "knowledge_base_id": "kb-refresh"},
"wiki_pages": [
{"slug": "wiki/kept", "title": "保留页", "content": "新内容"},
],
},
**common,
)
pages = await repository.list_pages(base_id=base["id"], include_chunks=True)
assert [page["slug"] for page in pages] == ["wiki/kept"]
assert pages[0]["content_markdown"] == "新内容"
@pytest.mark.asyncio
async def test_custom_base_can_be_renamed_and_deleted_without_affecting_global_base(
repository: AssistantKnowledgeRepository,
) -> None:
global_base = await repository.initialize(created_by="admin")
custom_base = await repository.create_base(
created_by="admin",
name="待编辑助手库",
description="旧描述",
)
await repository.import_package(
package={
"source": {"type": "weknora"},
"wiki_pages": [
{"slug": "wiki/delete-test", "title": "删除测试", "content": "保留在总库中的内容"}
],
"entities": [{"local_id": "e1", "type": "concept", "name": "删除测试实体"}],
"relations": [],
},
source_type="weknora",
source_key="kb-delete-test",
source_name="删除测试普通库",
trigger="manual_from_weknora",
created_by="admin",
mapping_id="kb-delete-test",
base_id=custom_base["id"],
)
updated = await repository.update_base(
custom_base["id"],
name="已编辑助手库",
description="新描述",
)
assert updated is not None
assert updated["name"] == "已编辑助手库"
assert updated["description"] == "新描述"
assert await repository.delete_base(custom_base["id"])
assert await repository.get_base(custom_base["id"]) is None
assert all(base["id"] != custom_base["id"] for base in await repository.list_bases())
assert await repository.list_pages(base_id=global_base["id"], query="保留在总库")
with pytest.raises(ValueError, match="不能删除"):
await repository.delete_base(global_base["id"])
@pytest.mark.asyncio
async def test_global_base_aligns_same_entity_title_and_searches_knowledge_body(
repository: AssistantKnowledgeRepository,
) -> None:
base = await repository.initialize(created_by="admin")
first_package = {
"source": {"provider": "weknora", "knowledge_base_id": "kb-one", "knowledge_base_name": "人员库"},
"wiki_pages": [
{
"slug": "people/zhang-san",
"title": "张 三",
"page_type": "article",
"content": "张三负责设备巡检。",
"summary": "摘要专用词:月度综述。",
}
],
"entities": [
{
"local_id": "one",
"type": "人物",
"name": "张 三",
"aliases": ["Zhang San"],
"source_wiki_slug": "people/zhang-san",
}
],
}
second_package = {
"source": {"provider": "weknora", "knowledge_base_id": "kb-two", "knowledge_base_name": "职责库"},
"wiki_pages": [
{
"slug": "entity/person-001",
"title": "Zhang San",
"page_type": "concept",
"content": "张三负责系统故障处置。",
"summary": "另一条摘要。",
}
],
"entities": [
{
"local_id": "two",
"type": "person",
"name": "张三",
"aliases": ["Zhang San"],
"source_wiki_slug": "entity/person-001",
}
],
}
for package, source_key in ((first_package, "kb-one"), (second_package, "kb-two")):
await repository.import_package(
package=package,
source_type="weknora",
source_key=source_key,
source_name=source_key,
trigger="vectorization_completed",
created_by="admin",
)
pages = await repository.list_pages(base_id=base["id"])
assert len(pages) == 1
assert "张三负责设备巡检。" in pages[0]["content_markdown"]
assert "张三负责系统故障处置。" in pages[0]["content_markdown"]
assert len(pages[0]["origins"]) == 2
assert await repository.search_pages(
base_ids=[base["id"]],
query="月度综述",
embedding_client=None,
) == []
results = await repository.search_pages(
base_ids=[base["id"]],
query="系统故障处置",
embedding_client=None,
)
assert len(results) == 1
assert "系统故障处置" in results[0]["matched_section"]["content"]
assert "月度综述" not in results[0]["matched_section"]["content"]
async with repository._sf() as session:
entities = (
await session.execute(
select(AssistantKnowledgeEntityRow).where(
AssistantKnowledgeEntityRow.base_id == base["id"],
AssistantKnowledgeEntityRow.status == "active",
)
)
).scalars().all()
assert len(entities) == 1
contributions = (
await session.execute(
select(AssistantKnowledgeEntityContributionRow).where(
AssistantKnowledgeEntityContributionRow.entity_id == entities[0].id
)
)
).scalars().all()
assert len(contributions) == 2
@pytest.mark.asyncio
async def test_uploaded_package_reuses_embedded_weknora_source_identity(
repository: AssistantKnowledgeRepository,
) -> None:
base = await repository.initialize(created_by="admin")
for upload_no, content in ((1, "旧知识"), (2, "新知识")):
job = await repository.create_queued_import_job(
source_type="weknora",
source_key=f"upload:{upload_no}",
source_name=f"upload-{upload_no}.jsonl",
trigger="manual_package_upload",
created_by="admin",
base_id=base["id"],
)
await repository.import_package_for_job(
job["id"],
package={
"source": {
"provider": "weknora",
"knowledge_base_id": "stable-kb-id",
"knowledge_base_name": "同一个普通知识库",
},
"wiki_pages": [
{
"slug": "wiki/stable",
"title": "稳定实体",
"content": content,
}
],
},
created_by="admin",
)
active_sources = await repository.list_sources(base_id=base["id"])
assert len(active_sources) == 1
assert active_sources[0]["source_key"] == "stable-kb-id"
pages = await repository.list_pages(base_id=base["id"])
assert len(pages) == 1
assert pages[0]["content_markdown"] == "新知识"
def test_assistant_citation_uses_matched_knowledge_instead_of_summary() -> None:
source = _source_from_assistant_result(
{
"id": "page-1",
"title": "设备知识",
"summary": "只是一段摘要",
"content_markdown": "完整页面正文",
"matched_section": {
"content": "实际命中的知识片段:设备过热时先断电。",
},
"score": 0.91,
},
1,
)
assert source["content"] == "实际命中的知识片段:设备过热时先断电。"
assert source["snippet"] == "实际命中的知识片段:设备过热时先断电。"
assert "只是一段摘要" not in source["snippet"]
def test_regular_wiki_citation_does_not_fall_back_to_summary() -> None:
source = _source_from_result(
{
"kind": "wiki_page",
"wiki_page_id": "wiki-1",
"title": "设备知识",
"summary": "只是一段摘要",
"content": "实际知识正文",
"matched_sections": [],
},
{"id": "kb-1", "name": "普通知识库"},
1,
)
assert source["content"] == "实际知识正文"
assert source["snippet"] == "实际知识正文"
assert "只是一段摘要" not in source["snippet"]
Loaded 100 of 109 files, more files were not shown because too many files have changed in this diff. Show more