diff --git a/batch-distill/distill_people.py b/batch-distill/distill_people.py index 3e422cc5..d5c4d171 100644 --- a/batch-distill/distill_people.py +++ b/batch-distill/distill_people.py @@ -44,7 +44,7 @@ except ImportError: # ════════════════════════════════════════════════════════════════════════ # 只改这一行：你的线上服务地址（不带结尾斜杠） # ════════════════════════════════════════════════════════════════════════ -BASE_URL = "http://localhost:8001" +BASE_URL = "http://47.94.209.59:2026/" # ════════════════════════════════════════════════════════════════════════ @@ -57,7 +57,7 @@ DEPTH = "full" # full 完整 / fast 精简 HERE = Path(__file__).resolve().parent OUTPUT_DIR = HERE / "output" -PEOPLE_FILE_DEFAULT = HERE / "people.xlsx" +PEOPLE_FILE_DEFAULT = HERE / "people.example.csv" # 执行时间窗：23:00 ~ 次日 08:00（跨午夜） WIN_START = dtime(23, 0) @@ -260,35 +260,63 @@ class DeerFlowClient: return "timeout" def fetch_result_text(self, thread_id: str, run_id: str) -> str | None: - """抓取该 run 的最终 AI 回复文本（由 py 保存为 md）。""" + """抓取该 thread 的最终 AI 回复文本（由 py 保存为 md）。 + + 注意：本部署的 `/runs/{rid}/messages`、`/messages`、`/events` 端点都返回空 + （事件库按登录用户过滤，后台 worker 写入时未带 user_id → 读不到）。 + 可靠来源是 LangGraph 检查点状态 `/api/threads/{tid}/state` → values.messages， + 其中每条消息形如 {"type": "human"|"ai"|"tool", "content": , ...}。 + 取最后一条有正文的 ai 消息即可。 + """ try: - r = self.s.get(f"{self.base}/api/threads/{thread_id}/runs/{run_id}/messages", - headers=self._headers(), params={"limit": 200}, timeout=HTTP_TIMEOUT) - if r.status_code == 200: - data = r.json().get("data", []) - for msg in reversed(data): - if msg.get("event_type") == "ai_message" or msg.get("type") == "ai": - txt = _extract_text(msg) - if txt and txt.strip(): - return txt + r = self.s.get(f"{self.base}/api/threads/{thread_id}/state", + headers=self._headers(), timeout=HTTP_TIMEOUT) + if r.status_code != 200: + print(f" 取回复返回 {r.status_code}：{r.text[:200]}") + return None + messages = (r.json().get("values") or {}).get("messages") or [] + # 从后往前找最后一条「有正文」的 AI 回复（跳过只含工具调用的中间消息） + for msg in reversed(messages): + if not isinstance(msg, dict) or msg.get("type") != "ai": + continue + txt = _extract_text(msg) + if txt and txt.strip(): + return txt except requests.RequestException as e: print(f" 取回复出错：{e}") return None -def _extract_text(msg: dict) -> str: - c = msg.get("content") or msg.get("text") or "" +def _extract_text(payload) -> str: + """从一条消息的 content 里提取纯文本。 + + payload 可能是： + - LangChain 消息的 model_dump 字典（正文在 payload["content"]） + - 直接的字符串 + - 内容块列表（[{"type":"text","text":"..."}, ...]） + """ + if payload is None: + return "" + # 若是整条消息的 model_dump，正文在其 content 字段 + if isinstance(payload, dict): + c = payload.get("content", payload.get("text", "")) + else: + c = payload if isinstance(c, str): return c if isinstance(c, list): parts = [] for b in c: if isinstance(b, dict): - parts.append(b.get("text") or b.get("content") or "") - else: - parts.append(str(b)) + # 只取文本块，跳过 tool_use / thinking 等非文本块 + if b.get("type") in (None, "text", "text_block"): + parts.append(b.get("text") or b.get("content") or "") + elif b.get("text"): + parts.append(b["text"]) + elif isinstance(b, str): + parts.append(b) return "\n".join(p for p in parts if p) - return str(c) + return str(c) if c else "" def _clean_markdown(text: str) -> str: diff --git a/frontend-web/src/components/workspace/agent-info-sidebar.tsx b/frontend-web/src/components/workspace/agent-info-sidebar.tsx index 803add70..b9f03351 100644 --- a/frontend-web/src/components/workspace/agent-info-sidebar.tsx +++ b/frontend-web/src/components/workspace/agent-info-sidebar.tsx @@ -2,35 +2,31 @@ import { BotIcon, - CheckIcon, InfoIcon, MessageCircleQuestionIcon, PanelRightCloseIcon, PanelRightOpenIcon, - PencilIcon, - PlusIcon, SparklesIcon, - XIcon, } from "lucide-react"; -import { useEffect, useMemo, useState } from "react"; -import { toast } from "sonner"; +import { useMemo, useState } from "react"; import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; import { SkillDetailDialog } from "@/components/workspace/skills/skill-detail-dialog"; import { Tooltip } from "@/components/workspace/tooltip"; import { type Agent } from "@/core/agents"; -import { useUpdateAgent } from "@/core/agents/hooks"; -import { getStoredAuth } from "@/core/auth"; import { useSkills } from "@/core/skills/hooks"; import { desensitize } from "@/lib/desensitize"; import { cn } from "@/lib/utils"; /** - * Collapsible right-hand panel on the agent chat page that surfaces the agent's - * profile (name / description / skills) plus a list of frequently-asked - * questions ("常用问题"). Clicking a question sends it as a message. The agent's - * owner (or an admin) can edit the question list inline. + * Collapsible right-hand "智能体简介" panel on the agent chat page. It is a + * read-only window onto the agent's profile — avatar, name, slogan/description, + * the admin-authored SOUL.md as a long-form blurb, the skill whitelist, and the + * admin-configured common questions surfaced as quick-ask entries. Clicking a + * question sends it as a message. + * + * Editing the common questions lives in the "编辑智能体" dialog (agent-card), NOT + * here — this panel never offers an edit affordance or an admin hint. */ export function AgentInfoSidebar({ agent, @@ -49,20 +45,12 @@ export function AgentInfoSidebar({ const [collapsed, setCollapsed] = useState(false); const [detailSkillName, setDetailSkillName] = useState(null); const { skills } = useSkills(); - const updateAgent = useUpdateAgent(); const displayName = desensitize(agent?.name ?? agentName); const description = agent?.description ? desensitize(agent.description) : undefined; - - // Owner / admin gate for inline editing of the common-questions list. - const storedAuth = useMemo(() => getStoredAuth(), []); - const canEdit = useMemo(() => { - if (!agent) return false; - if (storedAuth?.system_role === "admin") return true; - return agent.user_id != null && agent.user_id === storedAuth?.user_id; - }, [agent, storedAuth]); + const soul = agent?.soul?.trim() ? desensitize(agent.soul) : undefined; // Map a stored skill name to its Chinese display label (falls back to name). const skillLabel = useMemo(() => { @@ -78,45 +66,9 @@ export function AgentInfoSidebar({ [agent?.common_questions], ); - // --- Inline editing state --- - const [editing, setEditing] = useState(false); - const [drafts, setDrafts] = useState([]); - - const startEditing = () => { - setDrafts(questions.length > 0 ? [...questions] : [""]); - setEditing(true); - }; - - const cancelEditing = () => { - setEditing(false); - setDrafts([]); - }; - - const handleSave = async () => { - if (!agent) return; - const cleaned = drafts.map((q) => q.trim()).filter((q) => q.length > 0); - try { - await updateAgent.mutateAsync({ - id: agent.id, - request: { common_questions: cleaned }, - }); - toast.success("常用问题已保存"); - setEditing(false); - setDrafts([]); - } catch (error) { - toast.error(error instanceof Error ? error.message : String(error)); - } - }; - - // Leaving edit mode whenever the panel collapses keeps the state tidy. - useEffect(() => { - if (collapsed && editing) cancelEditing(); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [collapsed]); - if (collapsed) { return ( -
+
-
+
{/* Identity */} -
-
- +
+
+
-
+
{displayName}
{description && ( -

+

{description}

)}
+ {/* Soul — the agent's long-form personality blurb. */} + {soul && ( +
+

Soul

+

+ {soul} +

+
+ )} + {/* Skills */} {agent?.skills && agent.skills.length > 0 && ( -
-
+
+

技能 -

-
+ +
{agent.skills.map((name) => ( ))}
-
+ )} - {/* Common questions */} -
-
-
+ {/* Common questions — read-only quick-ask entries. */} + {questions.length > 0 && ( +
+

- 常用问题 -

- {canEdit && !editing && ( - - - - )} -
- - {editing ? ( -
- {drafts.map((value, index) => ( -
- { - const next = [...drafts]; - next[index] = e.currentTarget.value; - setDrafts(next); - }} - className="h-8 text-xs" - /> - -
- ))} - -
- - -
-
- ) : questions.length > 0 ? ( -
+ 推荐问题 + +
{questions.map((question, index) => ( ))}
- ) : ( -

- {canEdit ? "还没有常用问题，点击上方编辑添加。" : "暂无常用问题。"} -

- )} -
+ + )}
-
- +
+
-
+
Hi，我是{displayName}
{description && ( -

+

{description}

)} diff --git a/frontend-web/src/components/workspace/agents/agent-card.tsx b/frontend-web/src/components/workspace/agents/agent-card.tsx index f47b010d..427710cb 100644 --- a/frontend-web/src/components/workspace/agents/agent-card.tsx +++ b/frontend-web/src/components/workspace/agents/agent-card.tsx @@ -8,6 +8,7 @@ import { LockIcon, MessageSquareIcon, PinIcon, + PlusIcon, SearchIcon, ShieldIcon, SparklesIcon, @@ -16,6 +17,7 @@ import { TagsIcon, Trash2Icon, UserIcon, + XIcon, } from "lucide-react"; import { useRouter } from "next/navigation"; import { @@ -207,6 +209,12 @@ export function AgentCard({ agent, layout = "card", onPublisherClick }: AgentCar const [editName, setEditName] = useState(agent.name); const [editDescription, setEditDescription] = useState(agent.description ?? ""); const [editSoul, setEditSoul] = useState(agent.soul ?? ""); + // Common questions (常用问题) are authored here in the edit dialog and read + // back read-only on the agent chat page. Keep one trailing blank row so there + // is always an empty input to type the next question into. + const [editCommonQuestions, setEditCommonQuestions] = useState( + agent.common_questions ?? [], + ); const [editPublished, setEditPublished] = useState(agent.published); const [editIsPrivate, setEditIsPrivate] = useState(agent.is_private); const [selectedSkills, setSelectedSkills] = useState( @@ -250,8 +258,10 @@ export function AgentCard({ agent, layout = "card", onPublisherClick }: AgentCar setEditPublished(agent.published); setEditIsPrivate(agent.is_private); setSelectedSkills(agent.skills ?? []); + setEditCommonQuestions(agent.common_questions ?? []); setSkillSearch(""); }, [ + agent.common_questions, agent.description, agent.is_private, agent.name, @@ -269,6 +279,7 @@ export function AgentCard({ agent, layout = "card", onPublisherClick }: AgentCar // include `soul`) keep working. if (agentDetail) { setEditSoul(agentDetail.soul ?? ""); + setEditCommonQuestions(agentDetail.common_questions ?? []); } else { setEditSoul(agent.soul ?? ""); } @@ -345,6 +356,9 @@ export function AgentCard({ agent, layout = "card", onPublisherClick }: AgentCar published: editIsPrivate ? false : editPublished, is_private: editIsPrivate, skills: selectedSkills, + common_questions: editCommonQuestions + .map((q) => q.trim()) + .filter((q) => q.length > 0), }, }); toast.success("智能体已更新"); @@ -831,7 +845,7 @@ export function AgentCard({ agent, layout = "card", onPublisherClick }: AgentCar 编辑智能体:{desensitize(agent.name)} - 可以修改名称、描述、技能白名单、SOUL 内容与发布状态。 + 可以修改名称、描述、技能白名单、SOUL 内容、常用问题与发布状态。
@@ -935,6 +949,52 @@ export function AgentCard({ agent, layout = "card", onPublisherClick }: AgentCar className="min-h-40 font-mono text-xs" />
+
+ +

+ 这些问题会在该智能体的首页与简介栏作为推荐问题展示，点击即可一键提问。 +

+
+ {editCommonQuestions.map((value, index) => ( +
+ { + const next = [...editCommonQuestions]; + next[index] = e.currentTarget.value; + setEditCommonQuestions(next); + }} + className="h-9 text-sm" + /> + +
+ ))} + +
+

公开发布

diff --git a/frontend-web/src/core/knowledge/api.ts b/frontend-web/src/core/knowledge/api.ts index bc4ce250..45cd196e 100644 --- a/frontend-web/src/core/knowledge/api.ts +++ b/frontend-web/src/core/knowledge/api.ts @@ -127,6 +127,30 @@ export async function ingestThreadsBatch(payload: BatchIngestPayload = {}) { ); } +export interface IngestFileResult { + note: KnowledgeNote; + created: boolean; + notes: KnowledgeNote[]; + llm_fallback: boolean; +} + +/** Import a file (md/txt/pdf/word/ppt/excel) into the KB as wiki note(s). */ +export async function ingestKnowledgeFile( + file: File, + opts: { title?: string; tags?: string; status?: string; model_name?: string | null } = {}, +) { + const form = new FormData(); + form.append("file", file); + if (opts.title) form.append("title", opts.title); + if (opts.tags) form.append("tags", opts.tags); + if (opts.status) form.append("status", opts.status); + if (opts.model_name) form.append("model_name", opts.model_name); + // No Content-Type header — the browser sets the multipart boundary itself. + return parseResponse( + await apiFetch(`${baseUrl()}/ingest/file`, { method: "POST", body: form }), + ); +} + /** Resolve `[[wikilink]]` targets to note ids / vault pages. */ export async function resolveWikilinks(targets: string[]) { const res = await parseResponse<{ resolved: WikilinkResolutionMap }>( diff --git a/frontend-web/src/core/knowledge/hooks.ts b/frontend-web/src/core/knowledge/hooks.ts index b3a9b9fb..8f40e309 100644 --- a/frontend-web/src/core/knowledge/hooks.ts +++ b/frontend-web/src/core/knowledge/hooks.ts @@ -11,6 +11,7 @@ import { exportKnowledgeGraph, fetchVaultPage, getKnowledgeNote, + ingestKnowledgeFile, ingestThread, ingestThreadsBatch, listDuplicates, @@ -106,6 +107,16 @@ export function useKnowledgeActions() { mutationFn: ingestThreadsBatch, onSuccess: invalidate, }), + ingestFile: useMutation({ + mutationFn: ({ + file, + opts, + }: { + file: File; + opts?: Parameters[1]; + }) => ingestKnowledgeFile(file, opts), + onSuccess: invalidate, + }), exportGraph: useMutation({ mutationFn: exportKnowledgeGraph }), merge: useMutation({ mutationFn: ({ primaryId, duplicateIds }: { primaryId: string; duplicateIds: string[] }) => diff --git a/frontend-web/src/pages/AgentChatPage.tsx b/frontend-web/src/pages/AgentChatPage.tsx index 1cf155c2..38a3a3e0 100644 --- a/frontend-web/src/pages/AgentChatPage.tsx +++ b/frontend-web/src/pages/AgentChatPage.tsx @@ -13,11 +13,7 @@ import { ArtifactTrigger } from "@/components/workspace/artifacts"; import { ChatBox, useThreadChat } from "@/components/workspace/chats"; import { ExportTrigger } from "@/components/workspace/export-trigger"; import { InputBox } from "@/components/workspace/input-box"; -import { - MessageList, - MESSAGE_LIST_DEFAULT_PADDING_BOTTOM, - MESSAGE_LIST_FOLLOWUPS_EXTRA_PADDING_BOTTOM, -} from "@/components/workspace/messages"; +import { MessageList } from "@/components/workspace/messages"; import { ThreadContext } from "@/components/workspace/messages/context"; import { ThreadTitle } from "@/components/workspace/thread-title"; import { TodoList } from "@/components/workspace/todo-list"; @@ -32,7 +28,6 @@ import { useThreadStream } from "@/core/threads/hooks"; import { textOfMessage } from "@/core/threads/utils"; import { env } from "@/env"; import { desensitize } from "@/lib/desensitize"; -import { cn } from "@/lib/utils"; export default function AgentChatPage() { const routeBase = useRouteBase(); @@ -163,146 +158,156 @@ export default function AgentChatPage() { await thread.stop(); }, [thread]); - const messageListPaddingBottom = showFollowups - ? MESSAGE_LIST_DEFAULT_PADDING_BOTTOM + - MESSAGE_LIST_FOLLOWUPS_EXTRA_PADDING_BOTTOM - : undefined; const isLandingState = isNewThread && !hasStartedConversation; + const isDemoMode = env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true"; + + const status = thread.error + ? "error" + : thread.isLoading + ? "streaming" + : "ready"; return ( -
-
- {/* Agent badge */} -
- - - {agentDisplayName} - -
- -
- -
-
- - - - - - -
-
+ {agentDisplayName} + +
-
-
- handleSubmit({ text, files: [] })} - /> -
+
+ +
-
-
-
-
-
-
+
+ + + + + + +
+ -
+ {isLandingState ? ( + /* ── home: centered hero + suggested questions + input ────── + Everything lives inside the scrollable main column. No fixed, + no absolute, no viewport centering — so the avatar can't touch + the header and the input can't cover the sidebar. */ +
+ handleSubmit({ text, files: [] })} + disabled={isDemoMode} + className="mb-7" + /> +
- handleSubmit({ text, files: [] }) - } - disabled={ - env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" - } - /> - ) + disabled={isDemoMode} + onContextChange={(context) => + setSettings("context", context) } - disabled={env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true"} - onContextChange={(context) => setSettings("context", context)} onFollowupsVisibilityChange={setShowFollowups} onSubmit={handleSubmit} onStop={handleStop} /> + {isDemoMode && ( +
+ {t.common.notAvailableInDemoMode} +
+ )}
- {env.NEXT_PUBLIC_STATIC_WEBSITE_ONLY === "true" && ( -
- {t.common.notAvailableInDemoMode} -
- )}
-
+ ) : ( + /* ── conversation (active / history): messages scroll, input + docked at the bottom of the main column only. */ +
+ handleSubmit({ text, files: [] })} + /> + +
+
+
+
+
+ )}
+ {/* ── Right: read-only agent profile sidebar ───────────────────── */} handleSubmit({ text, files: [] })} />
diff --git a/frontend-web/src/pages/KnowledgeBasePage.tsx b/frontend-web/src/pages/KnowledgeBasePage.tsx index 59c2a3e0..167fa6e5 100644 --- a/frontend-web/src/pages/KnowledgeBasePage.tsx +++ b/frontend-web/src/pages/KnowledgeBasePage.tsx @@ -11,6 +11,7 @@ import { Search, Sparkles, Trash2, + Upload, X, } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; @@ -323,6 +324,119 @@ function BatchImportDialog({ ); } +function ImportFileDialog({ + open, + onOpenChange, +}: { + open: boolean; + onOpenChange: (v: boolean) => void; +}) { + const { ingestFile } = useKnowledgeActions(); + const navigate = useNavigate(); + const [file, setFile] = useState(null); + const [title, setTitle] = useState(""); + const [tags, setTags] = useState(""); + const [model, setModel] = useState(null); + + const reset = () => { + setFile(null); + setTitle(""); + setTags(""); + setModel(null); + }; + + const handleImport = async () => { + if (!file) { + toast.error("请选择文件"); + return; + } + try { + const result = await ingestFile.mutateAsync({ + file, + opts: { + title: title.trim() || undefined, + tags: tags.trim() || undefined, + status: "approved", + model_name: model, + }, + }); + const count = result.notes?.length ?? 1; + if (result.llm_fallback) { + // The model produced nothing usable — the file was stored verbatim as a + // draft instead of distilled into a wiki note. + toast.warning( + `已导入 ${count} 条（大模型异常，改用规则抽取并存为草稿）`, + ); + } else { + toast.success(`已从文件生成 ${count} 条知识`); + } + onOpenChange(false); + reset(); + if (result.note?.id) { + navigate(`/page/workspace/knowledge/${result.note.id}`); + } + } catch (error) { + toast.error(error instanceof Error ? error.message : "文件导入失败"); + } + }; + + return ( + + + + 导入文件生成 Wiki + + 上传文档（Markdown / TXT / PDF / Word / PPT / Excel），自动提炼为知识库 wiki 词条并写入 vault。 + + +
+ + setTitle(e.target.value)} + /> + setTags(e.target.value)} + /> + +
+ + + + + + + ); +} + function DedupDialog({ open, onOpenChange, @@ -698,6 +812,7 @@ export default function KnowledgeBasePage() { const [status, setStatus] = useState("all"); const [createOpen, setCreateOpen] = useState(false); const [batchOpen, setBatchOpen] = useState(false); + const [importOpen, setImportOpen] = useState(false); const [dedupOpen, setDedupOpen] = useState(false); const params = useMemo( @@ -752,6 +867,9 @@ export default function KnowledgeBasePage() {
+ @@ -814,6 +932,7 @@ export default function KnowledgeBasePage() {
+
diff --git a/frontend-web/src/pages/KnowledgeGraphPage.tsx b/frontend-web/src/pages/KnowledgeGraphPage.tsx index 4cacc1a9..93a16b86 100644 --- a/frontend-web/src/pages/KnowledgeGraphPage.tsx +++ b/frontend-web/src/pages/KnowledgeGraphPage.tsx @@ -4,13 +4,17 @@ import { useNavigate } from "react-router-dom"; import { toast } from "sonner"; import { Button } from "@/components/ui/button"; +import { useTheme } from "@/contexts/ThemeContext"; import { knowledgeExportFileUrl } from "@/core/knowledge/api"; import { useKnowledgeActions } from "@/core/knowledge/hooks"; export default function KnowledgeGraphPage() { const navigate = useNavigate(); + const { theme } = useTheme(); const { exportGraph } = useKnowledgeActions(); - const [src, setSrc] = useState(null); + // Keep the served base URL (without the theme) so a theme toggle can re-skin + // the iframe without regenerating the graph on the backend. + const [baseSrc, setBaseSrc] = useState(null); const [stats, setStats] = useState<{ nodes: number; edges: number } | null>(null); const regenerate = useCallback(async () => { @@ -18,7 +22,7 @@ export default function KnowledgeGraphPage() { const result = await exportGraph.mutateAsync(); setStats(result.stats); // Cache-bust the iframe so it reflects the freshly generated graph. - setSrc(`${knowledgeExportFileUrl("graph.html")}?t=${Date.now()}`); + setBaseSrc(`${knowledgeExportFileUrl("graph.html")}?t=${Date.now()}`); } catch (error) { toast.error(error instanceof Error ? error.message : "生成图谱失败"); } @@ -29,6 +33,10 @@ export default function KnowledgeGraphPage() { void regenerate(); }, [regenerate]); + // The graph HTML lives in an iframe and can't read the app's CSS theme, so we + // pass the active day/night mode via a query param; toggling re-renders it. + const src = baseSrc ? `${baseSrc}&theme=${theme}` : null; + return (
diff --git a/offline-backend-20260512/backend/CLAUDE.md b/offline-backend-20260512/backend/CLAUDE.md index cab36879..6b0e6b8d 100644 --- a/offline-backend-20260512/backend/CLAUDE.md +++ b/offline-backend-20260512/backend/CLAUDE.md @@ -250,7 +250,7 @@ FastAPI application on port 8001 with health check at `GET /health`. Set `GATEWA | **Scheduled Tasks** (`/api/scheduled-tasks`) | Per-user cron-like agent tasks. `GET/POST /` - list/create owned tasks; `PATCH/DELETE /{id}` - update/delete (owner); `POST /{id}/{pause,resume,trigger}`; `GET /{id}/runs`, `GET /runs/{rid}`, run-file preview/edit/download; `GET /runs/{rid}/messages` - agent messages for the run's underlying agent run (used to inspect a stuck/running run; authorized via `_load_run`); `GET /runs/{rid}/html` - generated HTML document for an `html_page` run; `DELETE /runs/{rid}` - owner-scoped, with an admin fallback so admins can delete any user's run; publish/subscribe under `/{id}/{publish,unpublish,subscribe}` and `GET /published`. **Admin (cross-user):** `GET /admin/all` - every task; `PATCH/DELETE /admin/{id}` - edit/delete any task; `POST /admin/{id}/trigger`; `GET /admin/{id}/runs`. Admin routes require `system_role == "admin"`. | | **HTML Page Favorites** (`/api/html-page-favorites`) | Bookmarked generated HTML pages, owner-scoped. `GET /` - list; `POST /` `{page_id, title?, description?, tags?}` - bookmark a `scheduled_html_pages` row (extracts a reusable style/layout summary via GLM5 at favorite time); a page can only be favorited once per user (`DuplicateFavoriteError` → HTTP 409). `GET/DELETE /{id}` - read/delete (the create-task reference picker deletes favorites here). A favorite can be selected as a style reference when creating a new `html_page` task. | | **Public Scheduled Tasks** (`/api/public/scheduled-tasks`) | **Unauthenticated** mirror for embedding. Adds `GET /runs/{rid}/html` and `GET /{task_id}/latest-html` for the `html_page` preview; the run result keeps `result.html` (only file paths are redacted). | -| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup). **Optional model selection**: both ingest endpoints accept `model_name` (overrides `knowledge.extract_model_name` for that capture; threaded through `KnowledgeService.capture_thread` → `extract_thread_llm_multi`). **Model-error surfacing**: when the chosen model produces nothing usable (timeout / API error / non-JSON) capture still **falls back to rule-based extraction and lands a draft** (`llm_fallback:true`); the single-thread (non-background) response and the batch response (`llm_fallback` count) carry this so the UI tells the user "已改用规则抽取". The "保存到知识库" chat button stays background/fire-and-forget (passes `model_name`, no per-save error toast); the batch-import dialog is synchronous and shows the fallback count. `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector→keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[…]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). Backed by the `deerflow.knowledge` module (see below). | +| **Knowledge** (`/api/knowledge`) | Product knowledge base, **global/shared** (no user isolation; writes record `created_by`/`updated_by` for audit). `POST /ingest/thread/{thread_id}` - sediment one thread into a note (reads messages from the checkpointer, `threads:read` owner-checked); `POST /ingest/threads/batch` - batch-sediment the caller's recent threads (`skip_existing` via `.manifest.json` dedup). **Optional model selection**: both ingest endpoints accept `model_name` (overrides `knowledge.extract_model_name` for that capture; threaded through `KnowledgeService.capture_thread` → `extract_thread_llm_multi`). **Model-error surfacing**: when the chosen model produces nothing usable (timeout / API error / non-JSON) capture still **falls back to rule-based extraction and lands a draft** (`llm_fallback:true`); the single-thread (non-background) response and the batch response (`llm_fallback` count) carry this so the UI tells the user "已改用规则抽取". The "保存到知识库" chat button stays background/fire-and-forget (passes `model_name`, no per-save error toast); the batch-import dialog is synchronous and shows the fallback count. `POST /ingest/file` - **import an uploaded file** (multipart: `file` + optional `title`/`tags`/`status`/`model_name`) into wiki note(s): the file is converted to Markdown (md/txt read directly, pdf/word/ppt/excel via `convert_file_to_markdown`), distilled by the same wiki-capture LLM pipeline as thread capture (`source_type="file"`), and falls back to a verbatim **draft** note when the model returns nothing usable (`llm_fallback:true`); `POST /ingest/search` - sediment search/tool results as a reference note; `POST /notes` - manual note; `GET /notes` (keyword/tag/source_type/status + pagination); `GET/PUT/DELETE /notes/{id}` (DELETE is soft = archive, `?hard=true` removes the vault file); `POST /search` - `mode=keyword\|vector\|hybrid` (vector→keyword fallback when no embedding endpoint); `GET /notes/{id}/versions` - edit history; `GET /duplicates` + `POST /merge` - dedup; `POST /reindex` - rebuild vector index; `GET /graph` - product graph data (notes + entities); `POST /export/graph` + `GET /export/files/{graph.json\|graph.html\|graph.graphml\|cypher.txt}` - obsidian-wiki `wiki-export` artifacts (self-contained `graph.html` for iframe); `POST /resolve` `{targets:[…]}` + `GET /notes/{id}/wikilinks` - resolve `[[wikilink]]` targets to a note id / vault page / missing (powers clickable wikilinks); `GET /vault-page?path=` - read a raw vault meta/hub Markdown page (no DB note). Backed by the `deerflow.knowledge` module (see below). | Proxied through nginx: `/api/langgraph/*` → LangGraph, all other `/api/*` → Gateway. @@ -290,7 +290,7 @@ Self-contained product knowledge base. **All knowledge capability lives in this - **DB mirror** — tables `knowledge_notes` / `knowledge_sources` / `knowledge_tags` / `knowledge_note_tags` (`knowledge/models.py`, registered in `persistence/models/__init__.py`, auto-created by `Base.metadata.create_all`; portable types `PortableLongText`/`BeijingDateTime`). Powers listing/filter/search/audit. Global/shared — no `user_id` column; `created_by`/`updated_by` are audit-only (resolved via `get_effective_user_id()`). - **Markdown vault** — obsidian-wiki compatible, default `{$DEER_FLOW_HOME}/data/knowledge/obsidian-vault`. `ObsidianWikiAdapter` (`obsidian_adapter.py`) writes wiki-capture frontmatter+body, and after **every** write syncs `index.md` / `log.md` (CAPTURE/INGEST/UPDATE/ARCHIVE/EXPORT) / `hot.md` / `.manifest.json` (source-hash dedup for `skip_existing`). `markdown_vault.py` handles frontmatter build/parse, slugging, wikilink extraction and **path-traversal-proof** resolution. -**Module layout:** `models.py` (ORM: notes/sources/tags/note_tags + embeddings/versions/entities/relations) · `repository.py` (DB only, no LLM) · `schemas.py` (NoteDraft/SourceDraft DTOs) · `skill_templates.py` · `markdown_vault.py` · `manifest.py` · `extractor.py` (rule-based fallback) · `llm_extractor.py` (**default**: distills declarative knowledge per wiki-capture, structured JSON incl. entities/relations; falls back to `extractor.py`) · `sources.py` · `exporter.py` (`graph.json`/`graph.graphml`/`cypher.txt` + self-contained vanilla-JS `graph.html`; entity nodes + relation edges) · `chunking.py` + `embeddings.py` (OpenAI-compatible embedding client, keyword fallback) + `search.py` (keyword/vector/hybrid) · `sensitive.py` (secret/PII redaction) · `dedup.py` (Jaccard + hash dedup) · `auto_ingest.py` (background queue/worker) · `obsidian_adapter.py` · `service.py` (`KnowledgeService` orchestrator — the only thing the router touches). Tests: `tests/test_knowledge.py` (20 cases). Frontend: `frontend-web/src/core/knowledge/*` + `pages/KnowledgeBasePage.tsx` / `KnowledgeGraphPage.tsx` + chat `knowledge-save-trigger.tsx` / `knowledge-rag-toggle.tsx`; spec at `frontend-web/docs/product-knowledge-base-dev.md`. +**Module layout:** `models.py` (ORM: notes/sources/tags/note_tags + embeddings/versions/entities/relations) · `repository.py` (DB only, no LLM) · `schemas.py` (NoteDraft/SourceDraft DTOs) · `skill_templates.py` · `markdown_vault.py` · `manifest.py` · `extractor.py` (rule-based fallback) · `llm_extractor.py` (**default**: distills declarative knowledge per wiki-capture, structured JSON incl. entities/relations; falls back to `extractor.py`) · `sources.py` · `exporter.py` (`graph.json`/`graph.graphml`/`cypher.txt` + self-contained vanilla-JS `graph.html`; entity nodes + relation edges. **`graph.html` is theme-aware** — it reads `?theme=light|dark` from its own URL since the embedding iframe can't inherit the app's CSS; `KnowledgeGraphPage.tsx` appends the active day/night mode and re-skins on toggle. The layout cools down and **freezes** so dense graphs stay clickable, node radius scales with degree, and hovering/clicking a node focuses it + its direct neighbours while dimming the rest + a detail panel — taming heavy node/edge clutter) · `chunking.py` + `embeddings.py` (OpenAI-compatible embedding client, keyword fallback) + `search.py` (keyword/vector/hybrid) · `sensitive.py` (secret/PII redaction) · `dedup.py` (Jaccard + hash dedup) · `auto_ingest.py` (background queue/worker) · `obsidian_adapter.py` · `service.py` (`KnowledgeService` orchestrator — the only thing the router touches; capture entry points `capture_thread` / `capture_search` / `capture_file` / `create_manual_note`, all converging on `_persist_draft`). Tests: `tests/test_knowledge.py` (20 cases). Frontend: `frontend-web/src/core/knowledge/*` + `pages/KnowledgeBasePage.tsx` / `KnowledgeGraphPage.tsx` + chat `knowledge-save-trigger.tsx` / `knowledge-rag-toggle.tsx`; spec at `frontend-web/docs/product-knowledge-base-dev.md`. **Wikilinks & multi-note capture** (`knowledge.{related_notes_enabled,related_notes_limit,multi_note_enabled}`, all default-on; `project_name` defaults to `zncm`): - **Navigable `[[wikilinks]]`** — `service.resolve_wikilink(s)` maps a `[[target]]` to a knowledge note (by id → `vault_path` → title) or a vault meta page (`read_vault_page`, path-traversal-proof), else `missing`. `markdown_vault.normalize_wikilink_target` canonicalizes targets (drops `|alias`, slashes, `.md`). The frontend `MarkdownRenderer` renders non-numeric `[[...]]` as clickable links from the resolver map (numeric `[[123]]` stay citation badges); note links navigate, vault links open a `VaultPageDialog`, missing links are greyed. diff --git a/offline-backend-20260512/backend/app/gateway/routers/knowledge.py b/offline-backend-20260512/backend/app/gateway/routers/knowledge.py index 9512f0f9..7b8e4e29 100644 --- a/offline-backend-20260512/backend/app/gateway/routers/knowledge.py +++ b/offline-backend-20260512/backend/app/gateway/routers/knowledge.py @@ -13,9 +13,12 @@ from __future__ import annotations import logging import mimetypes +import re +import tempfile +from pathlib import Path from typing import Any -from fastapi import APIRouter, BackgroundTasks, HTTPException, Request +from fastapi import APIRouter, BackgroundTasks, File, Form, HTTPException, Request, UploadFile from fastapi.responses import FileResponse from pydantic import BaseModel, Field @@ -26,6 +29,12 @@ from deerflow.knowledge.service import EmptyThreadError from deerflow.runtime import serialize_channel_values from deerflow.runtime.references import build_reference_batches from deerflow.runtime.user_context import get_effective_user_id +from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown + +# Plain-text/markdown files are read directly; binary documents are converted. +_TEXT_EXTENSIONS = {".md", ".markdown", ".txt", ".text", ""} +# Bound the in-memory read so a knowledge import can't OOM the gateway. +_MAX_IMPORT_BYTES = 25 * 1024 * 1024 logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/knowledge", tags=["knowledge"]) @@ -299,6 +308,80 @@ async def ingest_threads_batch(body: BatchIngestRequest, request: Request) -> di return {"ok": True, "processed": len(items), "created": created, "skipped": skipped, "failed": failed, "llm_fallback": llm_fallback, "items": items} +async def _extract_upload_text(file: UploadFile) -> tuple[str, str]: + """Read an uploaded file and return ``(filename, markdown_text)``. + + Plain-text/markdown is decoded directly; PDF/Office documents are converted + to Markdown via the shared converter. Raises ``HTTPException`` on empty / + oversized / unsupported / unparseable files. + """ + name = file.filename or "document" + ext = Path(name).suffix.lower() + raw = await file.read() + if not raw: + raise HTTPException(status_code=400, detail="文件为空") + if len(raw) > _MAX_IMPORT_BYTES: + raise HTTPException(status_code=413, detail=f"文件过大，最大支持 {_MAX_IMPORT_BYTES // (1024 * 1024)}MB") + + if ext in _TEXT_EXTENSIONS: + text = raw.decode("utf-8", errors="ignore") + elif ext in CONVERTIBLE_EXTENSIONS: + # Convert in an isolated temp dir; a fixed safe name avoids path issues + # while preserving the extension so the converter detects the type. + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) / f"import{ext}" + tmp_path.write_bytes(raw) + md_path = await convert_file_to_markdown(tmp_path) + if md_path is None: + raise HTTPException(status_code=422, detail=f"无法解析文件：{name}") + text = md_path.read_text(encoding="utf-8", errors="ignore") + else: + raise HTTPException(status_code=415, detail=f"不支持的文件类型：{ext or '未知'}（支持 md/txt/pdf/word/ppt/excel）") + + if not text.strip(): + raise HTTPException(status_code=422, detail="文件内容为空或无法提取文本") + return name, text + + +@router.post("/ingest/file") +@require_auth +async def ingest_file( + request: Request, + file: UploadFile = File(...), + title: str | None = Form(default=None), + tags: str | None = Form(default=None), + status: str = Form(default="approved"), + model_name: str | None = Form(default=None), +) -> dict[str, Any]: + """Import an uploaded file into the knowledge base as wiki note(s). + + Converts the file to Markdown, distills it via the wiki-capture LLM pipeline + (same as thread capture), and writes the resulting declarative note(s) into + the shared vault. ``tags`` is a comma/space-separated string. + """ + service = get_knowledge_service(request) + name, text = await _extract_upload_text(file) + tag_list = [t for t in re.split(r"[,，\s]+", tags or "") if t] + try: + result = await service.capture_file( + content_md=text, + filename=name, + title=(title or None), + tags=tag_list, + status=status, + created_by=get_effective_user_id(), + model_name=(model_name or None), + ) + except EmptyThreadError: + raise HTTPException(status_code=422, detail="文件内容无可沉淀的知识") + return { + "note": result["note"], + "created": result["created"], + "notes": result.get("notes", [result["note"]]), + "llm_fallback": result.get("llm_fallback", False), + } + + @router.post("/ingest/search") @require_auth async def ingest_search(body: IngestSearchRequest, request: Request) -> dict[str, Any]: diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/exporter.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/exporter.py index 2f91a5ee..2153df8c 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/exporter.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/exporter.py @@ -135,7 +135,15 @@ def to_cypher(graph: dict[str, Any]) -> str: def to_html(graph: dict[str, Any]) -> str: - """Self-contained force-directed graph (canvas + vanilla JS, no CDN).""" + """Self-contained, theme-aware force-directed graph (canvas + vanilla JS, no CDN). + + The page reads ``?theme=light|dark`` from its own URL so the embedding iframe + can stay in sync with the app's day/night mode (an iframe cannot inherit the + parent's CSS). The layout settles and then **freezes** so a dense graph stops + drifting and stays clickable; node size scales with degree; and hovering or + clicking a node focuses it + its direct neighbours while dimming everything + else, which cuts through heavy node/edge clutter. + """ data_json = json.dumps(graph, ensure_ascii=False) return _HTML_TEMPLATE.replace("__GRAPH_DATA__", data_json) @@ -178,50 +186,246 @@ _HTML_TEMPLATE = """ Knowledge Graph -
-
+
+
+
知识笔记
+
实体
+
+
+
×

+
重置视图
""" diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py index 5c86f697..e9d924a2 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/llm_extractor.py @@ -84,6 +84,46 @@ references（外部来源汇总）、decision（架构/设计决策）、journal }""" +_DOC_SYSTEM_PROMPT = """\ +你是知识库整理专家，遵循 obsidian-wiki 的 wiki-capture 规范，把一篇**文档**提炼成\ +**陈述式知识（declarative knowledge）**，而不是照抄原文。 + +要求： +- 只保留高价值内容：核心结论、关键事实、定义、流程、数据、决策与其依据。 +- 丢弃：目录、页眉页脚、版权声明、重复的样板文字、无意义的排版噪声。 +- 用自己的话重组为结构化知识，不要逐段复制原文。 +- 推断内容用 ^[inferred] 标记，不确定/冲突用 ^[ambiguous] 标记。 +- 分类 category 取值之一：synthesis（分析/方案/结论）、concepts（概念/框架）、\ +references（外部来源汇总）、decision（架构/设计决策）、journal（纪要）。 +- 用中文输出。 + +一篇文档可能只讲一个主题，也可能覆盖多个相互独立的主题。请把它拆成 1~4 页知识： +- 默认只产出 **1 页**；只有当文档明确包含**多个彼此独立、可各自成文**的主题时才拆成多页。 +- 每一页聚焦单一主题，标题与 summary 必须彼此区分。 +- 文档没有可沉淀的知识时返回空的 notes 数组。 + +只输出一个 JSON 对象，不要任何额外文字或 Markdown 代码块，结构与对话提炼一致： +{ + "should_ingest": true, + "reason": "为什么值得/不值得沉淀", + "notes": [ + { + "title": "简洁的知识标题（不超过 40 字）", + "summary": "1-2 句话概括这页知识", + "category": "synthesis", + "context": "文档背景或适用场景，1-3 句", + "findings": ["结论/要点（陈述式，可含 ^[inferred]）", "..."], + "reasoning": "为什么成立、取舍、不确定性", + "implications": ["后续建议/风险/下一步", "..."], + "tags": ["标签1", "标签2"], + "entities": [{"name": "实体名", "type": "technology|concept|person|org|product|other"}], + "relations": [{"from": "实体A", "to": "实体B", "type": "uses|relates_to|part_of|depends_on"}], + "confidence": 0.0 + } + ] +}""" + + def _build_transcript(turns: list[dict[str, str]], sources: list[SourceDraft], *, max_chars: int) -> str: parts: list[str] = [] for i, t in enumerate(turns, 1): @@ -171,21 +211,27 @@ def _notes_from_data(data: dict[str, Any]) -> list[dict[str, Any]]: def _build_note_draft( note_data: dict[str, Any], *, - thread_id: str, - thread_title: str | None, - turns: list[dict[str, str]], + source_type: str, + source_id: str | None, + source_key: str, + fallback_title: str, title_override: str | None, tags: list[str] | None, project_name: str | None, sources: list[SourceDraft], - source_key_suffix: str = "", ) -> NoteDraft: - """Render one note dict into a :class:`NoteDraft`.""" + """Render one note dict into a :class:`NoteDraft`. + + Source-agnostic: the caller supplies ``source_type`` / ``source_id`` / + ``source_key`` and a ``fallback_title`` (used when neither the user override + nor the LLM produced a usable title), so both thread and document capture + share one renderer. + """ category = str(note_data.get("category") or "synthesis").strip().lower() if category not in _VALID_CATEGORIES: category = "synthesis" - final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or extractor._auto_title(thread_title, turns) + final_title = (title_override or "").strip() or str(note_data.get("title") or "").strip() or fallback_title summary = str(note_data.get("summary") or "").strip() related = extractor.related_links(project_name) body = _render_body(note_data, final_title, related, sources) @@ -210,15 +256,15 @@ def _build_note_draft( summary=summary or final_title, content_md=body, category=category, - source_type="thread", - source_id=thread_id, + source_type=source_type, + source_id=source_id, tags=base_tags, confidence=max(0.0, min(confidence, 1.0)), related=related, sources=sources, entities=[e for e in entities if isinstance(e, dict) and e.get("name")], relations=[r for r in relations if isinstance(r, dict) and r.get("from") and r.get("to")], - source_key=f"thread:{thread_id}{source_key_suffix}", + source_key=source_key, ) @@ -293,6 +339,7 @@ async def extract_thread_llm_multi( else: note_dicts = note_dicts[: max(1, max_notes)] + fallback_title = extractor._auto_title(thread_title, turns) drafts: list[NoteDraft] = [] for i, nd in enumerate(note_dicts): # The user-supplied title override only applies to a single-note result; @@ -302,14 +349,100 @@ async def extract_thread_llm_multi( drafts.append( _build_note_draft( nd, - thread_id=thread_id, - thread_title=thread_title, - turns=turns, + source_type="thread", + source_id=thread_id, + source_key=f"thread:{thread_id}{suffix}", + fallback_title=fallback_title, + title_override=override, + tags=tags, + project_name=project_name, + sources=sources, + ) + ) + return drafts + + +async def extract_document_llm_multi( + text: str, + *, + doc_id: str, + doc_title: str | None = None, + title: str | None = None, + tags: list[str] | None = None, + project_name: str | None = "zncm", + sources: list[SourceDraft] | None = None, + model_name: str | None = None, + max_input_chars: int = 12000, + allow_multi: bool = True, + max_notes: int = 4, +) -> list[NoteDraft]: + """Distill an imported **document** into one *or several* knowledge notes. + + Mirrors :func:`extract_thread_llm_multi` but takes raw document text (already + converted to Markdown/plain text) instead of a conversation. Returns ``[]`` + when the LLM path fails (callers fall back to a verbatim note); raises + :class:`LLMRejectedIngest` when the model deliberately judges the document + worthless. Resulting notes carry ``source_type="file"``. + """ + text = (text or "").strip() + if not text: + return [] + sources = sources or [] + + try: + from langchain_core.messages import HumanMessage, SystemMessage + + from deerflow.models import create_chat_model + + model = create_chat_model(name=model_name or None, thinking_enabled=False) + body = text if len(text) <= max_input_chars else text[:max_input_chars] + "\n…（已截断）" + header = f"文档标题：{doc_title}\n\n" if doc_title else "" + prompt = [ + SystemMessage(content=_DOC_SYSTEM_PROMPT + "\n\n" + WIKI_CAPTURE_GUIDANCE), + HumanMessage(content=f"以下是一篇文档，请按规范提炼为知识 JSON：\n\n{header}{body}"), + ] + response = await model.ainvoke(prompt) + raw = response.content if isinstance(response.content, str) else str(response.content) + data = _parse_json(raw) + except Exception: + logger.exception("[knowledge] document %s — LLM extraction call failed; using rule-based fallback", doc_id) + return [] + + if not data: + logger.warning("[knowledge] document %s — LLM returned no parsable JSON; using rule-based fallback", doc_id) + return [] + + note_dicts = _notes_from_data(data) + if not note_dicts: + reason = str(data.get("reason") or "")[:200] + if _is_falsey(data.get("should_ingest")): + logger.warning("[knowledge] document %s SKIPPED — model judged not worth ingesting: %s", doc_id, reason) + raise LLMRejectedIngest(reason or "no knowledge value") + logger.warning( + "[knowledge] document %s — LLM produced no notes (should_ingest=%s, reason=%r); using rule-based fallback", + doc_id, + data.get("should_ingest"), + reason, + ) + return [] + note_dicts = note_dicts[:1] if not allow_multi else note_dicts[: max(1, max_notes)] + + fallback_title = (doc_title or title or "文档知识").strip() or "文档知识" + drafts: list[NoteDraft] = [] + for i, nd in enumerate(note_dicts): + override = title if (len(note_dicts) == 1) else None + suffix = "" if len(note_dicts) == 1 else f"#{i}" + drafts.append( + _build_note_draft( + nd, + source_type="file", + source_id=doc_id, + source_key=f"file:{doc_id}{suffix}", + fallback_title=fallback_title, title_override=override, tags=tags, project_name=project_name, sources=sources, - source_key_suffix=suffix, ) ) return drafts diff --git a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py index d1fbe3f9..71d5f245 100644 --- a/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py +++ b/offline-backend-20260512/backend/packages/harness/deerflow/knowledge/service.py @@ -394,6 +394,129 @@ class KnowledgeService: ) return await self._persist_draft(draft, created_by=created_by) + def _verbatim_file_draft(self, body: str, filename: str, *, title: str | None, tags: list[str] | None, doc_id: str) -> NoteDraft: + """Build a verbatim fallback note from a file when LLM distillation fails. + + Lands as a ``draft`` (like the thread fallback) since it is the raw + document, not distilled knowledge. + """ + base_tags = list(tags or []) + for default in (self.project_name, "knowledge-base"): + if default and default not in base_tags: + base_tags.append(default) + name = (title or Path(filename).stem or "文档知识").strip() or "文档知识" + summary = next((ln.strip() for ln in body.splitlines() if ln.strip()), name)[:280] + return NoteDraft( + title=name, + summary=summary, + content_md=body, + category="references", + source_type="file", + source_id=doc_id, + status="draft", + tags=base_tags, + confidence=0.4, + related=extractor.related_links(self.project_name), + source_key=f"file:{doc_id}", + ) + + async def capture_file( + self, + *, + content_md: str, + filename: str, + title: str | None = None, + tags: list[str] | None = None, + status: str = "approved", + created_by: str | None = None, + model_name: str | None = None, + skip_existing: bool = False, + ) -> dict[str, Any]: + """Import a file (already converted to Markdown/text) into knowledge notes. + + Runs the same wiki-capture LLM distillation as thread capture, so an + uploaded document becomes one or more declarative wiki notes in the vault. + Falls back to a verbatim ``draft`` note when the LLM is disabled or returns + nothing usable. Returns ``{"note", "created", "notes", "llm_fallback"}``; + raises :class:`EmptyThreadError` when there is nothing to keep. + """ + import hashlib + + text = (content_md or "").strip() + if not text: + raise EmptyThreadError(f"File {filename} is empty") + # Strip any frontmatter the document carries; the vault writer adds its own. + _, parsed = MarkdownVault.parse(text) + body = (parsed or text).strip() + if not body: + raise EmptyThreadError(f"File {filename} has no body content") + # Stable id from name + size so re-importing the same file can be skipped. + doc_id = hashlib.sha1(f"{filename}:{len(body)}".encode("utf-8")).hexdigest()[:16] + + logger.info( + "[knowledge] importing file '%s' (chars=%d, doc_id=%s, llm_extract=%s, skip_existing=%s)", + filename, + len(body), + doc_id, + self.config.llm_extract_enabled, + skip_existing, + ) + + if skip_existing: + existing = await self.repo.find_by_source("file", doc_id) + if existing: + logger.info("[knowledge] file '%s' already imported (note %s); skipping", filename, existing.get("id")) + return {"note": existing, "created": False, "notes": [existing], "llm_fallback": False} + + drafts: list[NoteDraft] = [] + llm_fallback = False + if self.config.llm_extract_enabled: + try: + drafts = await llm_extractor.extract_document_llm_multi( + body, + doc_id=doc_id, + doc_title=title or Path(filename).stem, + title=title, + tags=tags, + project_name=self.project_name, + model_name=model_name or self.config.extract_model_name, + max_input_chars=self.config.extract_max_input_chars, + allow_multi=self.config.multi_note_enabled, + ) + except llm_extractor.LLMRejectedIngest as exc: + raise EmptyThreadError(f"File {filename} judged not worth ingesting: {exc}") from exc + if not drafts: + llm_fallback = True + if not drafts: + drafts = [self._verbatim_file_draft(body, filename, title=title, tags=tags, doc_id=doc_id)] + + effective_status = status + if llm_fallback and normalize_status(status) == "approved": + effective_status = "draft" + normalized = normalize_status(effective_status) + for draft in drafts: + draft.status = normalized + + # Cross-link sibling notes split from the same document (resolved by title + # once all are persisted), mirroring multi-note thread capture. + if len(drafts) > 1: + for i, draft in enumerate(drafts): + siblings = [f"[[{o.title}]]" for j, o in enumerate(drafts) if j != i and o.title] + if siblings: + draft.content_md = _inject_related_notes(draft.content_md, siblings) + draft.related = list(dict.fromkeys([*siblings, *draft.related])) + + notes = [await self._persist_draft(draft, created_by=created_by) for draft in drafts] + logger.info( + "[knowledge] file '%s' imported → %d note(s) [%s], status=%s, llm_fallback=%s", + filename, + len(notes), + ", ".join(n.get("title", "") for n in notes), + normalized, + llm_fallback, + ) + return {"note": notes[0], "created": True, "notes": notes, "llm_fallback": llm_fallback} + # -- CRUD ------------------------------------------------------------- async def list_notes( self,