feat(knowledge): support standalone file ingestion and configurable wiki entry

Use neutral knowledge-service labels and sanitize user-facing task errors.
This commit is contained in:
吉鑫升 committed 2026-09-07 17:07:27 +08:00
1 parent f9b6d7e322
commit 3ca5528640
24 files changed
+549 -54

No files matched your search

+8 -3
View File
@@ -1762,9 +1762,12 @@ gone.
Assistant knowledge has one global full-library base (`知识梳理总库`) and any
number of admin-created custom bases. Ordinary users have read/search access and
administrators own all mutations. Assistant bases only accept WeKnora export
packages or imports triggered from a WeKnora mapping. Do not add direct skill
imports or arbitrary local-file imports to assistant knowledge. Importing into a
administrators own all mutations. Assistant bases accept local documents independently
of WeKnora via `/api/assistant-knowledge/imports/files`, as well as WeKnora export
packages or imports triggered from a WeKnora mapping. Local documents use the
Gateway `assistant_file_ingest` pipeline: persisted original files, chunked LLM Wiki
extraction, cached body embeddings, and import-job status/retry. Do not add direct skill
imports. Importing into a
custom assistant base must also mirror the package into the global base for
cross-source entity alignment. Admins may rename any assistant base and delete
custom bases. Deletion removes the custom base's Wiki pages, vectors, entities,
@@ -1773,6 +1776,8 @@ deletable because it owns full-library aggregation.
### Wiki package and local encoder invariants (2026-09)
Ordinary knowledge detail landing is globally configured by `ordinary_knowledge_landing` (`wiki` default / `weknora`), with admin-only PUT `/api/system-settings/ordinary-knowledge-landing`. Explicit document/graph/article links bypass this default. Wiki reading and the embedded WeKnora view provide reciprocal navigation; `?view=weknora` explicitly selects the iframe and must not loop back to the default Wiki route.
- `knowledge_transfer.sync_to_global` is called after successful explicit local
Wiki vectorization; it skips conversation/system mappings. Do not enable a new
watcher or automatic remote-write scheduler as a side effect.
@@ -1,5 +1,11 @@
# DeerFlow Backend
普通知识库默认详情页面:管理员可在知识库顶部选择“本系统 Wiki 索引”或“嵌入 WeKnora”,对应 `ordinary_knowledge_landing`(默认 `wiki`)。Wiki 库首次进入默认展示本系统索引,右上角“打开 WeKnora”切换到内嵌管理页;内嵌页可返回索引。非 Wiki 库仍进入 WeKnora 文档页。设置通过 `/api/system-settings/ordinary-knowledge-landing` 保存,仅管理员可修改;指定文章、文档、图谱的直达链接优先于默认设置。
助手知识库可独立于 WeKnora 使用:管理员进入助手知识库,选择或创建库后点击“上传文件生成 Wiki”。支持 Markdown/TXT/CSV、PDF、Word、Excel、PowerPoint(单文件 100 MB,解析正文上限 400 万字符)。文件先写入运行目录 `assistant-knowledge/files/{job_id}`,后台分段调用已配置的语言模型生成分类 Wiki、实体关系,再通过 DeerFlow 编码器对正文分段向量化,保存到助手库并汇入知识梳理总库。无需配置 WeKnora URL;配置后仍可直接上传,也保留 WeKnora 数据包导入。文件处理沿用管理员写入权限,所有用户可以浏览和搜索。
导入记录显示解析、Wiki 生成和向量化阶段;失败或服务中断后可“重试 / 恢复处理”,复用已落盘的 Wiki 和相同模型指纹的向量。离线编码器须预先放置模型并在管理员设置中配置;模型或编码器不可用会明确失败,不把摘要冒充 Wiki 或虚构向量。扫描件是否可解析取决于部署的文档解析/OCR 能力。
DeerFlow is a LangGraph-based AI super agent with sandbox execution, persistent memory, and extensible tool integration. The backend enables AI agents to execute code, browse the web, manage files, delegate tasks to subagents, and retain context across conversations - all in isolated, per-thread environments.
---
@@ -0,0 +1,191 @@
"""Local document → Wiki → vectors, independent of the WeKnora service."""
from __future__ import annotations
import asyncio
import hashlib
import json
from pathlib import Path
from typing import Any
from deerflow.config.runtime_paths import runtime_home
from deerflow.knowledge.llm_extractor import extract_document_llm_multi
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown
ALLOWED_EXTENSIONS = CONVERTIBLE_EXTENSIONS | {".md", ".markdown", ".txt", ".csv"}
MAX_FILE_BYTES = 100 * 1024 * 1024
MAX_TEXT_CHARS = 4_000_000
_tasks: dict[str, asyncio.Task] = {}
_slots = asyncio.Semaphore(2)
_queue_lock = asyncio.Lock()
def file_job_dir(job_id: str) -> Path:
from uuid import UUID
return runtime_home() / "assistant-knowledge" / "files" / str(UUID(job_id))
def save_json(path: Path, value: Any) -> None:
temporary = path.with_suffix(".tmp")
temporary.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
temporary.replace(path)
def read_json(path: Path) -> Any:
return json.loads(path.read_text(encoding="utf-8"))
def split_text(text: str, size: int):
"""Bound every extraction/embedding input without truncating the document."""
for offset in range(0, len(text), size):
yield text[offset : offset + size]
def notes_package(notes: list[dict], source: dict) -> dict:
from deerflow.persistence.assistant_knowledge.store import _entity_alignment_key
merged: dict[str, dict] = {}
for raw in notes:
key = _entity_alignment_key(raw["title"])
if key not in merged:
merged[key] = dict(raw)
continue
note = merged[key]
note["content_md"] = "\n\n".join(dict.fromkeys([note["content_md"], raw["content_md"]]))
for field in ("tags", "entities", "relations"):
note[field] = [*note.get(field, []), *raw.get(field, [])]
note["tags"] = list(dict.fromkeys(note["tags"]))
pages, entities, relations = [], [], []
for index, note in enumerate(merged.values()):
slug = f"files/{source['digest']}/{index}"
pages.append(
{
"slug": slug,
"title": note["title"],
"content": note["content_md"],
"page_type": "article",
"tags": note.get("tags", []),
"category_path": [note.get("category") or "知识文档"],
"source_refs": [{"type": "file", "title": source["filename"]}],
}
)
for entity in note.get("entities", []):
entities.append({"id": entity["name"], "name": entity["name"], "entity_type": entity.get("type", "concept"), "aliases": entity.get("aliases", []), "source_wiki_slug": slug})
for relation in note.get("relations", []):
relations.append({"source": relation["from"], "target": relation["to"], "predicate": relation.get("type", "related_to"), "source_wiki_slug": slug})
# Persist real sibling links, understood by the shared Wiki reader.
for index, page in enumerate(pages):
others = [p for p in pages[max(0, index - 4) : index + 5] if p["slug"] != page["slug"]]
page["out_links"] = [p["slug"] for p in others]
page["in_links"] = list(page["out_links"])
if others:
page["content"] += "\n\n## 相关知识\n" + "\n".join(f"- [[{p['slug']}|{p['title']}]]" for p in others)
return {"source": source, "wiki_pages": pages, "entities": entities, "relations": relations, "vectors": []}
async def run_file_job(app, job_id: str) -> None:
store = app.state.assistant_knowledge_store
root = file_job_dir(job_id)
try:
async with _slots:
manifest = await asyncio.to_thread(read_json, root / "manifest.json")
await store.update_import_job(job_id, status="running", phase="parsing")
package_file = root / "wiki.json"
if package_file.exists():
package = await asyncio.to_thread(read_json, package_file)
else:
text_file = root / "parsed.md"
if not text_file.exists():
original = root / manifest["stored_name"]
if original.suffix in CONVERTIBLE_EXTENSIONS:
converted = await asyncio.to_thread(lambda: asyncio.run(convert_file_to_markdown(original)))
if converted is None:
raise ValueError("无法解析文档,请检查文件是否损坏或是否需要 OCR")
text = await asyncio.to_thread(converted.read_text, encoding="utf-8")
else:
raw = await asyncio.to_thread(original.read_bytes)
try:
text = raw.decode("utf-8-sig")
except UnicodeDecodeError:
text = raw.decode("gb18030")
if not text.strip() or len(text) > MAX_TEXT_CHARS:
raise ValueError("文件没有可提取文本,或解析正文超过 400 万字符,请拆分文件")
await asyncio.to_thread(text_file.write_text, text, encoding="utf-8")
else:
text = await asyncio.to_thread(text_file.read_text, encoding="utf-8")
notes = []
parts = list(split_text(text, 10000))
for index, part in enumerate(parts):
await store.update_import_job(job_id, phase="generating_wiki", counts={"parts_total": len(parts), "parts_done": index})
cache = root / f"notes-{index}.json"
if cache.exists():
drafts = await asyncio.to_thread(read_json, cache)
else:
from dataclasses import asdict
extracted = await asyncio.wait_for(
extract_document_llm_multi(
part,
doc_id=f"{job_id}:{index}",
doc_title=manifest["filename"],
max_input_chars=12000,
max_notes=6,
),
timeout=300,
)
if not extracted:
raise ValueError("Wiki 生成失败,请在设置中检查可用的语言模型,然后重试")
drafts = [asdict(note) for note in extracted]
await asyncio.to_thread(save_json, cache, drafts)
notes.extend(drafts)
package = notes_package(notes, {"type": "file", "filename": manifest["filename"], "digest": manifest["digest"]})
await asyncio.to_thread(save_json, package_file, package)
embedding = getattr(app.state, "llmwiki_embedding", None)
if embedding is None:
raise ValueError("Wiki 已生成;请管理员在设置中配置本地编码器或编码服务,然后重试向量化")
vectors = []
sections = [(p, i, section) for p in package["wiki_pages"] for i, section in enumerate(split_text(p["content"], 1600))]
for index, (page, section_index, content) in enumerate(sections):
await store.update_import_job(job_id, phase="vectorizing", counts={"wiki_pages": len(package["wiki_pages"]), "vectors_total": len(sections), "vectors": index})
digest = hashlib.sha256(content.encode()).hexdigest()
cache = root / f"vector-{digest}.json"
record = await asyncio.to_thread(read_json, cache) if cache.exists() else None
if record is None or record["embedding_fingerprint"] != embedding.fingerprint:
values = await embedding.embed_texts([content])
record = {"vector": values[0], "embedding_model": embedding.config.model, "embedding_fingerprint": embedding.fingerprint, "content_hash": digest}
await asyncio.to_thread(save_json, cache, record)
vectors.append({**record, "wiki_slug": page["slug"], "section_index": section_index, "section_content": content})
package["vectors"] = vectors
await store.update_import_job(job_id, phase="importing")
await store.import_package_for_job(job_id, package=package, created_by=manifest["actor"])
except asyncio.CancelledError:
await store.update_import_job(job_id, status="failed", phase="interrupted", error="服务停止,已保留处理进度,可重试", completed=True)
raise
except Exception as exc:
await store.update_import_job(job_id, status="failed", phase="failed", error=str(exc) or type(exc).__name__, completed=True)
def start_file_job(app, job_id: str) -> bool:
if job_id in _tasks:
return False
task = asyncio.create_task(run_file_job(app, job_id))
_tasks[job_id] = task
task.add_done_callback(lambda _: _tasks.pop(job_id, None))
return True
async def stop_file_jobs() -> None:
tasks = list(_tasks.values())
for task in tasks:
task.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
async def retry_file_job(app, job_id: str) -> bool:
async with _queue_lock:
if job_id in _tasks:
return False
await app.state.assistant_knowledge_store.update_import_job(job_id, status="queued", phase="queued")
return start_file_job(app, job_id)
@@ -701,6 +701,9 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
try:
yield
finally:
from app.gateway.assistant_file_ingest import stop_file_jobs
await stop_file_jobs()
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
register_local_wiki_index_runtime(store=None, search=None, sync=None, retrieval=None)
@@ -152,6 +152,18 @@ def _list_export_manifests(mapping_id: str, *, limit: int) -> list[dict[str, Any
def _delete_base_package_files(base_id: str, import_job_ids: list[str]) -> None:
"""Delete package artifacts owned by one assistant base only."""
import shutil
from app.gateway.assistant_file_ingest import file_job_dir
for job_id in import_job_ids:
# UUID validation confines cleanup to exactly one job directory.
try:
directory = file_job_dir(job_id)
except ValueError:
continue
if directory.is_dir() and not directory.is_symlink():
shutil.rmtree(directory)
root = _package_root()
if not root.is_dir():
return
@@ -448,10 +460,71 @@ async def import_skills(request: Request, _body: SkillImportRequest) -> dict[str
raise HTTPException(status_code=410, detail="不支持技能直接归纳到助手知识库,请先归纳到 WeKnora 普通知识库后再导入助手知识库")
@router.post("/imports/files", status_code=410)
async def import_files(request: Request) -> dict[str, Any]:
@router.post("/imports/files", status_code=202)
async def import_files(
request: Request,
file: UploadFile = File(...),
assistant_base_id: str | None = Form(default=None),
) -> dict[str, Any]:
import hashlib
from app.gateway.assistant_file_ingest import ALLOWED_EXTENSIONS, MAX_FILE_BYTES, file_job_dir, save_json, start_file_job
actor = await _actor(request, admin=True)
filename = (file.filename or "document").replace("\\", "/").rsplit("/", 1)[-1][:240]
extension = Path(filename).suffix.lower()
if extension not in ALLOWED_EXTENSIONS:
await file.close()
raise HTTPException(status_code=415, detail="支持 Markdown、TXT、CSV、PDF、Word、Excel、PowerPoint 文件")
job = await _store(request).create_queued_import_job(
source_type="file", source_key=f"file-upload:{uuid4()}", source_name=filename,
trigger="local_file_upload", created_by=actor, base_id=assistant_base_id,
metadata={"type": "file", "filename": filename}, phase="uploading",
)
root = file_job_dir(job["id"])
await asyncio.to_thread(root.mkdir, parents=True, exist_ok=True)
path = root / f"original{extension}"
size, digest = 0, hashlib.sha256()
try:
with path.open("wb") as handle:
while chunk := await file.read(1024 * 1024):
size += len(chunk)
if size > MAX_FILE_BYTES:
raise HTTPException(status_code=413, detail="单个文件最大 100 MB,请拆分后上传")
digest.update(chunk)
await asyncio.to_thread(handle.write, chunk)
if not size:
raise HTTPException(status_code=422, detail="上传文件为空")
await asyncio.to_thread(save_json, root / "manifest.json", {
"filename": filename, "stored_name": path.name, "digest": digest.hexdigest(), "actor": actor,
})
job = await _store(request).update_import_job(job["id"], phase="queued", counts={"uploaded_bytes": size})
start_file_job(request.app, job["id"])
return job
except Exception as exc:
await _store(request).update_import_job(job["id"], status="failed", phase="upload_failed", error=str(getattr(exc, "detail", exc)), completed=True)
await asyncio.to_thread(path.unlink, missing_ok=True)
raise
finally:
await file.close()
@router.post("/import-jobs/{job_id}/retry-file", status_code=202)
async def retry_file_import(request: Request, job_id: str) -> dict[str, Any]:
from app.gateway.assistant_file_ingest import file_job_dir, retry_file_job
await _actor(request, admin=True)
raise HTTPException(status_code=410, detail="助手知识库不支持导入任意本地文件,请从 WeKnora 普通知识库导出后导入")
store = _store(request)
job = await store.get_job(job_id)
if not job or job.get("trigger") != "local_file_upload":
raise HTTPException(status_code=404, detail="文件处理任务不存在")
if job["status"] == "completed":
raise HTTPException(status_code=409, detail="文件已处理完成")
if not (file_job_dir(job_id) / "manifest.json").exists():
raise HTTPException(status_code=409, detail="文件上传未完成,请重新上传")
if not await retry_file_job(request.app, job_id):
raise HTTPException(status_code=409, detail="文件仍在处理中")
return await store.get_job(job_id)
@router.post("/imports/weknora-package", status_code=202)
@@ -56,6 +56,22 @@ class KnowledgeBaseProductVersionUpdateRequest(BaseModel):
version: Literal["v1", "v2"]
class OrdinaryKnowledgeLanding(BaseModel):
mode: Literal["wiki", "weknora"] = "wiki"
@router.get("/ordinary-knowledge-landing", response_model=OrdinaryKnowledgeLanding)
async def get_ordinary_knowledge_landing() -> OrdinaryKnowledgeLanding:
return OrdinaryKnowledgeLanding(mode=load_system_settings().ordinary_knowledge_landing)
@router.put("/ordinary-knowledge-landing", response_model=OrdinaryKnowledgeLanding)
async def update_ordinary_knowledge_landing(body: OrdinaryKnowledgeLanding, request: Request) -> OrdinaryKnowledgeLanding:
await _require_admin(request)
saved = save_system_settings(load_system_settings().model_copy(update={"ordinary_knowledge_landing": body.mode}))
return OrdinaryKnowledgeLanding(mode=saved.ordinary_knowledge_landing)
class PromptPrefixUpdateRequest(BaseModel):
enabled: bool | None = None
switch_label: str | None = None
@@ -538,6 +538,7 @@ class SystemSettings(BaseModel):
default="v1",
description="v1 opens the ordinary WeKnora experience; v2 opens the DeerFlow assistant knowledge base.",
)
ordinary_knowledge_landing: Literal["wiki", "weknora"] = "wiki"
model_config = ConfigDict(extra="ignore")
@@ -241,6 +241,8 @@ class AssistantKnowledgeRepository:
) -> tuple[str, str]:
metadata = package.get("source") or {}
provider = str(metadata.get("provider") or metadata.get("type") or "").strip().casefold()
if provider == "file" and metadata.get("digest"):
return f"file:{metadata['digest']}", str(metadata.get("filename") or fallback_name)
if provider == "weknora":
source_key = str(
metadata.get("knowledge_base_id")
@@ -0,0 +1,100 @@
import asyncio
from types import SimpleNamespace
import httpx
import pytest
from fastapi import FastAPI
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from app.gateway import assistant_file_ingest as ingest
from app.gateway.routers import assistant_knowledge as router
from deerflow.knowledge.schemas import NoteDraft
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
from deerflow.persistence.base import Base
def test_same_topic_across_document_parts_preserves_all_knowledge():
package = ingest.notes_package([
{"title": "设备维护", "content_md": "每周清理过滤网", "tags": ["运维"]},
{"title": "设备 维护", "content_md": "每月检查电机温度", "tags": ["设备"]},
], {"digest": "document", "filename": "维护指南.md"})
assert len(package["wiki_pages"]) == 1
assert "每周清理过滤网" in package["wiki_pages"][0]["content"]
assert "每月检查电机温度" in package["wiki_pages"][0]["content"]
assert package["wiki_pages"][0]["tags"] == ["运维", "设备"]
@pytest.mark.asyncio
async def test_standalone_upload_vectors_retry_and_dedup(tmp_path, monkeypatch):
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
async with engine.begin() as connection:
await connection.run_sync(Base.metadata.create_all)
store = AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
base = await store.create_base(name="独立文档库", created_by="admin")
app = FastAPI()
app.include_router(router.router)
app.state.assistant_knowledge_store = store
app.state.llmwiki_embedding = None
monkeypatch.setattr(ingest, "runtime_home", lambda: tmp_path)
extraction_calls = []
async def actor(*args, **kwargs):
return "admin"
async def extract(text, **kwargs):
extraction_calls.append(text)
return [NoteDraft(title="冷却系统", summary="摘要占位", content_md="# 冷却系统\n风扇的额定转速为每分钟 1200 转。", tags=["设备"], entities=[{"name": "风扇", "type": "product"}])]
class Encoder:
fingerprint = "local-test-3"
dimensions = 3
config = SimpleNamespace(model="offline-test")
async def embed_texts(self, texts):
return [[1.0, 0.0, 0.0] for _ in texts]
async def embed_query(self, text):
return [1.0, 0.0, 0.0]
monkeypatch.setattr(router, "_actor", actor)
monkeypatch.setattr(ingest, "extract_document_llm_multi", extract)
async def finish():
if ingest._tasks:
await asyncio.gather(*list(ingest._tasks.values()))
try:
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
payload = {"assistant_base_id": base["id"]}
response = await client.post("/api/assistant-knowledge/imports/files", data=payload, files={"file": ("../../设备.md", "风扇额定转速为 1200 转。".encode())})
assert response.status_code == 202, response.text
job_id = response.json()["id"]
await finish()
assert (await store.get_job(job_id))["status"] == "failed"
assert (ingest.file_job_dir(job_id) / "wiki.json").exists()
app.state.llmwiki_embedding = Encoder()
retry = await client.post(f"/api/assistant-knowledge/import-jobs/{job_id}/retry-file")
assert retry.status_code == 202
await finish()
job = await store.get_job(job_id)
assert job["status"] == "completed", job
assert job["counts"]["vectors"] == 1
assert len(extraction_calls) == 1
results = await store.search_pages(base_ids=[base["id"]], query="额定转速", embedding_client=app.state.llmwiki_embedding)
assert results and results[0]["retrieval_mode"] == "vector"
assert "1200" in results[0]["matched_section"]["content"]
assert "摘要占位" not in results[0]["matched_section"]["content"]
global_base = await store.get_base()
assert len(await store.list_pages(base_id=global_base["id"])) == 1
again = await client.post("/api/assistant-knowledge/imports/files", data=payload, files={"file": ("设备.md", "风扇额定转速为 1200 转。".encode())})
assert again.status_code == 202
await finish()
assert len(await store.list_sources(base_id=base["id"])) == 1
assert len(await store.list_pages(base_id=global_base["id"])) == 1
bad = await client.post("/api/assistant-knowledge/imports/files", data=payload, files={"file": ("script.py", b"print('no')")})
assert bad.status_code == 415
empty = await client.post("/api/assistant-knowledge/imports/files", data=payload, files={"file": ("empty.txt", b"")})
assert empty.status_code == 422
finally:
await ingest.stop_file_jobs()
await engine.dispose()