feat(knowledge): support standalone file ingestion and configurable wiki entry

Use neutral knowledge-service labels and sanitize user-facing task errors.

offline-backend-20260512/backend/app/gateway/assistant_file_ingest.py

@@ -0,0 +1,191 @@
+"""Local document → Wiki → vectors, independent of the WeKnora service."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+from pathlib import Path
+from typing import Any
+
+from deerflow.config.runtime_paths import runtime_home
+from deerflow.knowledge.llm_extractor import extract_document_llm_multi
+from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown
+
+ALLOWED_EXTENSIONS = CONVERTIBLE_EXTENSIONS | {".md", ".markdown", ".txt", ".csv"}
+MAX_FILE_BYTES = 100 * 1024 * 1024
+MAX_TEXT_CHARS = 4_000_000
+_tasks: dict[str, asyncio.Task] = {}
+_slots = asyncio.Semaphore(2)
+_queue_lock = asyncio.Lock()
+
+
+def file_job_dir(job_id: str) -> Path:
+    from uuid import UUID
+
+    return runtime_home() / "assistant-knowledge" / "files" / str(UUID(job_id))
+
+
+def save_json(path: Path, value: Any) -> None:
+    temporary = path.with_suffix(".tmp")
+    temporary.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
+    temporary.replace(path)
+
+
+def read_json(path: Path) -> Any:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def split_text(text: str, size: int):
+    """Bound every extraction/embedding input without truncating the document."""
+    for offset in range(0, len(text), size):
+        yield text[offset : offset + size]
+
+
+def notes_package(notes: list[dict], source: dict) -> dict:
+    from deerflow.persistence.assistant_knowledge.store import _entity_alignment_key
+
+    merged: dict[str, dict] = {}
+    for raw in notes:
+        key = _entity_alignment_key(raw["title"])
+        if key not in merged:
+            merged[key] = dict(raw)
+            continue
+        note = merged[key]
+        note["content_md"] = "\n\n".join(dict.fromkeys([note["content_md"], raw["content_md"]]))
+        for field in ("tags", "entities", "relations"):
+            note[field] = [*note.get(field, []), *raw.get(field, [])]
+        note["tags"] = list(dict.fromkeys(note["tags"]))
+    pages, entities, relations = [], [], []
+    for index, note in enumerate(merged.values()):
+        slug = f"files/{source['digest']}/{index}"
+        pages.append(
+            {
+                "slug": slug,
+                "title": note["title"],
+                "content": note["content_md"],
+                "page_type": "article",
+                "tags": note.get("tags", []),
+                "category_path": [note.get("category") or "知识文档"],
+                "source_refs": [{"type": "file", "title": source["filename"]}],
+            }
+        )
+        for entity in note.get("entities", []):
+            entities.append({"id": entity["name"], "name": entity["name"], "entity_type": entity.get("type", "concept"), "aliases": entity.get("aliases", []), "source_wiki_slug": slug})
+        for relation in note.get("relations", []):
+            relations.append({"source": relation["from"], "target": relation["to"], "predicate": relation.get("type", "related_to"), "source_wiki_slug": slug})
+    # Persist real sibling links, understood by the shared Wiki reader.
+    for index, page in enumerate(pages):
+        others = [p for p in pages[max(0, index - 4) : index + 5] if p["slug"] != page["slug"]]
+        page["out_links"] = [p["slug"] for p in others]
+        page["in_links"] = list(page["out_links"])
+        if others:
+            page["content"] += "\n\n## 相关知识\n" + "\n".join(f"- [[{p['slug']}|{p['title']}]]" for p in others)
+    return {"source": source, "wiki_pages": pages, "entities": entities, "relations": relations, "vectors": []}
+
+
+async def run_file_job(app, job_id: str) -> None:
+    store = app.state.assistant_knowledge_store
+    root = file_job_dir(job_id)
+    try:
+        async with _slots:
+            manifest = await asyncio.to_thread(read_json, root / "manifest.json")
+            await store.update_import_job(job_id, status="running", phase="parsing")
+            package_file = root / "wiki.json"
+            if package_file.exists():
+                package = await asyncio.to_thread(read_json, package_file)
+            else:
+                text_file = root / "parsed.md"
+                if not text_file.exists():
+                    original = root / manifest["stored_name"]
+                    if original.suffix in CONVERTIBLE_EXTENSIONS:
+                        converted = await asyncio.to_thread(lambda: asyncio.run(convert_file_to_markdown(original)))
+                        if converted is None:
+                            raise ValueError("无法解析文档，请检查文件是否损坏或是否需要 OCR")
+                        text = await asyncio.to_thread(converted.read_text, encoding="utf-8")
+                    else:
+                        raw = await asyncio.to_thread(original.read_bytes)
+                        try:
+                            text = raw.decode("utf-8-sig")
+                        except UnicodeDecodeError:
+                            text = raw.decode("gb18030")
+                    if not text.strip() or len(text) > MAX_TEXT_CHARS:
+                        raise ValueError("文件没有可提取文本，或解析正文超过 400 万字符，请拆分文件")
+                    await asyncio.to_thread(text_file.write_text, text, encoding="utf-8")
+                else:
+                    text = await asyncio.to_thread(text_file.read_text, encoding="utf-8")
+                notes = []
+                parts = list(split_text(text, 10000))
+                for index, part in enumerate(parts):
+                    await store.update_import_job(job_id, phase="generating_wiki", counts={"parts_total": len(parts), "parts_done": index})
+                    cache = root / f"notes-{index}.json"
+                    if cache.exists():
+                        drafts = await asyncio.to_thread(read_json, cache)
+                    else:
+                        from dataclasses import asdict
+
+                        extracted = await asyncio.wait_for(
+                            extract_document_llm_multi(
+                                part,
+                                doc_id=f"{job_id}:{index}",
+                                doc_title=manifest["filename"],
+                                max_input_chars=12000,
+                                max_notes=6,
+                            ),
+                            timeout=300,
+                        )
+                        if not extracted:
+                            raise ValueError("Wiki 生成失败，请在设置中检查可用的语言模型，然后重试")
+                        drafts = [asdict(note) for note in extracted]
+                        await asyncio.to_thread(save_json, cache, drafts)
+                    notes.extend(drafts)
+                package = notes_package(notes, {"type": "file", "filename": manifest["filename"], "digest": manifest["digest"]})
+                await asyncio.to_thread(save_json, package_file, package)
+            embedding = getattr(app.state, "llmwiki_embedding", None)
+            if embedding is None:
+                raise ValueError("Wiki 已生成；请管理员在设置中配置本地编码器或编码服务，然后重试向量化")
+            vectors = []
+            sections = [(p, i, section) for p in package["wiki_pages"] for i, section in enumerate(split_text(p["content"], 1600))]
+            for index, (page, section_index, content) in enumerate(sections):
+                await store.update_import_job(job_id, phase="vectorizing", counts={"wiki_pages": len(package["wiki_pages"]), "vectors_total": len(sections), "vectors": index})
+                digest = hashlib.sha256(content.encode()).hexdigest()
+                cache = root / f"vector-{digest}.json"
+                record = await asyncio.to_thread(read_json, cache) if cache.exists() else None
+                if record is None or record["embedding_fingerprint"] != embedding.fingerprint:
+                    values = await embedding.embed_texts([content])
+                    record = {"vector": values[0], "embedding_model": embedding.config.model, "embedding_fingerprint": embedding.fingerprint, "content_hash": digest}
+                    await asyncio.to_thread(save_json, cache, record)
+                vectors.append({**record, "wiki_slug": page["slug"], "section_index": section_index, "section_content": content})
+            package["vectors"] = vectors
+            await store.update_import_job(job_id, phase="importing")
+            await store.import_package_for_job(job_id, package=package, created_by=manifest["actor"])
+    except asyncio.CancelledError:
+        await store.update_import_job(job_id, status="failed", phase="interrupted", error="服务停止，已保留处理进度，可重试", completed=True)
+        raise
+    except Exception as exc:
+        await store.update_import_job(job_id, status="failed", phase="failed", error=str(exc) or type(exc).__name__, completed=True)
+
+
+def start_file_job(app, job_id: str) -> bool:
+    if job_id in _tasks:
+        return False
+    task = asyncio.create_task(run_file_job(app, job_id))
+    _tasks[job_id] = task
+    task.add_done_callback(lambda _: _tasks.pop(job_id, None))
+    return True
+
+
+async def stop_file_jobs() -> None:
+    tasks = list(_tasks.values())
+    for task in tasks:
+        task.cancel()
+    if tasks:
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+
+async def retry_file_job(app, job_id: str) -> bool:
+    async with _queue_lock:
+        if job_id in _tasks:
+            return False
+        await app.state.assistant_knowledge_store.update_import_job(job_id, status="queued", phase="queued")
+        return start_file_job(app, job_id)

offline-backend-20260512/backend/app/gateway/deps.py

@@ -701,6 +701,9 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
         try:
             yield
         finally:
+            from app.gateway.assistant_file_ingest import stop_file_jobs
+
+            await stop_file_jobs()
             from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
 
             register_local_wiki_index_runtime(store=None, search=None, sync=None, retrieval=None)

offline-backend-20260512/backend/app/gateway/routers/assistant_knowledge.py

@@ -152,6 +152,18 @@ def _list_export_manifests(mapping_id: str, *, limit: int) -> list[dict[str, Any
 
 def _delete_base_package_files(base_id: str, import_job_ids: list[str]) -> None:
     """Delete package artifacts owned by one assistant base only."""
+    import shutil
+
+    from app.gateway.assistant_file_ingest import file_job_dir
+
+    for job_id in import_job_ids:
+        # UUID validation confines cleanup to exactly one job directory.
+        try:
+            directory = file_job_dir(job_id)
+        except ValueError:
+            continue
+        if directory.is_dir() and not directory.is_symlink():
+            shutil.rmtree(directory)
     root = _package_root()
     if not root.is_dir():
         return
@@ -448,10 +460,71 @@ async def import_skills(request: Request, _body: SkillImportRequest) -> dict[str
     raise HTTPException(status_code=410, detail="不支持技能直接归纳到助手知识库，请先归纳到 WeKnora 普通知识库后再导入助手知识库")
 
 
-@router.post("/imports/files", status_code=410)
-async def import_files(request: Request) -> dict[str, Any]:
+@router.post("/imports/files", status_code=202)
+async def import_files(
+    request: Request,
+    file: UploadFile = File(...),
+    assistant_base_id: str | None = Form(default=None),
+) -> dict[str, Any]:
+    import hashlib
+
+    from app.gateway.assistant_file_ingest import ALLOWED_EXTENSIONS, MAX_FILE_BYTES, file_job_dir, save_json, start_file_job
+
+    actor = await _actor(request, admin=True)
+    filename = (file.filename or "document").replace("\\", "/").rsplit("/", 1)[-1][:240]
+    extension = Path(filename).suffix.lower()
+    if extension not in ALLOWED_EXTENSIONS:
+        await file.close()
+        raise HTTPException(status_code=415, detail="支持 Markdown、TXT、CSV、PDF、Word、Excel、PowerPoint 文件")
+    job = await _store(request).create_queued_import_job(
+        source_type="file", source_key=f"file-upload:{uuid4()}", source_name=filename,
+        trigger="local_file_upload", created_by=actor, base_id=assistant_base_id,
+        metadata={"type": "file", "filename": filename}, phase="uploading",
+    )
+    root = file_job_dir(job["id"])
+    await asyncio.to_thread(root.mkdir, parents=True, exist_ok=True)
+    path = root / f"original{extension}"
+    size, digest = 0, hashlib.sha256()
+    try:
+        with path.open("wb") as handle:
+            while chunk := await file.read(1024 * 1024):
+                size += len(chunk)
+                if size > MAX_FILE_BYTES:
+                    raise HTTPException(status_code=413, detail="单个文件最大 100 MB，请拆分后上传")
+                digest.update(chunk)
+                await asyncio.to_thread(handle.write, chunk)
+        if not size:
+            raise HTTPException(status_code=422, detail="上传文件为空")
+        await asyncio.to_thread(save_json, root / "manifest.json", {
+            "filename": filename, "stored_name": path.name, "digest": digest.hexdigest(), "actor": actor,
+        })
+        job = await _store(request).update_import_job(job["id"], phase="queued", counts={"uploaded_bytes": size})
+        start_file_job(request.app, job["id"])
+        return job
+    except Exception as exc:
+        await _store(request).update_import_job(job["id"], status="failed", phase="upload_failed", error=str(getattr(exc, "detail", exc)), completed=True)
+        await asyncio.to_thread(path.unlink, missing_ok=True)
+        raise
+    finally:
+        await file.close()
+
+
+@router.post("/import-jobs/{job_id}/retry-file", status_code=202)
+async def retry_file_import(request: Request, job_id: str) -> dict[str, Any]:
+    from app.gateway.assistant_file_ingest import file_job_dir, retry_file_job
+
     await _actor(request, admin=True)
-    raise HTTPException(status_code=410, detail="助手知识库不支持导入任意本地文件，请从 WeKnora 普通知识库导出后导入")
+    store = _store(request)
+    job = await store.get_job(job_id)
+    if not job or job.get("trigger") != "local_file_upload":
+        raise HTTPException(status_code=404, detail="文件处理任务不存在")
+    if job["status"] == "completed":
+        raise HTTPException(status_code=409, detail="文件已处理完成")
+    if not (file_job_dir(job_id) / "manifest.json").exists():
+        raise HTTPException(status_code=409, detail="文件上传未完成，请重新上传")
+    if not await retry_file_job(request.app, job_id):
+        raise HTTPException(status_code=409, detail="文件仍在处理中")
+    return await store.get_job(job_id)
 
 
 @router.post("/imports/weknora-package", status_code=202)

offline-backend-20260512/backend/app/gateway/routers/system_settings.py

@@ -56,6 +56,22 @@ class KnowledgeBaseProductVersionUpdateRequest(BaseModel):
     version: Literal["v1", "v2"]
 
 
+class OrdinaryKnowledgeLanding(BaseModel):
+    mode: Literal["wiki", "weknora"] = "wiki"
+
+
+@router.get("/ordinary-knowledge-landing", response_model=OrdinaryKnowledgeLanding)
+async def get_ordinary_knowledge_landing() -> OrdinaryKnowledgeLanding:
+    return OrdinaryKnowledgeLanding(mode=load_system_settings().ordinary_knowledge_landing)
+
+
+@router.put("/ordinary-knowledge-landing", response_model=OrdinaryKnowledgeLanding)
+async def update_ordinary_knowledge_landing(body: OrdinaryKnowledgeLanding, request: Request) -> OrdinaryKnowledgeLanding:
+    await _require_admin(request)
+    saved = save_system_settings(load_system_settings().model_copy(update={"ordinary_knowledge_landing": body.mode}))
+    return OrdinaryKnowledgeLanding(mode=saved.ordinary_knowledge_landing)
+
+
 class PromptPrefixUpdateRequest(BaseModel):
     enabled: bool | None = None
     switch_label: str | None = None