feat(knowledge): support standalone file ingestion and configurable wiki entry
Use neutral knowledge-service labels and sanitize user-facing task errors.
This commit is contained in:
1 parent
f9b6d7e322
commit
3ca5528640
24 files changed
+549
-54
No files matched your search
@@ -1762,9 +1762,12 @@ gone.
|
||||
|
||||
Assistant knowledge has one global full-library base (`知识梳理总库`) and any
|
||||
number of admin-created custom bases. Ordinary users have read/search access and
|
||||
administrators own all mutations. Assistant bases only accept WeKnora export
|
||||
packages or imports triggered from a WeKnora mapping. Do not add direct skill
|
||||
imports or arbitrary local-file imports to assistant knowledge. Importing into a
|
||||
administrators own all mutations. Assistant bases accept local documents independently
|
||||
of WeKnora via `/api/assistant-knowledge/imports/files`, as well as WeKnora export
|
||||
packages or imports triggered from a WeKnora mapping. Local documents use the
|
||||
Gateway `assistant_file_ingest` pipeline: persisted original files, chunked LLM Wiki
|
||||
extraction, cached body embeddings, and import-job status/retry. Do not add direct skill
|
||||
imports. Importing into a
|
||||
custom assistant base must also mirror the package into the global base for
|
||||
cross-source entity alignment. Admins may rename any assistant base and delete
|
||||
custom bases. Deletion removes the custom base's Wiki pages, vectors, entities,
|
||||
@@ -1773,6 +1776,8 @@ deletable because it owns full-library aggregation.
|
||||
|
||||
### Wiki package and local encoder invariants (2026-09)
|
||||
|
||||
Ordinary knowledge detail landing is globally configured by `ordinary_knowledge_landing` (`wiki` default / `weknora`), with admin-only PUT `/api/system-settings/ordinary-knowledge-landing`. Explicit document/graph/article links bypass this default. Wiki reading and the embedded WeKnora view provide reciprocal navigation; `?view=weknora` explicitly selects the iframe and must not loop back to the default Wiki route.
|
||||
|
||||
- `knowledge_transfer.sync_to_global` is called after successful explicit local
|
||||
Wiki vectorization; it skips conversation/system mappings. Do not enable a new
|
||||
watcher or automatic remote-write scheduler as a side effect.
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
# DeerFlow Backend
|
||||
|
||||
普通知识库默认详情页面:管理员可在知识库顶部选择“本系统 Wiki 索引”或“嵌入 WeKnora”,对应 `ordinary_knowledge_landing`(默认 `wiki`)。Wiki 库首次进入默认展示本系统索引,右上角“打开 WeKnora”切换到内嵌管理页;内嵌页可返回索引。非 Wiki 库仍进入 WeKnora 文档页。设置通过 `/api/system-settings/ordinary-knowledge-landing` 保存,仅管理员可修改;指定文章、文档、图谱的直达链接优先于默认设置。
|
||||
|
||||
助手知识库可独立于 WeKnora 使用:管理员进入助手知识库,选择或创建库后点击“上传文件生成 Wiki”。支持 Markdown/TXT/CSV、PDF、Word、Excel、PowerPoint(单文件 100 MB,解析正文上限 400 万字符)。文件先写入运行目录 `assistant-knowledge/files/{job_id}`,后台分段调用已配置的语言模型生成分类 Wiki、实体关系,再通过 DeerFlow 编码器对正文分段向量化,保存到助手库并汇入知识梳理总库。无需配置 WeKnora URL;配置后仍可直接上传,也保留 WeKnora 数据包导入。文件处理沿用管理员写入权限,所有用户可以浏览和搜索。
|
||||
|
||||
导入记录显示解析、Wiki 生成和向量化阶段;失败或服务中断后可“重试 / 恢复处理”,复用已落盘的 Wiki 和相同模型指纹的向量。离线编码器须预先放置模型并在管理员设置中配置;模型或编码器不可用会明确失败,不把摘要冒充 Wiki 或虚构向量。扫描件是否可解析取决于部署的文档解析/OCR 能力。
|
||||
|
||||
DeerFlow is a LangGraph-based AI super agent with sandbox execution, persistent memory, and extensible tool integration. The backend enables AI agents to execute code, browse the web, manage files, delegate tasks to subagents, and retain context across conversations - all in isolated, per-thread environments.
|
||||
|
||||
---
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Local document → Wiki → vectors, independent of the WeKnora service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.runtime_paths import runtime_home
|
||||
from deerflow.knowledge.llm_extractor import extract_document_llm_multi
|
||||
from deerflow.utils.file_conversion import CONVERTIBLE_EXTENSIONS, convert_file_to_markdown
|
||||
|
||||
ALLOWED_EXTENSIONS = CONVERTIBLE_EXTENSIONS | {".md", ".markdown", ".txt", ".csv"}
|
||||
MAX_FILE_BYTES = 100 * 1024 * 1024
|
||||
MAX_TEXT_CHARS = 4_000_000
|
||||
_tasks: dict[str, asyncio.Task] = {}
|
||||
_slots = asyncio.Semaphore(2)
|
||||
_queue_lock = asyncio.Lock()
|
||||
|
||||
|
||||
def file_job_dir(job_id: str) -> Path:
|
||||
from uuid import UUID
|
||||
|
||||
return runtime_home() / "assistant-knowledge" / "files" / str(UUID(job_id))
|
||||
|
||||
|
||||
def save_json(path: Path, value: Any) -> None:
|
||||
temporary = path.with_suffix(".tmp")
|
||||
temporary.write_text(json.dumps(value, ensure_ascii=False), encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def split_text(text: str, size: int):
|
||||
"""Bound every extraction/embedding input without truncating the document."""
|
||||
for offset in range(0, len(text), size):
|
||||
yield text[offset : offset + size]
|
||||
|
||||
|
||||
def notes_package(notes: list[dict], source: dict) -> dict:
|
||||
from deerflow.persistence.assistant_knowledge.store import _entity_alignment_key
|
||||
|
||||
merged: dict[str, dict] = {}
|
||||
for raw in notes:
|
||||
key = _entity_alignment_key(raw["title"])
|
||||
if key not in merged:
|
||||
merged[key] = dict(raw)
|
||||
continue
|
||||
note = merged[key]
|
||||
note["content_md"] = "\n\n".join(dict.fromkeys([note["content_md"], raw["content_md"]]))
|
||||
for field in ("tags", "entities", "relations"):
|
||||
note[field] = [*note.get(field, []), *raw.get(field, [])]
|
||||
note["tags"] = list(dict.fromkeys(note["tags"]))
|
||||
pages, entities, relations = [], [], []
|
||||
for index, note in enumerate(merged.values()):
|
||||
slug = f"files/{source['digest']}/{index}"
|
||||
pages.append(
|
||||
{
|
||||
"slug": slug,
|
||||
"title": note["title"],
|
||||
"content": note["content_md"],
|
||||
"page_type": "article",
|
||||
"tags": note.get("tags", []),
|
||||
"category_path": [note.get("category") or "知识文档"],
|
||||
"source_refs": [{"type": "file", "title": source["filename"]}],
|
||||
}
|
||||
)
|
||||
for entity in note.get("entities", []):
|
||||
entities.append({"id": entity["name"], "name": entity["name"], "entity_type": entity.get("type", "concept"), "aliases": entity.get("aliases", []), "source_wiki_slug": slug})
|
||||
for relation in note.get("relations", []):
|
||||
relations.append({"source": relation["from"], "target": relation["to"], "predicate": relation.get("type", "related_to"), "source_wiki_slug": slug})
|
||||
# Persist real sibling links, understood by the shared Wiki reader.
|
||||
for index, page in enumerate(pages):
|
||||
others = [p for p in pages[max(0, index - 4) : index + 5] if p["slug"] != page["slug"]]
|
||||
page["out_links"] = [p["slug"] for p in others]
|
||||
page["in_links"] = list(page["out_links"])
|
||||
if others:
|
||||
page["content"] += "\n\n## 相关知识\n" + "\n".join(f"- [[{p['slug']}|{p['title']}]]" for p in others)
|
||||
return {"source": source, "wiki_pages": pages, "entities": entities, "relations": relations, "vectors": []}
|
||||
|
||||
|
||||
async def run_file_job(app, job_id: str) -> None:
|
||||
store = app.state.assistant_knowledge_store
|
||||
root = file_job_dir(job_id)
|
||||
try:
|
||||
async with _slots:
|
||||
manifest = await asyncio.to_thread(read_json, root / "manifest.json")
|
||||
await store.update_import_job(job_id, status="running", phase="parsing")
|
||||
package_file = root / "wiki.json"
|
||||
if package_file.exists():
|
||||
package = await asyncio.to_thread(read_json, package_file)
|
||||
else:
|
||||
text_file = root / "parsed.md"
|
||||
if not text_file.exists():
|
||||
original = root / manifest["stored_name"]
|
||||
if original.suffix in CONVERTIBLE_EXTENSIONS:
|
||||
converted = await asyncio.to_thread(lambda: asyncio.run(convert_file_to_markdown(original)))
|
||||
if converted is None:
|
||||
raise ValueError("无法解析文档,请检查文件是否损坏或是否需要 OCR")
|
||||
text = await asyncio.to_thread(converted.read_text, encoding="utf-8")
|
||||
else:
|
||||
raw = await asyncio.to_thread(original.read_bytes)
|
||||
try:
|
||||
text = raw.decode("utf-8-sig")
|
||||
except UnicodeDecodeError:
|
||||
text = raw.decode("gb18030")
|
||||
if not text.strip() or len(text) > MAX_TEXT_CHARS:
|
||||
raise ValueError("文件没有可提取文本,或解析正文超过 400 万字符,请拆分文件")
|
||||
await asyncio.to_thread(text_file.write_text, text, encoding="utf-8")
|
||||
else:
|
||||
text = await asyncio.to_thread(text_file.read_text, encoding="utf-8")
|
||||
notes = []
|
||||
parts = list(split_text(text, 10000))
|
||||
for index, part in enumerate(parts):
|
||||
await store.update_import_job(job_id, phase="generating_wiki", counts={"parts_total": len(parts), "parts_done": index})
|
||||
cache = root / f"notes-{index}.json"
|
||||
if cache.exists():
|
||||
drafts = await asyncio.to_thread(read_json, cache)
|
||||
else:
|
||||
from dataclasses import asdict
|
||||
|
||||
extracted = await asyncio.wait_for(
|
||||
extract_document_llm_multi(
|
||||
part,
|
||||
doc_id=f"{job_id}:{index}",
|
||||
doc_title=manifest["filename"],
|
||||
max_input_chars=12000,
|
||||
max_notes=6,
|
||||
),
|
||||
timeout=300,
|
||||
)
|
||||
if not extracted:
|
||||
raise ValueError("Wiki 生成失败,请在设置中检查可用的语言模型,然后重试")
|
||||
drafts = [asdict(note) for note in extracted]
|
||||
await asyncio.to_thread(save_json, cache, drafts)
|
||||
notes.extend(drafts)
|
||||
package = notes_package(notes, {"type": "file", "filename": manifest["filename"], "digest": manifest["digest"]})
|
||||
await asyncio.to_thread(save_json, package_file, package)
|
||||
embedding = getattr(app.state, "llmwiki_embedding", None)
|
||||
if embedding is None:
|
||||
raise ValueError("Wiki 已生成;请管理员在设置中配置本地编码器或编码服务,然后重试向量化")
|
||||
vectors = []
|
||||
sections = [(p, i, section) for p in package["wiki_pages"] for i, section in enumerate(split_text(p["content"], 1600))]
|
||||
for index, (page, section_index, content) in enumerate(sections):
|
||||
await store.update_import_job(job_id, phase="vectorizing", counts={"wiki_pages": len(package["wiki_pages"]), "vectors_total": len(sections), "vectors": index})
|
||||
digest = hashlib.sha256(content.encode()).hexdigest()
|
||||
cache = root / f"vector-{digest}.json"
|
||||
record = await asyncio.to_thread(read_json, cache) if cache.exists() else None
|
||||
if record is None or record["embedding_fingerprint"] != embedding.fingerprint:
|
||||
values = await embedding.embed_texts([content])
|
||||
record = {"vector": values[0], "embedding_model": embedding.config.model, "embedding_fingerprint": embedding.fingerprint, "content_hash": digest}
|
||||
await asyncio.to_thread(save_json, cache, record)
|
||||
vectors.append({**record, "wiki_slug": page["slug"], "section_index": section_index, "section_content": content})
|
||||
package["vectors"] = vectors
|
||||
await store.update_import_job(job_id, phase="importing")
|
||||
await store.import_package_for_job(job_id, package=package, created_by=manifest["actor"])
|
||||
except asyncio.CancelledError:
|
||||
await store.update_import_job(job_id, status="failed", phase="interrupted", error="服务停止,已保留处理进度,可重试", completed=True)
|
||||
raise
|
||||
except Exception as exc:
|
||||
await store.update_import_job(job_id, status="failed", phase="failed", error=str(exc) or type(exc).__name__, completed=True)
|
||||
|
||||
|
||||
def start_file_job(app, job_id: str) -> bool:
|
||||
if job_id in _tasks:
|
||||
return False
|
||||
task = asyncio.create_task(run_file_job(app, job_id))
|
||||
_tasks[job_id] = task
|
||||
task.add_done_callback(lambda _: _tasks.pop(job_id, None))
|
||||
return True
|
||||
|
||||
|
||||
async def stop_file_jobs() -> None:
|
||||
tasks = list(_tasks.values())
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
|
||||
async def retry_file_job(app, job_id: str) -> bool:
|
||||
async with _queue_lock:
|
||||
if job_id in _tasks:
|
||||
return False
|
||||
await app.state.assistant_knowledge_store.update_import_job(job_id, status="queued", phase="queued")
|
||||
return start_file_job(app, job_id)
|
||||
@@ -701,6 +701,9 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
from app.gateway.assistant_file_ingest import stop_file_jobs
|
||||
|
||||
await stop_file_jobs()
|
||||
from deerflow.integrations.weknora.local_index.runtime import register_local_wiki_index_runtime
|
||||
|
||||
register_local_wiki_index_runtime(store=None, search=None, sync=None, retrieval=None)
|
||||
|
||||
@@ -152,6 +152,18 @@ def _list_export_manifests(mapping_id: str, *, limit: int) -> list[dict[str, Any
|
||||
|
||||
def _delete_base_package_files(base_id: str, import_job_ids: list[str]) -> None:
|
||||
"""Delete package artifacts owned by one assistant base only."""
|
||||
import shutil
|
||||
|
||||
from app.gateway.assistant_file_ingest import file_job_dir
|
||||
|
||||
for job_id in import_job_ids:
|
||||
# UUID validation confines cleanup to exactly one job directory.
|
||||
try:
|
||||
directory = file_job_dir(job_id)
|
||||
except ValueError:
|
||||
continue
|
||||
if directory.is_dir() and not directory.is_symlink():
|
||||
shutil.rmtree(directory)
|
||||
root = _package_root()
|
||||
if not root.is_dir():
|
||||
return
|
||||
@@ -448,10 +460,71 @@ async def import_skills(request: Request, _body: SkillImportRequest) -> dict[str
|
||||
raise HTTPException(status_code=410, detail="不支持技能直接归纳到助手知识库,请先归纳到 WeKnora 普通知识库后再导入助手知识库")
|
||||
|
||||
|
||||
@router.post("/imports/files", status_code=410)
|
||||
async def import_files(request: Request) -> dict[str, Any]:
|
||||
@router.post("/imports/files", status_code=202)
|
||||
async def import_files(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
assistant_base_id: str | None = Form(default=None),
|
||||
) -> dict[str, Any]:
|
||||
import hashlib
|
||||
|
||||
from app.gateway.assistant_file_ingest import ALLOWED_EXTENSIONS, MAX_FILE_BYTES, file_job_dir, save_json, start_file_job
|
||||
|
||||
actor = await _actor(request, admin=True)
|
||||
filename = (file.filename or "document").replace("\\", "/").rsplit("/", 1)[-1][:240]
|
||||
extension = Path(filename).suffix.lower()
|
||||
if extension not in ALLOWED_EXTENSIONS:
|
||||
await file.close()
|
||||
raise HTTPException(status_code=415, detail="支持 Markdown、TXT、CSV、PDF、Word、Excel、PowerPoint 文件")
|
||||
job = await _store(request).create_queued_import_job(
|
||||
source_type="file", source_key=f"file-upload:{uuid4()}", source_name=filename,
|
||||
trigger="local_file_upload", created_by=actor, base_id=assistant_base_id,
|
||||
metadata={"type": "file", "filename": filename}, phase="uploading",
|
||||
)
|
||||
root = file_job_dir(job["id"])
|
||||
await asyncio.to_thread(root.mkdir, parents=True, exist_ok=True)
|
||||
path = root / f"original{extension}"
|
||||
size, digest = 0, hashlib.sha256()
|
||||
try:
|
||||
with path.open("wb") as handle:
|
||||
while chunk := await file.read(1024 * 1024):
|
||||
size += len(chunk)
|
||||
if size > MAX_FILE_BYTES:
|
||||
raise HTTPException(status_code=413, detail="单个文件最大 100 MB,请拆分后上传")
|
||||
digest.update(chunk)
|
||||
await asyncio.to_thread(handle.write, chunk)
|
||||
if not size:
|
||||
raise HTTPException(status_code=422, detail="上传文件为空")
|
||||
await asyncio.to_thread(save_json, root / "manifest.json", {
|
||||
"filename": filename, "stored_name": path.name, "digest": digest.hexdigest(), "actor": actor,
|
||||
})
|
||||
job = await _store(request).update_import_job(job["id"], phase="queued", counts={"uploaded_bytes": size})
|
||||
start_file_job(request.app, job["id"])
|
||||
return job
|
||||
except Exception as exc:
|
||||
await _store(request).update_import_job(job["id"], status="failed", phase="upload_failed", error=str(getattr(exc, "detail", exc)), completed=True)
|
||||
await asyncio.to_thread(path.unlink, missing_ok=True)
|
||||
raise
|
||||
finally:
|
||||
await file.close()
|
||||
|
||||
|
||||
@router.post("/import-jobs/{job_id}/retry-file", status_code=202)
|
||||
async def retry_file_import(request: Request, job_id: str) -> dict[str, Any]:
|
||||
from app.gateway.assistant_file_ingest import file_job_dir, retry_file_job
|
||||
|
||||
await _actor(request, admin=True)
|
||||
raise HTTPException(status_code=410, detail="助手知识库不支持导入任意本地文件,请从 WeKnora 普通知识库导出后导入")
|
||||
store = _store(request)
|
||||
job = await store.get_job(job_id)
|
||||
if not job or job.get("trigger") != "local_file_upload":
|
||||
raise HTTPException(status_code=404, detail="文件处理任务不存在")
|
||||
if job["status"] == "completed":
|
||||
raise HTTPException(status_code=409, detail="文件已处理完成")
|
||||
if not (file_job_dir(job_id) / "manifest.json").exists():
|
||||
raise HTTPException(status_code=409, detail="文件上传未完成,请重新上传")
|
||||
if not await retry_file_job(request.app, job_id):
|
||||
raise HTTPException(status_code=409, detail="文件仍在处理中")
|
||||
return await store.get_job(job_id)
|
||||
|
||||
|
||||
@router.post("/imports/weknora-package", status_code=202)
|
||||
|
||||
@@ -56,6 +56,22 @@ class KnowledgeBaseProductVersionUpdateRequest(BaseModel):
|
||||
version: Literal["v1", "v2"]
|
||||
|
||||
|
||||
class OrdinaryKnowledgeLanding(BaseModel):
|
||||
mode: Literal["wiki", "weknora"] = "wiki"
|
||||
|
||||
|
||||
@router.get("/ordinary-knowledge-landing", response_model=OrdinaryKnowledgeLanding)
|
||||
async def get_ordinary_knowledge_landing() -> OrdinaryKnowledgeLanding:
|
||||
return OrdinaryKnowledgeLanding(mode=load_system_settings().ordinary_knowledge_landing)
|
||||
|
||||
|
||||
@router.put("/ordinary-knowledge-landing", response_model=OrdinaryKnowledgeLanding)
|
||||
async def update_ordinary_knowledge_landing(body: OrdinaryKnowledgeLanding, request: Request) -> OrdinaryKnowledgeLanding:
|
||||
await _require_admin(request)
|
||||
saved = save_system_settings(load_system_settings().model_copy(update={"ordinary_knowledge_landing": body.mode}))
|
||||
return OrdinaryKnowledgeLanding(mode=saved.ordinary_knowledge_landing)
|
||||
|
||||
|
||||
class PromptPrefixUpdateRequest(BaseModel):
|
||||
enabled: bool | None = None
|
||||
switch_label: str | None = None
|
||||
|
||||
@@ -538,6 +538,7 @@ class SystemSettings(BaseModel):
|
||||
default="v1",
|
||||
description="v1 opens the ordinary WeKnora experience; v2 opens the DeerFlow assistant knowledge base.",
|
||||
)
|
||||
ordinary_knowledge_landing: Literal["wiki", "weknora"] = "wiki"
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
|
||||
|
||||
|
||||
+2
@@ -241,6 +241,8 @@ class AssistantKnowledgeRepository:
|
||||
) -> tuple[str, str]:
|
||||
metadata = package.get("source") or {}
|
||||
provider = str(metadata.get("provider") or metadata.get("type") or "").strip().casefold()
|
||||
if provider == "file" and metadata.get("digest"):
|
||||
return f"file:{metadata['digest']}", str(metadata.get("filename") or fallback_name)
|
||||
if provider == "weknora":
|
||||
source_key = str(
|
||||
metadata.get("knowledge_base_id")
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||||
|
||||
from app.gateway import assistant_file_ingest as ingest
|
||||
from app.gateway.routers import assistant_knowledge as router
|
||||
from deerflow.knowledge.schemas import NoteDraft
|
||||
from deerflow.persistence.assistant_knowledge import AssistantKnowledgeRepository
|
||||
from deerflow.persistence.base import Base
|
||||
|
||||
|
||||
def test_same_topic_across_document_parts_preserves_all_knowledge():
|
||||
package = ingest.notes_package([
|
||||
{"title": "设备维护", "content_md": "每周清理过滤网", "tags": ["运维"]},
|
||||
{"title": "设备 维护", "content_md": "每月检查电机温度", "tags": ["设备"]},
|
||||
], {"digest": "document", "filename": "维护指南.md"})
|
||||
assert len(package["wiki_pages"]) == 1
|
||||
assert "每周清理过滤网" in package["wiki_pages"][0]["content"]
|
||||
assert "每月检查电机温度" in package["wiki_pages"][0]["content"]
|
||||
assert package["wiki_pages"][0]["tags"] == ["运维", "设备"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_standalone_upload_vectors_retry_and_dedup(tmp_path, monkeypatch):
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||||
async with engine.begin() as connection:
|
||||
await connection.run_sync(Base.metadata.create_all)
|
||||
store = AssistantKnowledgeRepository(async_sessionmaker(engine, expire_on_commit=False))
|
||||
base = await store.create_base(name="独立文档库", created_by="admin")
|
||||
app = FastAPI()
|
||||
app.include_router(router.router)
|
||||
app.state.assistant_knowledge_store = store
|
||||
app.state.llmwiki_embedding = None
|
||||
monkeypatch.setattr(ingest, "runtime_home", lambda: tmp_path)
|
||||
extraction_calls = []
|
||||
|
||||
async def actor(*args, **kwargs):
|
||||
return "admin"
|
||||
|
||||
async def extract(text, **kwargs):
|
||||
extraction_calls.append(text)
|
||||
return [NoteDraft(title="冷却系统", summary="摘要占位", content_md="# 冷却系统\n风扇的额定转速为每分钟 1200 转。", tags=["设备"], entities=[{"name": "风扇", "type": "product"}])]
|
||||
|
||||
class Encoder:
|
||||
fingerprint = "local-test-3"
|
||||
dimensions = 3
|
||||
config = SimpleNamespace(model="offline-test")
|
||||
|
||||
async def embed_texts(self, texts):
|
||||
return [[1.0, 0.0, 0.0] for _ in texts]
|
||||
|
||||
async def embed_query(self, text):
|
||||
return [1.0, 0.0, 0.0]
|
||||
|
||||
monkeypatch.setattr(router, "_actor", actor)
|
||||
monkeypatch.setattr(ingest, "extract_document_llm_multi", extract)
|
||||
|
||||
async def finish():
|
||||
if ingest._tasks:
|
||||
await asyncio.gather(*list(ingest._tasks.values()))
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client:
|
||||
payload = {"assistant_base_id": base["id"]}
|
||||
response = await client.post("/api/assistant-knowledge/imports/files", data=payload, files={"file": ("../../设备.md", "风扇额定转速为 1200 转。".encode())})
|
||||
assert response.status_code == 202, response.text
|
||||
job_id = response.json()["id"]
|
||||
await finish()
|
||||
assert (await store.get_job(job_id))["status"] == "failed"
|
||||
assert (ingest.file_job_dir(job_id) / "wiki.json").exists()
|
||||
app.state.llmwiki_embedding = Encoder()
|
||||
retry = await client.post(f"/api/assistant-knowledge/import-jobs/{job_id}/retry-file")
|
||||
assert retry.status_code == 202
|
||||
await finish()
|
||||
job = await store.get_job(job_id)
|
||||
assert job["status"] == "completed", job
|
||||
assert job["counts"]["vectors"] == 1
|
||||
assert len(extraction_calls) == 1
|
||||
results = await store.search_pages(base_ids=[base["id"]], query="额定转速", embedding_client=app.state.llmwiki_embedding)
|
||||
assert results and results[0]["retrieval_mode"] == "vector"
|
||||
assert "1200" in results[0]["matched_section"]["content"]
|
||||
assert "摘要占位" not in results[0]["matched_section"]["content"]
|
||||
global_base = await store.get_base()
|
||||
assert len(await store.list_pages(base_id=global_base["id"])) == 1
|
||||
again = await client.post("/api/assistant-knowledge/imports/files", data=payload, files={"file": ("设备.md", "风扇额定转速为 1200 转。".encode())})
|
||||
assert again.status_code == 202
|
||||
await finish()
|
||||
assert len(await store.list_sources(base_id=base["id"])) == 1
|
||||
assert len(await store.list_pages(base_id=global_base["id"])) == 1
|
||||
bad = await client.post("/api/assistant-knowledge/imports/files", data=payload, files={"file": ("script.py", b"print('no')")})
|
||||
assert bad.status_code == 415
|
||||
empty = await client.post("/api/assistant-knowledge/imports/files", data=payload, files={"file": ("empty.txt", b"")})
|
||||
assert empty.status_code == 422
|
||||
finally:
|
||||
await ingest.stop_file_jobs()
|
||||
await engine.dispose()
|
||||
Reference in new issue
Block a user