feat: add skill and assistant knowledge bases
This commit is contained in:
1 parent
dd0ee282df
commit
fbd8dec48f
45 files changed
+5262
-21
No files matched your search
@@ -0,0 +1,338 @@
|
||||
"""Read-mostly global assistant knowledge API and admin import actions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from deerflow.assistant_knowledge import export_package_from_skill_ir
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.skill_knowledge import build_skill_ir, enrich_converted_documents, scan_skill
|
||||
|
||||
router = APIRouter(prefix="/api/assistant-knowledge", tags=["assistant-knowledge"])
|
||||
export_router = APIRouter(tags=["assistant-knowledge"])
|
||||
|
||||
|
||||
async def _actor(request: Request, *, admin: bool = False) -> str | None:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return None
|
||||
if admin and getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="该操作仅管理员可用")
|
||||
return str(user.id)
|
||||
|
||||
|
||||
def _store(request: Request):
|
||||
store = getattr(request.app.state, "assistant_knowledge_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="助手知识库存储不可用")
|
||||
return store
|
||||
|
||||
|
||||
class InitializeRequest(BaseModel):
|
||||
name: str = Field(default="知识梳理", min_length=1, max_length=255)
|
||||
|
||||
|
||||
class SkillImportRequest(BaseModel):
|
||||
skill_names: list[str] = Field(min_length=1, max_length=200)
|
||||
force: bool = False
|
||||
|
||||
|
||||
@router.get("/base")
|
||||
async def get_base(request: Request) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"base": await _store(request).get_base()}
|
||||
|
||||
|
||||
@router.post("/base/initialize", status_code=201)
|
||||
async def initialize(request: Request, body: InitializeRequest) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
return {"base": await _store(request).initialize(created_by=actor, name=body.name)}
|
||||
|
||||
|
||||
@router.get("/wiki-pages")
|
||||
async def list_wiki_pages(request: Request, q: str | None = Query(default=None), page_type: str | None = Query(default=None), limit: int = Query(default=100, ge=1, le=500)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
pages = await _store(request).list_pages(query=q, page_type=page_type, limit=limit)
|
||||
return {"pages": pages, "total": len(pages)}
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}/revisions/compare")
|
||||
async def compare_revisions(request: Request, slug: str, left: str = Query(...), right: str = Query(...)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
result = await _store(request).compare_revisions(left, right)
|
||||
if result is None:
|
||||
raise HTTPException(status_code=404, detail="版本不存在或不属于同一页面")
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}/revisions")
|
||||
async def list_revisions(request: Request, slug: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"revisions": await _store(request).list_revisions(slug)}
|
||||
|
||||
|
||||
@router.post("/wiki-pages/{slug:path}/revisions/{revision_id}/rollback")
|
||||
async def rollback_revision(request: Request, slug: str, revision_id: str) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
row = await _store(request).rollback(slug=slug, revision_id=revision_id, actor=actor)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="页面或版本不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}")
|
||||
async def get_wiki_page(request: Request, slug: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_page(slug)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="Wiki 页面不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
async def search(request: Request, q: str = Query(..., min_length=1), page_type: str | None = Query(default=None)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
rows = await _store(request).list_pages(query=q, page_type=page_type, limit=100)
|
||||
return {"results": rows, "total": len(rows)}
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
async def list_sources(request: Request) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"sources": await _store(request).list_sources()}
|
||||
|
||||
|
||||
@router.get("/entities/{entity_id}")
|
||||
async def get_entity(request: Request, entity_id: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_entity(entity_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="实体不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/relations/{relation_id}")
|
||||
async def get_relation(request: Request, relation_id: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_relation(relation_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="关系不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.delete("/sources/{source_id}", status_code=204)
|
||||
async def delete_source(request: Request, source_id: str) -> None:
|
||||
await _actor(request, admin=True)
|
||||
if not await _store(request).delete_source(source_id):
|
||||
raise HTTPException(status_code=404, detail="来源不存在")
|
||||
|
||||
|
||||
@router.get("/import-jobs")
|
||||
async def list_import_jobs(request: Request, limit: int = Query(default=100, ge=1, le=500)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"jobs": await _store(request).list_jobs(limit)}
|
||||
|
||||
|
||||
@router.get("/import-jobs/{job_id}")
|
||||
async def get_import_job(request: Request, job_id: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_job(job_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="导入任务不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/imports/skills", status_code=202)
|
||||
async def import_skills(request: Request, body: SkillImportRequest) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
base = await _store(request).initialize(created_by=actor)
|
||||
target = {"target_type": "assistant", "target_mode": "assistant", "target_id": base["id"], "target_name": base["name"]}
|
||||
job = await request.app.state.skill_knowledge_store.create_job(
|
||||
skill_names=body.skill_names,
|
||||
targets=[target],
|
||||
created_by=actor,
|
||||
force=body.force,
|
||||
review_mode="auto_high_confidence",
|
||||
trigger="skill_direct",
|
||||
)
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return job
|
||||
|
||||
|
||||
@router.post("/imports/files", status_code=201)
|
||||
async def import_files(request: Request, files: list[UploadFile] = File(...)) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
if not files or len(files) > 50:
|
||||
raise HTTPException(status_code=422, detail="一次可导入 1-50 个文件")
|
||||
batch_id = str(uuid4())
|
||||
jobs: list[dict[str, Any]] = []
|
||||
with tempfile.TemporaryDirectory(prefix="assistant-knowledge-") as tmp:
|
||||
root = Path(tmp)
|
||||
for upload in files:
|
||||
filename = Path(upload.filename or "upload.bin").name
|
||||
content = await upload.read()
|
||||
if len(content) > 50 * 1024 * 1024:
|
||||
raise HTTPException(status_code=413, detail=f"文件过大:{filename}")
|
||||
path = root / filename
|
||||
await asyncio.to_thread(path.write_bytes, content)
|
||||
scan = await asyncio.to_thread(scan_skill, f"local-{batch_id}", root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
if scan.blocked_files:
|
||||
raise HTTPException(status_code=422, detail={"message": "文件命中敏感信息阻断规则", "blocked_files": scan.blocked_files})
|
||||
ir = await asyncio.to_thread(build_skill_ir, scan, display_name="本地文件导入", description="管理员本地文件导入")
|
||||
package = export_package_from_skill_ir(ir)
|
||||
job = await _store(request).import_package(
|
||||
package=package,
|
||||
source_type="local_file",
|
||||
source_key=batch_id,
|
||||
source_name="、".join(Path(file.filename or "upload").name for file in files)[:255],
|
||||
trigger="manual_file_upload",
|
||||
created_by=actor,
|
||||
upload_batch_id=batch_id,
|
||||
)
|
||||
jobs.append(job)
|
||||
return {"batch_id": batch_id, "jobs": jobs}
|
||||
|
||||
|
||||
async def _export_weknora_package(client: Any, remote_id: str, source: dict[str, Any]) -> dict[str, Any]:
|
||||
pages: list[dict[str, Any]] = []
|
||||
page_no = 1
|
||||
while True:
|
||||
page = await client.list_wiki_pages(remote_id, page=page_no, page_size=100)
|
||||
batch = page.get("pages") or []
|
||||
pages.extend(batch)
|
||||
if len(pages) >= int(page.get("total") or len(pages)) or not batch:
|
||||
break
|
||||
page_no += 1
|
||||
documents: list[dict[str, Any]] = []
|
||||
chunks: list[dict[str, Any]] = []
|
||||
page_no = 1
|
||||
while True:
|
||||
page = await client.list_documents(remote_id, page=page_no, page_size=100)
|
||||
batch = page.get("items") or []
|
||||
documents.extend(batch)
|
||||
for document in batch:
|
||||
document_id = str(document.get("id") or document.get("knowledge_id") or "")
|
||||
if not document_id:
|
||||
continue
|
||||
chunk_page = 1
|
||||
while True:
|
||||
values = await client.list_chunks(document_id, page=chunk_page, page_size=100)
|
||||
chunk_batch = values.get("items") or []
|
||||
chunks.extend([{**row, "source_document_id": document_id} for row in chunk_batch])
|
||||
if len(chunk_batch) < 100:
|
||||
break
|
||||
chunk_page += 1
|
||||
if len(documents) >= int(page.get("total") or len(documents)) or not batch:
|
||||
break
|
||||
page_no += 1
|
||||
try:
|
||||
graph = await client.get_wiki_graph(remote_id, limit=500)
|
||||
except WeKnoraError:
|
||||
graph = {"nodes": [], "edges": [], "capability_warning": "graph API unavailable"}
|
||||
entities = [{"id": row.get("id"), "name": row.get("title") or row.get("name"), "type": row.get("type") or "concept", "confidence": 1.0} for row in graph.get("nodes") or [] if row.get("id") and (row.get("title") or row.get("name"))]
|
||||
relations = [
|
||||
{"source_entity_id": row.get("source"), "target_entity_id": row.get("target"), "predicate": row.get("predicate") or "references", "confidence": 1.0, "evidence": [{"source": "weknora_graph"}]} for row in graph.get("edges") or []
|
||||
]
|
||||
return {"source": source, "wiki_pages": pages, "documents": documents, "chunks": chunks, "entities": entities, "relations": relations, "graph": graph}
|
||||
|
||||
|
||||
@export_router.post("/api/llmwiki/knowledge-bases/{mapping_id}/export-to-assistant", status_code=202)
|
||||
async def export_to_assistant(request: Request, mapping_id: str) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
|
||||
if not runtime.weknora_enabled:
|
||||
raise HTTPException(status_code=503, detail="未配置普通知识库服务")
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, actor or "system", write=False, is_admin=True)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="普通知识库不存在")
|
||||
client = build_weknora_client(runtime)
|
||||
package = await _export_weknora_package(
|
||||
client,
|
||||
str(mapping["weknora_id"]),
|
||||
{
|
||||
"type": "weknora",
|
||||
"knowledge_base_id": mapping_id,
|
||||
"remote_knowledge_base_id": mapping["weknora_id"],
|
||||
"knowledge_base_name": mapping["name"],
|
||||
},
|
||||
)
|
||||
return await _store(request).import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key=mapping_id,
|
||||
source_name=str(mapping["name"]),
|
||||
trigger="manual_from_weknora",
|
||||
created_by=actor,
|
||||
mapping_id=mapping_id,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/sources/{source_id}/reimport", status_code=202)
|
||||
async def reimport_source(request: Request, source_id: str) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
store = _store(request)
|
||||
source = await store.get_source(source_id)
|
||||
if source is None:
|
||||
raise HTTPException(status_code=404, detail="来源不存在")
|
||||
if source["source_type"] == "skill":
|
||||
base = await store.initialize(created_by=actor)
|
||||
job = await request.app.state.skill_knowledge_store.create_job(
|
||||
skill_names=[str(source.get("skill_name") or source["source_key"])],
|
||||
targets=[
|
||||
{
|
||||
"target_type": "assistant",
|
||||
"target_mode": "assistant",
|
||||
"target_id": base["id"],
|
||||
"target_name": base["name"],
|
||||
}
|
||||
],
|
||||
created_by=actor,
|
||||
force=True,
|
||||
review_mode="auto_high_confidence",
|
||||
trigger="assistant_source_reimport",
|
||||
)
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return job
|
||||
if source["source_type"] != "weknora":
|
||||
raise HTTPException(status_code=409, detail="本地文件来源未保留原文件,请重新上传")
|
||||
mapping_id = str(source.get("knowledge_base_mapping_id") or source["source_key"])
|
||||
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
|
||||
if not runtime.weknora_enabled:
|
||||
raise HTTPException(status_code=503, detail="未配置普通知识库服务")
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(
|
||||
mapping_id,
|
||||
actor or "system",
|
||||
write=False,
|
||||
is_admin=True,
|
||||
)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="来源普通知识库不存在")
|
||||
package = await _export_weknora_package(
|
||||
build_weknora_client(runtime),
|
||||
str(mapping["weknora_id"]),
|
||||
{
|
||||
"type": "weknora",
|
||||
"knowledge_base_id": mapping_id,
|
||||
"remote_knowledge_base_id": mapping["weknora_id"],
|
||||
"knowledge_base_name": mapping["name"],
|
||||
},
|
||||
)
|
||||
return await store.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key=mapping_id,
|
||||
source_name=str(mapping["name"]),
|
||||
trigger="source_reimport",
|
||||
created_by=actor,
|
||||
mapping_id=mapping_id,
|
||||
)
|
||||
Reference in new issue
Block a user