feat: add skill and assistant knowledge bases
This commit is contained in:
1 parent
dd0ee282df
commit
fbd8dec48f
45 files changed
+5262
-21
No files matched your search
@@ -30,9 +30,10 @@ from app.gateway.routers import (
|
||||
ai_writing,
|
||||
admin_active_runs,
|
||||
admin_users,
|
||||
agents,
|
||||
artifacts,
|
||||
assistants_compat,
|
||||
agents,
|
||||
artifacts,
|
||||
assistant_knowledge,
|
||||
assistants_compat,
|
||||
auth,
|
||||
browser_context,
|
||||
business_mapping,
|
||||
@@ -83,9 +84,10 @@ from app.gateway.routers import (
|
||||
roundtable_task_drafts,
|
||||
runs,
|
||||
scheduled_tasks,
|
||||
sensitive_words,
|
||||
skill_addresses,
|
||||
skills,
|
||||
sensitive_words,
|
||||
skill_addresses,
|
||||
skill_knowledge,
|
||||
skills,
|
||||
suggestions,
|
||||
system_settings,
|
||||
tags,
|
||||
@@ -1503,8 +1505,12 @@ This gateway provides custom endpoints for models, MCP configuration, skills, an
|
||||
# Memory API is mounted at /api/memory
|
||||
app.include_router(memory.router)
|
||||
|
||||
# Skills API is mounted at /api/skills
|
||||
app.include_router(skills.router)
|
||||
# Skills API is mounted at /api/skills
|
||||
app.include_router(skills.router)
|
||||
# Admin skill distillation matrix + global assistant knowledge base.
|
||||
app.include_router(skill_knowledge.router)
|
||||
app.include_router(assistant_knowledge.router)
|
||||
app.include_router(assistant_knowledge.export_router)
|
||||
|
||||
# Artifacts API is mounted at /api/threads/{thread_id}/artifacts
|
||||
app.include_router(artifacts.router)
|
||||
|
||||
@@ -315,6 +315,33 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
app.state.llmwiki_store = make_llmwiki_store(sf)
|
||||
register_llmwiki_store(app.state.llmwiki_store)
|
||||
|
||||
# Skill distillation and the single global assistant knowledge base.
|
||||
# Both repositories live in ``deerflow.*``; only this composition layer
|
||||
# wires them to Gateway executors and remote adapters.
|
||||
from deerflow.persistence.assistant_knowledge import make_assistant_knowledge_store
|
||||
from deerflow.persistence.skill_knowledge import make_skill_knowledge_store
|
||||
|
||||
app.state.assistant_knowledge_store = make_assistant_knowledge_store(sf)
|
||||
app.state.skill_knowledge_store = make_skill_knowledge_store(sf)
|
||||
if app.state.skill_knowledge_store is not None and app.state.assistant_knowledge_store is not None:
|
||||
from app.gateway.skill_knowledge_job_dispatcher import SkillKnowledgeJobDispatcher
|
||||
from app.gateway.skill_knowledge_job_executor import SkillKnowledgeJobExecutor
|
||||
|
||||
app.state.skill_knowledge_executor = SkillKnowledgeJobExecutor(app)
|
||||
app.state.skill_knowledge_dispatcher = SkillKnowledgeJobDispatcher(
|
||||
app.state.skill_knowledge_store,
|
||||
app.state.skill_knowledge_executor,
|
||||
)
|
||||
app.state.skill_knowledge_dispatcher.start()
|
||||
from app.gateway.skill_knowledge_auto_sync import SkillKnowledgeAutoSyncService
|
||||
|
||||
app.state.skill_knowledge_auto_sync = SkillKnowledgeAutoSyncService(app)
|
||||
app.state.skill_knowledge_auto_sync.start()
|
||||
else:
|
||||
app.state.skill_knowledge_executor = None
|
||||
app.state.skill_knowledge_dispatcher = None
|
||||
app.state.skill_knowledge_auto_sync = None
|
||||
|
||||
# DeerFlow-local Wiki mirror/vector index. The store is always wired so
|
||||
# migrations and status inspection work while feature flags are off;
|
||||
# network/search services only exist when explicitly enabled.
|
||||
@@ -580,10 +607,16 @@ async def langgraph_runtime(app: FastAPI) -> AsyncGenerator[None, None]:
|
||||
dispatcher = getattr(app.state, "roundtable_job_dispatcher", None)
|
||||
if dispatcher is not None:
|
||||
await dispatcher.stop()
|
||||
dr_dispatcher = getattr(app.state, "deep_research_dispatcher", None)
|
||||
if dr_dispatcher is not None:
|
||||
await dr_dispatcher.stop()
|
||||
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
|
||||
dr_dispatcher = getattr(app.state, "deep_research_dispatcher", None)
|
||||
if dr_dispatcher is not None:
|
||||
await dr_dispatcher.stop()
|
||||
skill_knowledge_auto_sync = getattr(app.state, "skill_knowledge_auto_sync", None)
|
||||
if skill_knowledge_auto_sync is not None:
|
||||
await skill_knowledge_auto_sync.close()
|
||||
skill_knowledge_dispatcher = getattr(app.state, "skill_knowledge_dispatcher", None)
|
||||
if skill_knowledge_dispatcher is not None:
|
||||
await skill_knowledge_dispatcher.close()
|
||||
queue = getattr(app.state, "knowledge_auto_ingest_queue", None)
|
||||
if queue is not None:
|
||||
from deerflow.knowledge.auto_ingest import set_auto_ingest_queue
|
||||
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
"""Read-mostly global assistant knowledge API and admin import actions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import APIRouter, File, HTTPException, Query, Request, UploadFile
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from deerflow.assistant_knowledge import export_package_from_skill_ir
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.skill_knowledge import build_skill_ir, enrich_converted_documents, scan_skill
|
||||
|
||||
router = APIRouter(prefix="/api/assistant-knowledge", tags=["assistant-knowledge"])
|
||||
export_router = APIRouter(tags=["assistant-knowledge"])
|
||||
|
||||
|
||||
async def _actor(request: Request, *, admin: bool = False) -> str | None:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return None
|
||||
if admin and getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="该操作仅管理员可用")
|
||||
return str(user.id)
|
||||
|
||||
|
||||
def _store(request: Request):
|
||||
store = getattr(request.app.state, "assistant_knowledge_store", None)
|
||||
if store is None:
|
||||
raise HTTPException(status_code=503, detail="助手知识库存储不可用")
|
||||
return store
|
||||
|
||||
|
||||
class InitializeRequest(BaseModel):
|
||||
name: str = Field(default="知识梳理", min_length=1, max_length=255)
|
||||
|
||||
|
||||
class SkillImportRequest(BaseModel):
|
||||
skill_names: list[str] = Field(min_length=1, max_length=200)
|
||||
force: bool = False
|
||||
|
||||
|
||||
@router.get("/base")
|
||||
async def get_base(request: Request) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"base": await _store(request).get_base()}
|
||||
|
||||
|
||||
@router.post("/base/initialize", status_code=201)
|
||||
async def initialize(request: Request, body: InitializeRequest) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
return {"base": await _store(request).initialize(created_by=actor, name=body.name)}
|
||||
|
||||
|
||||
@router.get("/wiki-pages")
|
||||
async def list_wiki_pages(request: Request, q: str | None = Query(default=None), page_type: str | None = Query(default=None), limit: int = Query(default=100, ge=1, le=500)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
pages = await _store(request).list_pages(query=q, page_type=page_type, limit=limit)
|
||||
return {"pages": pages, "total": len(pages)}
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}/revisions/compare")
|
||||
async def compare_revisions(request: Request, slug: str, left: str = Query(...), right: str = Query(...)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
result = await _store(request).compare_revisions(left, right)
|
||||
if result is None:
|
||||
raise HTTPException(status_code=404, detail="版本不存在或不属于同一页面")
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}/revisions")
|
||||
async def list_revisions(request: Request, slug: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"revisions": await _store(request).list_revisions(slug)}
|
||||
|
||||
|
||||
@router.post("/wiki-pages/{slug:path}/revisions/{revision_id}/rollback")
|
||||
async def rollback_revision(request: Request, slug: str, revision_id: str) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
row = await _store(request).rollback(slug=slug, revision_id=revision_id, actor=actor)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="页面或版本不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/wiki-pages/{slug:path}")
|
||||
async def get_wiki_page(request: Request, slug: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_page(slug)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="Wiki 页面不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
async def search(request: Request, q: str = Query(..., min_length=1), page_type: str | None = Query(default=None)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
rows = await _store(request).list_pages(query=q, page_type=page_type, limit=100)
|
||||
return {"results": rows, "total": len(rows)}
|
||||
|
||||
|
||||
@router.get("/sources")
|
||||
async def list_sources(request: Request) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"sources": await _store(request).list_sources()}
|
||||
|
||||
|
||||
@router.get("/entities/{entity_id}")
|
||||
async def get_entity(request: Request, entity_id: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_entity(entity_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="实体不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/relations/{relation_id}")
|
||||
async def get_relation(request: Request, relation_id: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_relation(relation_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="关系不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.delete("/sources/{source_id}", status_code=204)
|
||||
async def delete_source(request: Request, source_id: str) -> None:
|
||||
await _actor(request, admin=True)
|
||||
if not await _store(request).delete_source(source_id):
|
||||
raise HTTPException(status_code=404, detail="来源不存在")
|
||||
|
||||
|
||||
@router.get("/import-jobs")
|
||||
async def list_import_jobs(request: Request, limit: int = Query(default=100, ge=1, le=500)) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
return {"jobs": await _store(request).list_jobs(limit)}
|
||||
|
||||
|
||||
@router.get("/import-jobs/{job_id}")
|
||||
async def get_import_job(request: Request, job_id: str) -> dict[str, Any]:
|
||||
await _actor(request)
|
||||
row = await _store(request).get_job(job_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="导入任务不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/imports/skills", status_code=202)
|
||||
async def import_skills(request: Request, body: SkillImportRequest) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
base = await _store(request).initialize(created_by=actor)
|
||||
target = {"target_type": "assistant", "target_mode": "assistant", "target_id": base["id"], "target_name": base["name"]}
|
||||
job = await request.app.state.skill_knowledge_store.create_job(
|
||||
skill_names=body.skill_names,
|
||||
targets=[target],
|
||||
created_by=actor,
|
||||
force=body.force,
|
||||
review_mode="auto_high_confidence",
|
||||
trigger="skill_direct",
|
||||
)
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return job
|
||||
|
||||
|
||||
@router.post("/imports/files", status_code=201)
|
||||
async def import_files(request: Request, files: list[UploadFile] = File(...)) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
if not files or len(files) > 50:
|
||||
raise HTTPException(status_code=422, detail="一次可导入 1-50 个文件")
|
||||
batch_id = str(uuid4())
|
||||
jobs: list[dict[str, Any]] = []
|
||||
with tempfile.TemporaryDirectory(prefix="assistant-knowledge-") as tmp:
|
||||
root = Path(tmp)
|
||||
for upload in files:
|
||||
filename = Path(upload.filename or "upload.bin").name
|
||||
content = await upload.read()
|
||||
if len(content) > 50 * 1024 * 1024:
|
||||
raise HTTPException(status_code=413, detail=f"文件过大:{filename}")
|
||||
path = root / filename
|
||||
await asyncio.to_thread(path.write_bytes, content)
|
||||
scan = await asyncio.to_thread(scan_skill, f"local-{batch_id}", root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
if scan.blocked_files:
|
||||
raise HTTPException(status_code=422, detail={"message": "文件命中敏感信息阻断规则", "blocked_files": scan.blocked_files})
|
||||
ir = await asyncio.to_thread(build_skill_ir, scan, display_name="本地文件导入", description="管理员本地文件导入")
|
||||
package = export_package_from_skill_ir(ir)
|
||||
job = await _store(request).import_package(
|
||||
package=package,
|
||||
source_type="local_file",
|
||||
source_key=batch_id,
|
||||
source_name="、".join(Path(file.filename or "upload").name for file in files)[:255],
|
||||
trigger="manual_file_upload",
|
||||
created_by=actor,
|
||||
upload_batch_id=batch_id,
|
||||
)
|
||||
jobs.append(job)
|
||||
return {"batch_id": batch_id, "jobs": jobs}
|
||||
|
||||
|
||||
async def _export_weknora_package(client: Any, remote_id: str, source: dict[str, Any]) -> dict[str, Any]:
|
||||
pages: list[dict[str, Any]] = []
|
||||
page_no = 1
|
||||
while True:
|
||||
page = await client.list_wiki_pages(remote_id, page=page_no, page_size=100)
|
||||
batch = page.get("pages") or []
|
||||
pages.extend(batch)
|
||||
if len(pages) >= int(page.get("total") or len(pages)) or not batch:
|
||||
break
|
||||
page_no += 1
|
||||
documents: list[dict[str, Any]] = []
|
||||
chunks: list[dict[str, Any]] = []
|
||||
page_no = 1
|
||||
while True:
|
||||
page = await client.list_documents(remote_id, page=page_no, page_size=100)
|
||||
batch = page.get("items") or []
|
||||
documents.extend(batch)
|
||||
for document in batch:
|
||||
document_id = str(document.get("id") or document.get("knowledge_id") or "")
|
||||
if not document_id:
|
||||
continue
|
||||
chunk_page = 1
|
||||
while True:
|
||||
values = await client.list_chunks(document_id, page=chunk_page, page_size=100)
|
||||
chunk_batch = values.get("items") or []
|
||||
chunks.extend([{**row, "source_document_id": document_id} for row in chunk_batch])
|
||||
if len(chunk_batch) < 100:
|
||||
break
|
||||
chunk_page += 1
|
||||
if len(documents) >= int(page.get("total") or len(documents)) or not batch:
|
||||
break
|
||||
page_no += 1
|
||||
try:
|
||||
graph = await client.get_wiki_graph(remote_id, limit=500)
|
||||
except WeKnoraError:
|
||||
graph = {"nodes": [], "edges": [], "capability_warning": "graph API unavailable"}
|
||||
entities = [{"id": row.get("id"), "name": row.get("title") or row.get("name"), "type": row.get("type") or "concept", "confidence": 1.0} for row in graph.get("nodes") or [] if row.get("id") and (row.get("title") or row.get("name"))]
|
||||
relations = [
|
||||
{"source_entity_id": row.get("source"), "target_entity_id": row.get("target"), "predicate": row.get("predicate") or "references", "confidence": 1.0, "evidence": [{"source": "weknora_graph"}]} for row in graph.get("edges") or []
|
||||
]
|
||||
return {"source": source, "wiki_pages": pages, "documents": documents, "chunks": chunks, "entities": entities, "relations": relations, "graph": graph}
|
||||
|
||||
|
||||
@export_router.post("/api/llmwiki/knowledge-bases/{mapping_id}/export-to-assistant", status_code=202)
|
||||
async def export_to_assistant(request: Request, mapping_id: str) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
|
||||
if not runtime.weknora_enabled:
|
||||
raise HTTPException(status_code=503, detail="未配置普通知识库服务")
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(mapping_id, actor or "system", write=False, is_admin=True)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="普通知识库不存在")
|
||||
client = build_weknora_client(runtime)
|
||||
package = await _export_weknora_package(
|
||||
client,
|
||||
str(mapping["weknora_id"]),
|
||||
{
|
||||
"type": "weknora",
|
||||
"knowledge_base_id": mapping_id,
|
||||
"remote_knowledge_base_id": mapping["weknora_id"],
|
||||
"knowledge_base_name": mapping["name"],
|
||||
},
|
||||
)
|
||||
return await _store(request).import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key=mapping_id,
|
||||
source_name=str(mapping["name"]),
|
||||
trigger="manual_from_weknora",
|
||||
created_by=actor,
|
||||
mapping_id=mapping_id,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/sources/{source_id}/reimport", status_code=202)
|
||||
async def reimport_source(request: Request, source_id: str) -> dict[str, Any]:
|
||||
actor = await _actor(request, admin=True)
|
||||
store = _store(request)
|
||||
source = await store.get_source(source_id)
|
||||
if source is None:
|
||||
raise HTTPException(status_code=404, detail="来源不存在")
|
||||
if source["source_type"] == "skill":
|
||||
base = await store.initialize(created_by=actor)
|
||||
job = await request.app.state.skill_knowledge_store.create_job(
|
||||
skill_names=[str(source.get("skill_name") or source["source_key"])],
|
||||
targets=[
|
||||
{
|
||||
"target_type": "assistant",
|
||||
"target_mode": "assistant",
|
||||
"target_id": base["id"],
|
||||
"target_name": base["name"],
|
||||
}
|
||||
],
|
||||
created_by=actor,
|
||||
force=True,
|
||||
review_mode="auto_high_confidence",
|
||||
trigger="assistant_source_reimport",
|
||||
)
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return job
|
||||
if source["source_type"] != "weknora":
|
||||
raise HTTPException(status_code=409, detail="本地文件来源未保留原文件,请重新上传")
|
||||
mapping_id = str(source.get("knowledge_base_mapping_id") or source["source_key"])
|
||||
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
|
||||
if not runtime.weknora_enabled:
|
||||
raise HTTPException(status_code=503, detail="未配置普通知识库服务")
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(
|
||||
mapping_id,
|
||||
actor or "system",
|
||||
write=False,
|
||||
is_admin=True,
|
||||
)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail="来源普通知识库不存在")
|
||||
package = await _export_weknora_package(
|
||||
build_weknora_client(runtime),
|
||||
str(mapping["weknora_id"]),
|
||||
{
|
||||
"type": "weknora",
|
||||
"knowledge_base_id": mapping_id,
|
||||
"remote_knowledge_base_id": mapping["weknora_id"],
|
||||
"knowledge_base_name": mapping["name"],
|
||||
},
|
||||
)
|
||||
return await store.import_package(
|
||||
package=package,
|
||||
source_type="weknora",
|
||||
source_key=mapping_id,
|
||||
source_name=str(mapping["name"]),
|
||||
trigger="source_reimport",
|
||||
created_by=actor,
|
||||
mapping_id=mapping_id,
|
||||
)
|
||||
@@ -0,0 +1,334 @@
|
||||
"""Administrator APIs for skill knowledge bindings and matrix jobs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Literal
|
||||
|
||||
from croniter import croniter
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from app.gateway.deps import get_optional_user_from_request
|
||||
from app.gateway.skill_knowledge_job_executor import _skill_directory
|
||||
from deerflow.config.system_settings import (
|
||||
SkillKnowledgeAutoSyncSettings,
|
||||
load_system_settings,
|
||||
save_system_settings,
|
||||
)
|
||||
from deerflow.integrations.weknora.runtime import (
|
||||
build_weknora_client,
|
||||
get_resolved_llmwiki_runtime,
|
||||
)
|
||||
from deerflow.skill_knowledge import scan_skill
|
||||
|
||||
router = APIRouter(prefix="/api/skill-knowledge", tags=["skill-knowledge"])
|
||||
|
||||
|
||||
async def _require_admin(request: Request) -> str | None:
|
||||
user = await get_optional_user_from_request(request)
|
||||
if user is None:
|
||||
return None
|
||||
if getattr(user, "system_role", None) != "admin":
|
||||
raise HTTPException(status_code=403, detail="技能知识归纳仅管理员可操作")
|
||||
return str(user.id)
|
||||
|
||||
|
||||
def _store(request: Request):
|
||||
value = getattr(request.app.state, "skill_knowledge_store", None)
|
||||
if value is None:
|
||||
raise HTTPException(status_code=503, detail="技能知识归纳存储不可用")
|
||||
return value
|
||||
|
||||
|
||||
class SyncTarget(BaseModel):
|
||||
target_type: Literal["weknora", "assistant"]
|
||||
target_mode: Literal["wiki", "document", "faq", "conversation_archive", "readonly_sidecar", "assistant"]
|
||||
target_id: str = Field(min_length=1, max_length=128)
|
||||
target_name: str = Field(default="", max_length=255)
|
||||
remote_business_key: str | None = Field(default=None, max_length=512)
|
||||
readonly: bool = False
|
||||
|
||||
|
||||
class CreateSyncJobRequest(BaseModel):
|
||||
skill_names: list[str] = Field(min_length=1, max_length=200)
|
||||
targets: list[SyncTarget] = Field(min_length=1, max_length=50)
|
||||
force: bool = False
|
||||
review_mode: Literal["required", "auto_high_confidence", "off"] = "auto_high_confidence"
|
||||
|
||||
@field_validator("skill_names")
|
||||
@classmethod
|
||||
def validate_names(cls, values: list[str]) -> list[str]:
|
||||
result = list(dict.fromkeys(value.strip() for value in values if value.strip()))
|
||||
if not result:
|
||||
raise ValueError("At least one skill is required")
|
||||
return result
|
||||
|
||||
|
||||
class ReviewRequest(BaseModel):
|
||||
review_ids: list[str] = Field(min_length=1, max_length=500)
|
||||
action: Literal["approve", "reject"]
|
||||
edits: dict[str, dict[str, Any]] = Field(default_factory=dict)
|
||||
reason: str | None = Field(default=None, max_length=1024)
|
||||
|
||||
|
||||
class DetachRequest(BaseModel):
|
||||
delete_contributions: bool
|
||||
|
||||
|
||||
class AutoSyncSettingsUpdate(BaseModel):
|
||||
watch_enabled: bool | None = None
|
||||
schedule_enabled: bool | None = None
|
||||
schedule_cron: str | None = Field(default=None, min_length=5, max_length=128)
|
||||
debounce_seconds: int | None = Field(default=None, ge=10, le=86_400)
|
||||
write_remote_enabled: bool | None = None
|
||||
review_mode: Literal["required", "auto_high_confidence", "off"] | None = None
|
||||
|
||||
@field_validator("schedule_cron")
|
||||
@classmethod
|
||||
def validate_cron(cls, value: str | None) -> str | None:
|
||||
if value is not None and not croniter.is_valid(value):
|
||||
raise ValueError("无效的 cron 表达式")
|
||||
return value
|
||||
|
||||
|
||||
@router.get("/bindings")
|
||||
async def list_bindings(
|
||||
request: Request,
|
||||
skill_names: list[str] | None = Query(default=None),
|
||||
target_id: str | None = Query(default=None),
|
||||
include_detached: bool = Query(default=False),
|
||||
) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
rows = await _store(request).list_bindings(skill_names=skill_names, target_id=target_id, include_detached=include_detached)
|
||||
return {"bindings": rows}
|
||||
|
||||
|
||||
@router.get("/source-status")
|
||||
async def source_status(request: Request, skill_names: list[str] = Query(...)) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
bindings = await _store(request).list_bindings(skill_names=skill_names)
|
||||
by_skill: dict[str, dict[str, Any]] = {}
|
||||
for name in list(dict.fromkeys(skill_names)):
|
||||
try:
|
||||
root = _skill_directory(request.app.state.config, name)
|
||||
result = await asyncio.to_thread(scan_skill, name, root)
|
||||
by_skill[name] = {"source_digest": result.source_digest, "counts": result.counts, "blocked_files": result.blocked_files}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
by_skill[name] = {"error": str(exc)}
|
||||
for binding in bindings:
|
||||
status = by_skill.get(binding["skill_name"])
|
||||
if status and status.get("source_digest") and status["source_digest"] != binding.get("synced_digest"):
|
||||
binding["computed_status"] = "stale"
|
||||
return {"skills": by_skill, "bindings": bindings}
|
||||
|
||||
|
||||
@router.get("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
|
||||
async def get_auto_sync_settings(request: Request) -> SkillKnowledgeAutoSyncSettings:
|
||||
await _require_admin(request)
|
||||
return load_system_settings().skill_knowledge_auto_sync
|
||||
|
||||
|
||||
@router.put("/auto-sync-settings", response_model=SkillKnowledgeAutoSyncSettings)
|
||||
async def update_auto_sync_settings(request: Request, body: AutoSyncSettingsUpdate) -> SkillKnowledgeAutoSyncSettings:
|
||||
await _require_admin(request)
|
||||
current = load_system_settings()
|
||||
updated = current.skill_knowledge_auto_sync.model_copy(update=body.model_dump(exclude_unset=True))
|
||||
saved = save_system_settings(current.model_copy(update={"skill_knowledge_auto_sync": updated}))
|
||||
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
|
||||
if service is not None:
|
||||
service.nudge()
|
||||
return saved.skill_knowledge_auto_sync
|
||||
|
||||
|
||||
async def _set_binding_auto_sync(request: Request, binding_id: str, *, enabled: bool) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
row = await _store(request).set_binding_auto_sync(binding_id, enabled=enabled)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
service = getattr(request.app.state, "skill_knowledge_auto_sync", None)
|
||||
if service is not None:
|
||||
service.nudge()
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/enable-auto-sync")
|
||||
async def enable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
|
||||
return await _set_binding_auto_sync(request, binding_id, enabled=True)
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/disable-auto-sync")
|
||||
async def disable_binding_auto_sync(request: Request, binding_id: str) -> dict[str, Any]:
|
||||
return await _set_binding_auto_sync(request, binding_id, enabled=False)
|
||||
|
||||
|
||||
@router.post("/sync-jobs", status_code=202)
|
||||
async def create_sync_job(request: Request, body: CreateSyncJobRequest) -> dict[str, Any]:
|
||||
actor = await _require_admin(request)
|
||||
skill_store = request.app.state.skill_store
|
||||
missing = [name for name in body.skill_names if await skill_store.get_any(name) is None]
|
||||
if missing:
|
||||
raise HTTPException(status_code=404, detail=f"技能不存在:{', '.join(missing[:10])}")
|
||||
targets: list[dict[str, Any]] = []
|
||||
for target in body.targets:
|
||||
data = target.model_dump()
|
||||
if target.target_type == "weknora":
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(target.target_id, actor or "system", write=True, is_admin=True)
|
||||
if mapping is None:
|
||||
raise HTTPException(status_code=404, detail=f"普通知识库不存在:{target.target_id}")
|
||||
data["target_name"] = target.target_name or str(mapping.get("name") or target.target_id)
|
||||
data["remote_business_key"] = target.remote_business_key or f"weknora:kb:{mapping['weknora_id']}"
|
||||
else:
|
||||
base = await request.app.state.assistant_knowledge_store.get_base()
|
||||
if base is None:
|
||||
base = await request.app.state.assistant_knowledge_store.initialize(created_by=actor)
|
||||
if target.target_id not in {base["id"], "global"}:
|
||||
raise HTTPException(status_code=404, detail="助手知识库仅允许选择全局知识梳理库")
|
||||
data["target_id"] = base["id"]
|
||||
data["target_name"] = base["name"]
|
||||
data["remote_business_key"] = f"assistant:base:{base['id']}"
|
||||
targets.append(data)
|
||||
job = await _store(request).create_job(
|
||||
skill_names=body.skill_names,
|
||||
targets=targets,
|
||||
created_by=actor,
|
||||
force=body.force,
|
||||
review_mode=body.review_mode,
|
||||
)
|
||||
dispatcher = getattr(request.app.state, "skill_knowledge_dispatcher", None)
|
||||
if dispatcher is not None:
|
||||
dispatcher.nudge()
|
||||
return job
|
||||
|
||||
|
||||
@router.get("/sync-jobs")
|
||||
async def list_sync_jobs(request: Request, limit: int = Query(default=50, ge=1, le=200)) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
return {"jobs": await _store(request).list_jobs(limit=limit)}
|
||||
|
||||
|
||||
@router.get("/sync-jobs/{job_id}")
|
||||
async def get_sync_job(request: Request, job_id: str) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
row = await _store(request).get_job(job_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="归纳任务不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/sync-jobs/{job_id}/cancel")
|
||||
async def cancel_sync_job(request: Request, job_id: str) -> dict[str, bool]:
|
||||
await _require_admin(request)
|
||||
if not await _store(request).cancel_job(job_id):
|
||||
raise HTTPException(status_code=404, detail="归纳任务不存在")
|
||||
return {"canceled": True}
|
||||
|
||||
|
||||
@router.post("/sync-items/{item_id}/retry", status_code=202)
|
||||
async def retry_sync_item(request: Request, item_id: str) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
row = await _store(request).requeue_item(item_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="归纳任务项不存在")
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return row
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/resync", status_code=202)
|
||||
async def resync_binding(request: Request, binding_id: str) -> dict[str, Any]:
|
||||
actor = await _require_admin(request)
|
||||
binding = await _store(request).get_binding(binding_id)
|
||||
if binding is None or not binding.get("active"):
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
target = {
|
||||
"target_type": binding["target_type"],
|
||||
"target_mode": binding["target_mode"],
|
||||
"target_id": binding["target_id"],
|
||||
"target_name": binding["target_name"],
|
||||
"remote_business_key": binding.get("remote_business_key"),
|
||||
}
|
||||
job = await _store(request).create_job(
|
||||
skill_names=[binding["skill_name"]],
|
||||
targets=[target],
|
||||
created_by=actor,
|
||||
force=True,
|
||||
review_mode="auto_high_confidence",
|
||||
)
|
||||
request.app.state.skill_knowledge_dispatcher.nudge()
|
||||
return job
|
||||
|
||||
|
||||
@router.post("/bindings/{binding_id}/detach")
|
||||
async def detach_binding(request: Request, binding_id: str, body: DetachRequest) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
store = _store(request)
|
||||
binding = await store.get_binding(binding_id)
|
||||
if binding is None:
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
if await store.has_active_item(binding_id):
|
||||
raise HTTPException(status_code=409, detail="该绑定仍有运行中的归纳项,请先取消任务")
|
||||
cleanup_failed = False
|
||||
if body.delete_contributions:
|
||||
try:
|
||||
if binding["target_type"] == "assistant":
|
||||
sources = await request.app.state.assistant_knowledge_store.list_sources()
|
||||
for source in sources:
|
||||
if source.get("source_type") == "skill" and source.get("source_key") == binding["skill_name"]:
|
||||
await request.app.state.assistant_knowledge_store.delete_source(source["id"])
|
||||
elif binding["target_type"] == "weknora":
|
||||
mapping = await request.app.state.llmwiki_store.get_authorized(
|
||||
str(binding["target_id"]),
|
||||
"system",
|
||||
write=True,
|
||||
is_admin=True,
|
||||
)
|
||||
runtime = get_resolved_llmwiki_runtime(request.app.state.config)
|
||||
if mapping is None or not runtime.weknora_enabled:
|
||||
raise RuntimeError("远端普通知识库当前不可写")
|
||||
client = build_weknora_client(runtime)
|
||||
remote_kb_id = str(mapping["weknora_id"])
|
||||
for artifact in await store.list_artifacts(binding_id):
|
||||
if artifact["kind"] == "wiki_page" and artifact.get("slug"):
|
||||
await client.delete_wiki_page(remote_kb_id, artifact["slug"])
|
||||
elif artifact.get("remote_id") and artifact["kind"] in {
|
||||
"raw_source",
|
||||
"summary_document",
|
||||
"faq_projection",
|
||||
"conversation_archive",
|
||||
"graph_projection",
|
||||
}:
|
||||
await client.delete_document(str(artifact["remote_id"]))
|
||||
except Exception: # noqa: BLE001
|
||||
cleanup_failed = True
|
||||
row = await store.detach_binding(
|
||||
binding_id,
|
||||
delete_contributions=body.delete_contributions,
|
||||
cleanup_failed=cleanup_failed,
|
||||
)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="绑定不存在")
|
||||
return row
|
||||
|
||||
|
||||
@router.get("/reviews")
|
||||
async def list_reviews(
|
||||
request: Request,
|
||||
job_id: str | None = Query(default=None),
|
||||
item_id: str | None = Query(default=None),
|
||||
status: str | None = Query(default="pending"),
|
||||
) -> dict[str, Any]:
|
||||
await _require_admin(request)
|
||||
return {"reviews": await _store(request).list_reviews(job_id=job_id, item_id=item_id, status=status)}
|
||||
|
||||
|
||||
@router.post("/reviews/actions")
|
||||
async def review_actions(request: Request, body: ReviewRequest) -> dict[str, Any]:
|
||||
actor = await _require_admin(request)
|
||||
item_ids = await _store(request).review(body.review_ids, action=body.action, actor=actor, edits=body.edits, reason=body.reason)
|
||||
resumed: list[str] = []
|
||||
executor = request.app.state.skill_knowledge_executor
|
||||
for item_id in item_ids:
|
||||
if await _store(request).pending_review_count(item_id) == 0:
|
||||
asyncio.create_task(executor.resume_reviewed_item(item_id))
|
||||
resumed.append(item_id)
|
||||
return {"updated": len(body.review_ids), "resumed_item_ids": resumed}
|
||||
@@ -44,6 +44,14 @@ class PromptPrefixResponse(BaseModel):
|
||||
prompt_prefix: str = Field(..., description="Prefix text prepended to the first user message")
|
||||
|
||||
|
||||
class KnowledgeBaseProductVersionResponse(BaseModel):
|
||||
version: Literal["v1", "v2"] = "v1"
|
||||
|
||||
|
||||
class KnowledgeBaseProductVersionUpdateRequest(BaseModel):
|
||||
version: Literal["v1", "v2"]
|
||||
|
||||
|
||||
class PromptPrefixUpdateRequest(BaseModel):
|
||||
enabled: bool | None = None
|
||||
switch_label: str | None = None
|
||||
@@ -167,6 +175,21 @@ async def update_prompt_prefix(body: PromptPrefixUpdateRequest, request: Request
|
||||
return _to_response(saved.prompt_prefix)
|
||||
|
||||
|
||||
@router.get("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
|
||||
async def get_knowledge_base_product_version() -> KnowledgeBaseProductVersionResponse:
|
||||
return KnowledgeBaseProductVersionResponse(version=load_system_settings().knowledge_base_product_version)
|
||||
|
||||
|
||||
@router.put("/knowledge-base-product-version", response_model=KnowledgeBaseProductVersionResponse)
|
||||
async def update_knowledge_base_product_version(
|
||||
body: KnowledgeBaseProductVersionUpdateRequest,
|
||||
request: Request,
|
||||
) -> KnowledgeBaseProductVersionResponse:
|
||||
await _require_admin(request)
|
||||
saved = save_system_settings(load_system_settings().model_copy(update={"knowledge_base_product_version": body.version}))
|
||||
return KnowledgeBaseProductVersionResponse(version=saved.knowledge_base_product_version)
|
||||
|
||||
|
||||
@router.get("/plugin-management", response_model=PluginManagementResponse)
|
||||
async def get_plugin_management() -> PluginManagementResponse:
|
||||
"""Return the iframe plugin domain -> agent mappings."""
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Debounced file watcher and cron scheduler for opted-in knowledge bindings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from croniter import croniter
|
||||
|
||||
from app.gateway.skill_knowledge_job_executor import _skill_directory
|
||||
from deerflow.config.system_settings import load_system_settings
|
||||
from deerflow.skill_knowledge import scan_skill
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkillKnowledgeAutoSyncService:
|
||||
def __init__(self, app: Any, *, poll_seconds: float = 10.0) -> None:
|
||||
self.app = app
|
||||
self.store = app.state.skill_knowledge_store
|
||||
self.poll_seconds = poll_seconds
|
||||
self._task: asyncio.Task | None = None
|
||||
self._wake = asyncio.Event()
|
||||
self._observed: dict[str, tuple[str, datetime]] = {}
|
||||
self._schedule_expression = ""
|
||||
self._next_schedule: datetime | None = None
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._loop())
|
||||
|
||||
def nudge(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._task is None:
|
||||
return
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _queue_skill(
|
||||
self,
|
||||
skill_name: str,
|
||||
bindings: list[dict[str, Any]],
|
||||
*,
|
||||
trigger: str,
|
||||
review_mode: str,
|
||||
) -> None:
|
||||
targets = [
|
||||
{
|
||||
"target_type": binding["target_type"],
|
||||
"target_mode": binding["target_mode"],
|
||||
"target_id": binding["target_id"],
|
||||
"target_name": binding["target_name"],
|
||||
"remote_business_key": binding.get("remote_business_key"),
|
||||
}
|
||||
for binding in bindings
|
||||
]
|
||||
job = await self.store.create_job(
|
||||
skill_names=[skill_name],
|
||||
targets=targets,
|
||||
created_by=None,
|
||||
trigger=trigger,
|
||||
review_mode=review_mode,
|
||||
)
|
||||
if job["total_count"]:
|
||||
self.app.state.skill_knowledge_dispatcher.nudge()
|
||||
|
||||
async def _watch(
|
||||
self,
|
||||
bindings_by_skill: dict[str, list[dict[str, Any]]],
|
||||
*,
|
||||
debounce_seconds: int,
|
||||
write_remote_enabled: bool,
|
||||
review_mode: str,
|
||||
) -> None:
|
||||
now = datetime.now(UTC)
|
||||
for skill_name, bindings in bindings_by_skill.items():
|
||||
try:
|
||||
root = _skill_directory(self.app.state.config, skill_name)
|
||||
scan = await asyncio.to_thread(scan_skill, skill_name, root)
|
||||
except Exception:
|
||||
logger.exception("Failed to fingerprint skill %s", skill_name)
|
||||
continue
|
||||
synced = {str(binding.get("synced_digest") or "") for binding in bindings}
|
||||
if synced == {scan.source_digest}:
|
||||
self._observed.pop(skill_name, None)
|
||||
continue
|
||||
await self.store.mark_skill_stale(skill_name)
|
||||
previous = self._observed.get(skill_name)
|
||||
if previous is None or previous[0] != scan.source_digest:
|
||||
self._observed[skill_name] = (scan.source_digest, now)
|
||||
continue
|
||||
if (now - previous[1]).total_seconds() < debounce_seconds:
|
||||
continue
|
||||
if write_remote_enabled:
|
||||
current = await self.store.list_bindings(skill_names=[skill_name])
|
||||
stale = [binding for binding in current if binding.get("auto_sync_enabled") and binding.get("status") == "stale"]
|
||||
if stale:
|
||||
await self._queue_skill(
|
||||
skill_name,
|
||||
stale,
|
||||
trigger="watcher",
|
||||
review_mode=review_mode,
|
||||
)
|
||||
self._observed.pop(skill_name, None)
|
||||
|
||||
def _schedule_due(self, expression: str, now: datetime) -> bool:
|
||||
if expression != self._schedule_expression or self._next_schedule is None:
|
||||
self._schedule_expression = expression
|
||||
self._next_schedule = croniter(expression, now).get_next(datetime)
|
||||
return False
|
||||
if now < self._next_schedule:
|
||||
return False
|
||||
self._next_schedule = croniter(expression, now).get_next(datetime)
|
||||
return True
|
||||
|
||||
async def _tick(self) -> None:
|
||||
settings = (await asyncio.to_thread(load_system_settings)).skill_knowledge_auto_sync
|
||||
bindings = [row for row in await self.store.list_bindings() if row.get("auto_sync_enabled")]
|
||||
by_skill: dict[str, list[dict[str, Any]]] = {}
|
||||
for binding in bindings:
|
||||
by_skill.setdefault(str(binding["skill_name"]), []).append(binding)
|
||||
if settings.watch_enabled:
|
||||
await self._watch(
|
||||
by_skill,
|
||||
debounce_seconds=settings.debounce_seconds,
|
||||
write_remote_enabled=settings.write_remote_enabled,
|
||||
review_mode=settings.review_mode,
|
||||
)
|
||||
if settings.schedule_enabled and settings.write_remote_enabled and by_skill and self._schedule_due(settings.schedule_cron, datetime.now(UTC)):
|
||||
for skill_name, skill_bindings in by_skill.items():
|
||||
await self._queue_skill(
|
||||
skill_name,
|
||||
skill_bindings,
|
||||
trigger="scheduled",
|
||||
review_mode=settings.review_mode,
|
||||
)
|
||||
|
||||
async def _loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
await self._tick()
|
||||
except Exception:
|
||||
logger.exception("Skill knowledge automatic sync tick failed")
|
||||
try:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=self.poll_seconds)
|
||||
except TimeoutError:
|
||||
pass
|
||||
self._wake.clear()
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Small persistent-queue dispatcher for skill knowledge items."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SkillKnowledgeJobDispatcher:
|
||||
def __init__(
|
||||
self,
|
||||
store: Any,
|
||||
executor: Any,
|
||||
*,
|
||||
interval_seconds: float = 2.0,
|
||||
concurrency: int = 3,
|
||||
lease_seconds: int = 900,
|
||||
) -> None:
|
||||
self.store = store
|
||||
self.executor = executor
|
||||
self.interval_seconds = interval_seconds
|
||||
self.lease_seconds = lease_seconds
|
||||
self.owner = f"{socket.gethostname()}:{uuid4()}"
|
||||
self.semaphore = asyncio.Semaphore(concurrency)
|
||||
self._wake = asyncio.Event()
|
||||
self._task: asyncio.Task | None = None
|
||||
self._running: set[str] = set()
|
||||
|
||||
def start(self) -> None:
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._loop())
|
||||
|
||||
def nudge(self) -> None:
|
||||
self._wake.set()
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._task is not None:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def _run(self, item: dict) -> None:
|
||||
item_id = str(item["id"])
|
||||
async with self.semaphore:
|
||||
heartbeat = asyncio.create_task(self._heartbeat(item_id))
|
||||
try:
|
||||
await self.executor.run_item(item)
|
||||
finally:
|
||||
heartbeat.cancel()
|
||||
try:
|
||||
await heartbeat
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._running.discard(item_id)
|
||||
self._wake.set()
|
||||
|
||||
async def _heartbeat(self, item_id: str) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(min(30, max(5, self.lease_seconds // 3)))
|
||||
if not await self.store.renew_lease(
|
||||
item_id,
|
||||
owner=self.owner,
|
||||
lease_seconds=self.lease_seconds,
|
||||
):
|
||||
return
|
||||
|
||||
async def _loop(self) -> None:
|
||||
while True:
|
||||
try:
|
||||
for item in await self.store.list_queued_items(limit=20):
|
||||
item_id = str(item["id"])
|
||||
if item_id in self._running:
|
||||
continue
|
||||
claimed = await self.store.claim_item(
|
||||
item_id,
|
||||
owner=self.owner,
|
||||
lease_seconds=self.lease_seconds,
|
||||
)
|
||||
if claimed is None:
|
||||
continue
|
||||
self._running.add(item_id)
|
||||
asyncio.create_task(self._run(claimed))
|
||||
except Exception:
|
||||
logger.exception("Skill knowledge dispatcher tick failed")
|
||||
try:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=self.interval_seconds)
|
||||
except TimeoutError:
|
||||
pass
|
||||
self._wake.clear()
|
||||
@@ -0,0 +1,403 @@
|
||||
"""Background executor for one skill x target matrix item."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from deerflow.assistant_knowledge import export_package_from_skill_ir
|
||||
from deerflow.integrations.weknora.client import WeKnoraError
|
||||
from deerflow.integrations.weknora.runtime import build_weknora_client, get_resolved_llmwiki_runtime
|
||||
from deerflow.skill_knowledge import build_skill_ir, enrich_converted_documents, scan_skill
|
||||
from deerflow.skill_knowledge.graph_writer import parameterized_statements, plan_from_ir
|
||||
from deerflow.skills.parser import parse_skill_file
|
||||
from deerflow.skills.storage import get_or_new_skill_storage
|
||||
from deerflow.skills.types import SkillCategory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _skill_directory(app_config: Any, skill_name: str) -> Path:
|
||||
storage = get_or_new_skill_storage(app_config=app_config)
|
||||
root = storage.get_skills_root_path().resolve()
|
||||
for category in SkillCategory:
|
||||
direct = root / category.value / skill_name
|
||||
if (direct / "SKILL.md").is_file():
|
||||
return direct
|
||||
for category in SkillCategory:
|
||||
category_root = root / category.value
|
||||
if not category_root.is_dir():
|
||||
continue
|
||||
for skill_md in category_root.rglob("SKILL.md"):
|
||||
if skill_md.is_symlink():
|
||||
continue
|
||||
parsed = parse_skill_file(skill_md, category=category, relative_path=skill_md.parent.relative_to(category_root))
|
||||
if parsed and parsed.name == skill_name:
|
||||
return skill_md.parent
|
||||
raise FileNotFoundError(f"Skill '{skill_name}' not found under running skill roots")
|
||||
|
||||
|
||||
def _graph_projection(ir: dict[str, Any]) -> str:
|
||||
rows = []
|
||||
entities = {str(item.get("local_id")): item for item in ir.get("entities", [])}
|
||||
for relation in ir.get("relations", []):
|
||||
if float(relation.get("confidence") or 0) < 0.75 or not relation.get("evidence"):
|
||||
continue
|
||||
source = entities.get(str(relation.get("source_local_id")), {})
|
||||
target = entities.get(str(relation.get("target_local_id")), {})
|
||||
rows.append(
|
||||
json.dumps(
|
||||
{
|
||||
"source": source.get("name"),
|
||||
"source_type": source.get("type"),
|
||||
"predicate": relation.get("predicate"),
|
||||
"target": target.get("name"),
|
||||
"target_type": target.get("type"),
|
||||
"description": relation.get("description"),
|
||||
"confidence": relation.get("confidence"),
|
||||
"evidence": relation.get("evidence"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
class SkillKnowledgeJobExecutor:
|
||||
def __init__(self, app: Any) -> None:
|
||||
self.app = app
|
||||
self.store = app.state.skill_knowledge_store
|
||||
|
||||
async def _cancel_if_requested(self, item: dict[str, Any]) -> bool:
|
||||
job = await self.store.get_job(str(item["job_id"]))
|
||||
if not (job or {}).get("cancel_requested"):
|
||||
return False
|
||||
await self.store.update_item(
|
||||
str(item["id"]),
|
||||
status="canceled",
|
||||
phase="completed",
|
||||
progress=100,
|
||||
)
|
||||
return True
|
||||
|
||||
async def _assert_source_unchanged(self, item: dict[str, Any], expected_digest: str) -> None:
|
||||
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
|
||||
current = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
current = await enrich_converted_documents(current)
|
||||
if current.source_digest == expected_digest:
|
||||
return
|
||||
await self.store.mark_skill_stale(str(item["skill_name"]))
|
||||
raise RuntimeError("技能知识文件在归纳期间发生变化,旧版本保持激活,请重试")
|
||||
|
||||
async def _write_direct_graph(
|
||||
self,
|
||||
item: dict[str, Any],
|
||||
snapshot_id: str,
|
||||
ir: dict[str, Any],
|
||||
) -> dict[str, Any] | None:
|
||||
config = self.app.state.config.llmwiki.weknora.graph
|
||||
if not config.direct_write_enabled:
|
||||
return None
|
||||
if not config.neo4j_http_url or not config.username or not config.password:
|
||||
raise RuntimeError("Neo4j direct writer 已开启但后端连接配置不完整")
|
||||
plan = plan_from_ir(
|
||||
ir,
|
||||
target_type=str(item["target_type"]),
|
||||
target_id=str(item["target_id"]),
|
||||
binding_id=str(item["binding_id"]),
|
||||
snapshot_id=snapshot_id,
|
||||
job_id=str(item["job_id"]),
|
||||
)
|
||||
statements = parameterized_statements(plan)
|
||||
if not statements:
|
||||
return {"nodes": 0, "edges": 0, "statement_count": 0}
|
||||
endpoint = f"{config.neo4j_http_url.rstrip('/')}/db/{config.database}/tx/commit"
|
||||
async with httpx.AsyncClient(
|
||||
auth=(config.username, config.password),
|
||||
timeout=config.timeout_seconds,
|
||||
) as client:
|
||||
response = await client.post(endpoint, json={"statements": statements})
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
errors = payload.get("errors") or []
|
||||
if errors:
|
||||
raise RuntimeError(f"Neo4j transaction failed: {str(errors[0].get('code') or 'unknown')}")
|
||||
return {
|
||||
"nodes": len(plan.nodes),
|
||||
"edges": len(plan.edges),
|
||||
"statement_count": len(statements),
|
||||
"scope": plan.scope,
|
||||
}
|
||||
|
||||
async def run_item(self, item: dict[str, Any]) -> None:
|
||||
item_id = str(item["id"])
|
||||
try:
|
||||
await self.store.update_item(item_id, status="running", phase="scanning", progress=5)
|
||||
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
|
||||
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
await self.store.update_item(item_id, phase="extracting", progress=30)
|
||||
skill_record = await self.app.state.skill_store.get_any(str(item["skill_name"]))
|
||||
ir = await asyncio.to_thread(
|
||||
build_skill_ir,
|
||||
scan,
|
||||
display_name=(skill_record or {}).get("name_zh"),
|
||||
description=str((skill_record or {}).get("description") or ""),
|
||||
)
|
||||
snapshot = await self.store.get_or_create_snapshot(
|
||||
skill_name=str(item["skill_name"]),
|
||||
source_digest=scan.source_digest,
|
||||
extractor_version=str(ir["extractor_version"]),
|
||||
prompt_hash=str(ir["prompt_hash"]),
|
||||
model_name=str(ir["model_name"]),
|
||||
manifest=scan.manifest,
|
||||
ir=ir,
|
||||
counts=scan.counts,
|
||||
)
|
||||
snapshot_id = str(snapshot["id"])
|
||||
await self.store.update_item(item_id, phase="awaiting_review", progress=45, snapshot_id=snapshot_id)
|
||||
job = await self.store.get_job(str(item["job_id"]))
|
||||
review_mode = str((job or {}).get("review_mode") or "auto_high_confidence")
|
||||
review_objects: list[tuple[str, dict]] = []
|
||||
if review_mode == "required":
|
||||
review_objects = [("entity", obj) for obj in ir.get("entities", [])] + [("relation", obj) for obj in ir.get("relations", [])]
|
||||
elif review_mode == "auto_high_confidence":
|
||||
review_objects = [(kind, obj) for kind, rows in (("entity", ir.get("entities", [])), ("relation", ir.get("relations", []))) for obj in rows if float(obj.get("confidence") or 0) < 0.9]
|
||||
if review_objects:
|
||||
existing = await self.store.list_reviews(item_id=item_id, status=None)
|
||||
if not existing:
|
||||
await self.store.create_reviews(
|
||||
job_id=str(item["job_id"]),
|
||||
item_id=item_id,
|
||||
binding_id=str(item["binding_id"]),
|
||||
snapshot_id=snapshot_id,
|
||||
objects=review_objects,
|
||||
)
|
||||
await self.store.update_item(item_id, status="awaiting_review", phase="awaiting_review", progress=50)
|
||||
return
|
||||
await self._project_and_activate(item, snapshot_id, ir, scan)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Skill knowledge item failed: item=%s", item_id)
|
||||
await self.store.update_item(
|
||||
item_id,
|
||||
status="failed",
|
||||
phase="completed",
|
||||
progress=100,
|
||||
error_code=type(exc).__name__,
|
||||
error_message=str(exc)[:2000],
|
||||
)
|
||||
|
||||
async def resume_reviewed_item(self, item_id: str) -> None:
|
||||
if await self.store.pending_review_count(item_id):
|
||||
return
|
||||
jobs = await self.store.list_jobs(limit=100)
|
||||
item = None
|
||||
for job in jobs:
|
||||
detail = await self.store.get_job(str(job["id"]))
|
||||
item = next((row for row in (detail or {}).get("items", []) if row["id"] == item_id), None)
|
||||
if item:
|
||||
break
|
||||
if not item or not item.get("snapshot_id"):
|
||||
return
|
||||
snapshot = await self.store.get_snapshot(str(item["snapshot_id"]))
|
||||
ir = await self.store.effective_ir(str(item["snapshot_id"]), item_id)
|
||||
if not snapshot or not ir:
|
||||
return
|
||||
try:
|
||||
await self.store.update_item(item_id, status="running", phase="merging", progress=55)
|
||||
await self._project_and_activate(item, str(item["snapshot_id"]), ir, None)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Reviewed skill knowledge item failed: item=%s", item_id)
|
||||
await self.store.update_item(
|
||||
item_id,
|
||||
status="failed",
|
||||
phase="completed",
|
||||
progress=100,
|
||||
error_code=type(exc).__name__,
|
||||
error_message=str(exc)[:2000],
|
||||
)
|
||||
|
||||
async def _project_and_activate(self, item: dict[str, Any], snapshot_id: str, ir: dict[str, Any], scan: Any | None) -> None:
|
||||
if await self._cancel_if_requested(item):
|
||||
return
|
||||
expected_digest = str(ir.get("skill", {}).get("source_digest") or "")
|
||||
if scan is None:
|
||||
root = _skill_directory(self.app.state.config, str(item["skill_name"]))
|
||||
scan = await asyncio.to_thread(scan_skill, str(item["skill_name"]), root)
|
||||
scan = await enrich_converted_documents(scan)
|
||||
await self._assert_source_unchanged(item, expected_digest)
|
||||
await self.store.update_item(str(item["id"]), phase="writing_wiki", progress=65)
|
||||
target_type = str(item["target_type"])
|
||||
artifacts: list[dict[str, Any]] = []
|
||||
binding_status = "synced"
|
||||
page = ir["wiki"]["skill_page"]
|
||||
if target_type == "assistant":
|
||||
package = export_package_from_skill_ir(ir)
|
||||
await self.app.state.assistant_knowledge_store.import_package(
|
||||
package=package,
|
||||
source_type="skill",
|
||||
source_key=str(item["skill_name"]),
|
||||
source_name=str(ir["skill"].get("display_name") or item["skill_name"]),
|
||||
trigger="skill_direct",
|
||||
created_by=(await self.store.get_job(str(item["job_id"]))).get("created_by"),
|
||||
skill_name=str(item["skill_name"]),
|
||||
)
|
||||
artifacts.append({"kind": "assistant_wiki_revision", "slug": page["slug"], "content": page["content"], "business_key": f"assistant:wiki:{page['slug']}"})
|
||||
elif str(item["target_mode"]) == "readonly_sidecar":
|
||||
artifacts.append({"kind": "sidecar_wiki", "slug": page["slug"], "content": page["content"], "business_key": f"sidecar:{item['target_id']}:{page['slug']}"})
|
||||
artifacts.append({"kind": "sidecar_graph", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "content": _graph_projection(ir)})
|
||||
binding_status = "pending_remote_write"
|
||||
else:
|
||||
mapping = await self.app.state.llmwiki_store.get_authorized(str(item["target_id"]), "system", write=True, is_admin=True)
|
||||
if mapping is None:
|
||||
raise ValueError("Target WeKnora mapping does not exist")
|
||||
runtime = get_resolved_llmwiki_runtime(self.app.state.config)
|
||||
if not runtime.weknora_enabled:
|
||||
raise RuntimeError("普通知识库服务未配置")
|
||||
client = build_weknora_client(runtime)
|
||||
remote_kb_id = str(mapping["weknora_id"])
|
||||
if scan is not None:
|
||||
for file in scan.files:
|
||||
source_path = Path(scan.source_root) / file.path
|
||||
if file.text:
|
||||
remote = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/source/{file.path}", content=file.text)
|
||||
else:
|
||||
remote = await client.upload_document(
|
||||
remote_kb_id,
|
||||
filename=f"{item['skill_name']}-{source_path.name}",
|
||||
content=await asyncio.to_thread(source_path.read_bytes),
|
||||
)
|
||||
remote_id = str(remote.get("id") or remote.get("knowledge_id") or "") or None
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "raw_source",
|
||||
"slug": file.path,
|
||||
"remote_id": remote_id,
|
||||
"business_key": f"weknora:document:{remote_id}" if remote_id else None,
|
||||
"remote_refs": [{"provider": "weknora", "remote_kind": "document", "remote_id": remote_id}] if remote_id else [],
|
||||
}
|
||||
)
|
||||
target_mode = str(item["target_mode"])
|
||||
if target_mode == "wiki":
|
||||
remote_page: dict[str, Any]
|
||||
try:
|
||||
await client.get_wiki_page(remote_kb_id, page["slug"])
|
||||
except WeKnoraError as exc:
|
||||
if exc.status_code != 404:
|
||||
raise
|
||||
remote_page = await client.create_wiki_page(
|
||||
remote_kb_id,
|
||||
{
|
||||
"slug": page["slug"],
|
||||
"title": page["title"],
|
||||
"content": page["content"],
|
||||
"summary": str(ir["skill"].get("description") or ""),
|
||||
"page_type": "skill",
|
||||
},
|
||||
)
|
||||
else:
|
||||
remote_page = await client.update_wiki_page(
|
||||
remote_kb_id,
|
||||
page["slug"],
|
||||
{
|
||||
"title": page["title"],
|
||||
"content": page["content"],
|
||||
"summary": str(ir["skill"].get("description") or ""),
|
||||
},
|
||||
)
|
||||
remote_page_id = str(remote_page.get("id") or remote_page.get("page_id") or page["slug"])
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "wiki_page",
|
||||
"slug": page["slug"],
|
||||
"content": page["content"],
|
||||
"remote_id": remote_page_id,
|
||||
"business_key": f"weknora:wiki_page:{remote_page_id}",
|
||||
"remote_refs": [
|
||||
{
|
||||
"provider": "weknora",
|
||||
"remote_kind": "wiki_page",
|
||||
"remote_id": remote_page_id,
|
||||
"slug": page["slug"],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
else:
|
||||
document_content = page["content"]
|
||||
artifact_kind = "summary_document"
|
||||
if target_mode == "faq":
|
||||
artifact_kind = "faq_projection"
|
||||
document_content = f"问题:{page['title']} 能解决什么问题,如何使用?\n\n回答:\n{page['content']}"
|
||||
elif target_mode == "conversation_archive":
|
||||
artifact_kind = "conversation_archive"
|
||||
document_content = f"# 技能归纳会话归档:{page['title']}\n\n以下内容由技能知识归纳任务生成。\n\n{page['content']}"
|
||||
remote_document = await client.create_manual_document(
|
||||
remote_kb_id,
|
||||
title=f"skills/{item['skill_name']}/{target_mode}.md",
|
||||
content=document_content,
|
||||
)
|
||||
remote_document_id = str(remote_document.get("id") or remote_document.get("knowledge_id") or "") or None
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": artifact_kind,
|
||||
"slug": f"skills/{item['skill_name']}/{target_mode}.md",
|
||||
"content": document_content,
|
||||
"remote_id": remote_document_id,
|
||||
"business_key": (f"weknora:document:{remote_document_id}" if remote_document_id else None),
|
||||
"remote_refs": (
|
||||
[
|
||||
{
|
||||
"provider": "weknora",
|
||||
"remote_kind": "document",
|
||||
"remote_id": remote_document_id,
|
||||
}
|
||||
]
|
||||
if remote_document_id
|
||||
else []
|
||||
),
|
||||
}
|
||||
)
|
||||
projection = await client.create_manual_document(remote_kb_id, title=f"skills/{item['skill_name']}/graph-projection.jsonl", content=_graph_projection(ir))
|
||||
projection_id = str(projection.get("id") or projection.get("knowledge_id") or "") or None
|
||||
artifacts.append({"kind": "graph_projection", "slug": f"skills/{item['skill_name']}/graph-projection.jsonl", "remote_id": projection_id})
|
||||
try:
|
||||
graph_audit = await self._write_direct_graph(item, snapshot_id, ir)
|
||||
if graph_audit is not None:
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "neo4j_direct_write_audit",
|
||||
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
|
||||
"content": json.dumps(graph_audit, ensure_ascii=False),
|
||||
"metadata": graph_audit,
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("Optional Neo4j direct graph write failed: item=%s", item["id"])
|
||||
binding_status = "partial_failed"
|
||||
artifacts.append(
|
||||
{
|
||||
"kind": "neo4j_direct_write_audit",
|
||||
"slug": f"skills/{item['skill_name']}/neo4j-audit.json",
|
||||
"content": json.dumps(
|
||||
{"status": "failed", "error": type(exc).__name__},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
"metadata": {
|
||||
"status": "failed",
|
||||
"error_type": type(exc).__name__,
|
||||
},
|
||||
}
|
||||
)
|
||||
await self._assert_source_unchanged(item, expected_digest)
|
||||
await self.store.update_item(str(item["id"]), phase="activating", progress=90)
|
||||
await self.store.stage_and_activate(item_id=str(item["id"]), snapshot_id=snapshot_id, ir=ir, artifacts=artifacts, status=binding_status)
|
||||
Reference in new issue
Block a user